From 7138229c8cf0992321ce2ace222b0d0f567efc6c Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 25 May 2026 10:46:31 +0800 Subject: [PATCH 001/138] feat: Add metadata system tables --- src/paimon/CMakeLists.txt | 7 +- .../prefetch_file_batch_reader_impl_test.cpp | 5 +- .../core/catalog/file_system_catalog.cpp | 26 +- .../core/catalog/file_system_catalog_test.cpp | 100 +++- src/paimon/core/core_options.cpp | 1 - .../generic_row_to_arrow_array_converter.cpp | 70 +++ .../io/generic_row_to_arrow_array_converter.h | 55 +++ src/paimon/core/schema/table_schema.h | 4 + .../table/system/audit_log_system_table.cpp | 4 +- .../table/system/audit_log_system_table.h | 2 + .../core/table/system/binlog_system_table.h | 1 + .../table/system/in_memory_system_table.cpp | 117 +++++ ...ystem_table.h => in_memory_system_table.h} | 22 +- .../table/system/metadata_system_tables.cpp | 430 ++++++++++++++++++ .../table/system/metadata_system_tables.h | 131 ++++++ .../table/system/options_system_table.cpp | 156 ------- src/paimon/core/table/system/system_table.cpp | 131 +++++- src/paimon/core/table/system/system_table.h | 10 + .../core/table/system/system_table_scan.h | 2 + .../core/table/system/system_table_schema.h | 1 + src/paimon/core/utils/branch_manager.cpp | 56 +++ src/paimon/core/utils/branch_manager.h | 15 + src/paimon/core/utils/consumer_manager.cpp | 123 +++++ src/paimon/core/utils/consumer_manager.h | 66 +++ .../core/utils/consumer_manager_test.cpp | 51 +++ src/paimon/core/utils/tag_manager.cpp | 35 +- src/paimon/core/utils/tag_manager.h | 5 + src/paimon/core/utils/tag_manager_test.cpp | 43 ++ test/inte/read_inte_test.cpp | 188 ++++++++ 29 files changed, 1642 insertions(+), 215 deletions(-) create mode 100644 src/paimon/core/io/generic_row_to_arrow_array_converter.cpp create mode 100644 src/paimon/core/io/generic_row_to_arrow_array_converter.h create mode 100644 src/paimon/core/table/system/in_memory_system_table.cpp rename src/paimon/core/table/system/{options_system_table.h => in_memory_system_table.h} (70%) create mode 100644 src/paimon/core/table/system/metadata_system_tables.cpp create mode 100644 src/paimon/core/table/system/metadata_system_tables.h delete mode 100644 src/paimon/core/table/system/options_system_table.cpp create mode 100644 src/paimon/core/utils/branch_manager.cpp create mode 100644 src/paimon/core/utils/consumer_manager.cpp create mode 100644 src/paimon/core/utils/consumer_manager.h create mode 100644 src/paimon/core/utils/consumer_manager_test.cpp diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 1c629386..5673a55d 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -203,6 +203,7 @@ set(PAIMON_CORE_SRCS core/index/index_file_handler.cpp core/index/global_index_meta.cpp core/index/index_file_meta_serializer.cpp + core/io/generic_row_to_arrow_array_converter.cpp core/io/meta_to_arrow_array_converter.cpp core/io/async_key_value_producer_and_consumer.cpp core/io/data_file_meta_09_serializer.cpp @@ -321,11 +322,14 @@ set(PAIMON_CORE_SRCS core/table/source/data_evolution_batch_scan.cpp core/table/system/audit_log_system_table.cpp core/table/system/binlog_system_table.cpp - core/table/system/options_system_table.cpp + core/table/system/in_memory_system_table.cpp + core/table/system/metadata_system_tables.cpp core/table/system/system_table.cpp core/table/system/system_table_scan.cpp core/table/system/system_table_schema.cpp core/tag/tag.cpp + core/utils/branch_manager.cpp + core/utils/consumer_manager.cpp core/utils/field_mapping.cpp core/utils/file_store_path_factory.cpp core/utils/file_utils.cpp @@ -708,6 +712,7 @@ if(PAIMON_BUILD_TESTS) core/table/system/system_table_test.cpp core/tag/tag_test.cpp core/utils/branch_manager_test.cpp + core/utils/consumer_manager_test.cpp core/utils/file_store_path_factory_cache_test.cpp core/utils/field_mapping_test.cpp core/utils/file_store_path_factory_test.cpp diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp index d9b58218..d2d6b5aa 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp @@ -18,6 +18,7 @@ */ #include "paimon/common/reader/prefetch_file_batch_reader_impl.h" +#include #include #include "arrow/compute/api.h" @@ -92,7 +93,7 @@ class ControlledMockFormatReaderBuilder : public ReaderBuilder { Result> Build( const std::shared_ptr& path) const override { - size_t index = build_count_++; + size_t index = build_count_.fetch_add(1); Status set_read_ranges_status = index < set_read_ranges_statuses_.size() ? set_read_ranges_statuses_[index] : Status::OK(); @@ -107,7 +108,7 @@ class ControlledMockFormatReaderBuilder : public ReaderBuilder { std::vector> read_ranges_; bool need_prefetch_ = true; std::vector set_read_ranges_statuses_; - mutable size_t build_count_ = 0; + mutable std::atomic build_count_{0}; }; struct TestParam { diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 277f0b8a..ee34275b 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -367,27 +367,11 @@ Result> FileSystemCatalog::GetSchemaExternalPaths( Result> FileSystemCatalog::GetTableBranches( const std::string& table_path) const { - std::vector branches; - std::string branch_dir = PathUtil::JoinPath(table_path, "branch"); - PAIMON_ASSIGN_OR_RAISE(bool branch_dir_exists, fs_->Exists(branch_dir)); - if (!branch_dir_exists) { - return branches; - } - - std::vector> file_status_list; - PAIMON_RETURN_NOT_OK(fs_->ListDir(branch_dir, &file_status_list)); - - for (const auto& file_status : file_status_list) { - if (file_status->IsDir()) { - std::string dir_name = PathUtil::GetName(file_status->GetPath()); - // Branch directory name format: branch-{branch_name} - const std::string branch_prefix = BranchManager::BRANCH_PREFIX; - if (StringUtils::StartsWith(dir_name, branch_prefix, /*start_pos=*/0)) { - std::string branch_name = dir_name.substr(branch_prefix.length()); - branches.push_back(branch_name); - } - } - } + PAIMON_ASSIGN_OR_RAISE(std::vector branches, + BranchManager::ListBranches(fs_, table_path)); + branches.erase( + std::remove(branches.begin(), branches.end(), BranchManager::DEFAULT_MAIN_BRANCH), + branches.end()); return branches; } diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 0ab2ac27..605a1444 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -92,7 +92,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { /*ignore_if_exists=*/true), "Cannot create database for system database"); } - // do not support create system table + /// Do not support create system table. { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -283,6 +283,100 @@ TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { "Cannot rename system table"); } +TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = + arrow::schema({arrow::field("pk", arrow::utf8()), arrow::field("v", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "tbl1"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + std::vector metadata_tables = {"snapshots", "schemas", "tags", "branches", + "consumers"}; + for (const auto& table_name : metadata_tables) { + Identifier system_identifier("db1", "tbl1$" + table_name); + ASSERT_OK_AND_ASSIGN(bool exists, catalog.TableExists(system_identifier)); + ASSERT_TRUE(exists) << table_name; + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_schema, + catalog.LoadTableSchema(system_identifier)); + ASSERT_TRUE(std::dynamic_pointer_cast(system_schema) != nullptr) + << table_name; + ASSERT_OK_AND_ASSIGN(auto c_schema, system_schema->GetArrowSchema()); + auto loaded_schema_result = arrow::ImportSchema(c_schema.get()); + ASSERT_TRUE(loaded_schema_result.ok()) << loaded_schema_result.status().ToString(); + ASSERT_GT(loaded_schema_result.ValueUnsafe()->num_fields(), 0) << table_name; + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr snapshots_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$snapshots"))); + ASSERT_OK_AND_ASSIGN(auto snapshots_c_schema, snapshots_schema->GetArrowSchema()); + auto snapshots_arrow_schema = arrow::ImportSchema(snapshots_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(snapshots_arrow_schema->field_names(), + (std::vector{ + "snapshot_id", "schema_id", "commit_user", "commit_identifier", "commit_kind", + "commit_time", "base_manifest_list", "delta_manifest_list", + "changelog_manifest_list", "total_record_count", "delta_record_count", + "changelog_record_count", "watermark", "next_row_id"})); + ASSERT_EQ(snapshots_arrow_schema->field(5)->type()->id(), arrow::Type::TIMESTAMP); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schemas_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$schemas"))); + ASSERT_OK_AND_ASSIGN(auto schemas_c_schema, schemas_schema->GetArrowSchema()); + auto schemas_arrow_schema = arrow::ImportSchema(schemas_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(schemas_arrow_schema->field_names(), + (std::vector{"schema_id", "fields", "partition_keys", "primary_keys", + "options", "comment", "update_time"})); + ASSERT_EQ(schemas_arrow_schema->field(6)->type()->id(), arrow::Type::TIMESTAMP); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr tags_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$tags"))); + ASSERT_OK_AND_ASSIGN(auto tags_c_schema, tags_schema->GetArrowSchema()); + auto tags_arrow_schema = arrow::ImportSchema(tags_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(tags_arrow_schema->field_names(), + (std::vector{"tag_name", "snapshot_id", "schema_id", "commit_time", + "record_count", "create_time", "time_retained"})); + ASSERT_EQ(tags_arrow_schema->field(3)->type()->id(), arrow::Type::TIMESTAMP); + ASSERT_EQ(tags_arrow_schema->field(5)->type()->id(), arrow::Type::TIMESTAMP); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr branches_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$branches"))); + ASSERT_OK_AND_ASSIGN(auto branches_c_schema, branches_schema->GetArrowSchema()); + auto branches_arrow_schema = arrow::ImportSchema(branches_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(branches_arrow_schema->field_names(), + (std::vector{"branch_name", "create_time"})); + ASSERT_EQ(branches_arrow_schema->field(1)->type()->id(), arrow::Type::TIMESTAMP); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr consumers_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$consumers"))); + ASSERT_OK_AND_ASSIGN(auto consumers_c_schema, consumers_schema->GetArrowSchema()); + auto consumers_arrow_schema = arrow::ImportSchema(consumers_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(consumers_arrow_schema->field_names(), + (std::vector{"consumer_id", "next_snapshot_id"})); + ASSERT_FALSE(consumers_arrow_schema->field(1)->nullable()); + + Identifier snapshots_identifier("db1", "tbl1$snapshots"); + ::ArrowSchema system_create_schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); + ASSERT_NOK_WITH_MSG( + catalog.CreateTable(snapshots_identifier, &system_create_schema, {}, {}, options, false), + "Cannot create table for system table"); + ArrowSchemaRelease(&system_create_schema); + ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.RenameTable(snapshots_identifier, Identifier("db1", "tbl2"), false), + "Cannot rename system table"); +} + TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { std::map options; options[Options::FILE_SYSTEM] = "local"; @@ -627,7 +721,7 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_OK_AND_ASSIGN(bool exist, catalog.TableExists(Identifier("test_db", "tbl1"))); ASSERT_FALSE(exist); - // Test 4: Drop system table + /// Test 4: Drop system table. ASSERT_NOK_WITH_MSG( catalog.DropTable(Identifier("test_db", "tbl$system"), /*ignore_if_not_exists=*/false), @@ -693,7 +787,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { /*ignore_if_not_exists=*/false), "Cannot rename table across databases. Cross-database rename is not supported."); - // Test 6: Rename system table + /// Test 6: Rename system table. ASSERT_NOK_WITH_MSG(catalog.RenameTable(Identifier("test_db", "tbl$system"), Identifier("test_db", "new_system_tbl"), /*ignore_if_not_exists=*/false), diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 4cb0c81a..bd94a4c0 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -671,7 +671,6 @@ struct CoreOptions::Impl { // Parse table-read.sequence-number.enabled - expose sequence number in system tables PAIMON_RETURN_NOT_OK(parser.Parse(Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, &table_read_sequence_number_enabled)); - // Parse key-value.sequence_number.enabled - internal sequence number read switch PAIMON_RETURN_NOT_OK(parser.Parse(Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, &key_value_sequence_number_enabled)); // Parse partial-update.remove-record-on-sequence-group diff --git a/src/paimon/core/io/generic_row_to_arrow_array_converter.cpp b/src/paimon/core/io/generic_row_to_arrow_array_converter.cpp new file mode 100644 index 00000000..253db982 --- /dev/null +++ b/src/paimon/core/io/generic_row_to_arrow_array_converter.cpp @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/generic_row_to_arrow_array_converter.h" + +#include +#include +#include +#include + +#include "arrow/array/builder_nested.h" +#include "arrow/memory_pool.h" +#include "arrow/util/checked_cast.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +Result> GenericRowToArrowArrayConverter::Create( + const std::shared_ptr& schema, arrow::MemoryPool* pool) { + std::unique_ptr array_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::MakeBuilder( + pool, std::make_shared(schema->fields()), &array_builder)); + + auto struct_builder = + arrow::internal::checked_pointer_cast(std::move(array_builder)); + assert(struct_builder); + std::vector appenders; + appenders.reserve(schema->num_fields()); + int32_t reserve_count = 1; + for (int32_t i = 0; i < schema->num_fields(); ++i) { + PAIMON_ASSIGN_OR_RAISE( + RowToArrowArrayConverter::AppendValueFunc func, + AppendField(/*use_view=*/true, struct_builder->field_builder(i), &reserve_count)); + appenders.emplace_back(std::move(func)); + } + return std::unique_ptr(new GenericRowToArrowArrayConverter( + reserve_count, std::move(appenders), std::move(struct_builder), nullptr)); +} + +Result GenericRowToArrowArrayConverter::NextBatch( + const std::vector& rows) { + PAIMON_RETURN_NOT_OK(ResetAndReserve()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + array_builder_->AppendValues(rows.size(), /*valid_bytes=*/nullptr)); + for (size_t i = 0; i < appenders_.size(); ++i) { + for (const auto& row : rows) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(appenders_[i](row, i)); + } + } + + return FinishAndAccumulate(); +} + +} // namespace paimon diff --git a/src/paimon/core/io/generic_row_to_arrow_array_converter.h b/src/paimon/core/io/generic_row_to_arrow_array_converter.h new file mode 100644 index 00000000..77df6c8f --- /dev/null +++ b/src/paimon/core/io/generic_row_to_arrow_array_converter.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/generic_row.h" +#include "paimon/core/io/row_to_arrow_array_converter.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/result.h" + +namespace arrow { +class MemoryPool; +class StructBuilder; +} // namespace arrow + +namespace paimon { + +/// Converts in-memory GenericRow values into a struct Arrow array. +class GenericRowToArrowArrayConverter + : public RowToArrowArrayConverter { + public: + static Result> Create( + const std::shared_ptr& schema, arrow::MemoryPool* pool); + + Result NextBatch(const std::vector& rows) override; + + private: + GenericRowToArrowArrayConverter(int32_t reserve_count, std::vector&& appenders, + std::unique_ptr&& array_builder, + std::unique_ptr&& arrow_pool) + : RowToArrowArrayConverter(reserve_count, std::move(appenders), std::move(array_builder), + std::move(arrow_pool)) {} +}; + +} // namespace paimon diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index c878151a..bc0c94e4 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -113,6 +113,10 @@ class TableSchema : public DataSchema, public Jsonizable { bool CrossPartitionUpdate() const; + int64_t TimeMillis() const { + return time_millis_; + } + private: JSONIZABLE_FRIEND_AND_DEFAULT_CTOR(TableSchema); diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index bc330d99..f5cd896d 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -136,8 +136,8 @@ class ChangelogBatchReader : public BatchReader { private: Result> CopyToStablePool( const std::shared_ptr& array) const { - // The imported data batch may release its C Arrow buffers after this wrapper returns. - // Keep returned system-table arrays independent of that input batch lifetime. + /// The imported data batch may release its C Arrow buffers after this wrapper returns. + /// Keep returned system-table arrays independent of that input batch lifetime. PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr result, arrow::Concatenate({array}, arrow_pool_)); return result; diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h index df1829ae..8592ea7b 100644 --- a/src/paimon/core/table/system/audit_log_system_table.h +++ b/src/paimon/core/table/system/audit_log_system_table.h @@ -30,6 +30,7 @@ namespace paimon { class FileSystem; class TableSchema; +/// Converts data columns when wrapping the base table changelog reader. class ChangelogBatchConverter { public: virtual ~ChangelogBatchConverter() = default; @@ -38,6 +39,7 @@ class ChangelogBatchConverter { const std::shared_ptr& array, arrow::MemoryPool* pool) const = 0; }; +/// System table for `T$audit_log`, exposing row-level changelog records with rowkind. class AuditLogSystemTable : public SystemTable { public: static constexpr const char* kName = "audit_log"; diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h index 6ba95862..0685fcd8 100644 --- a/src/paimon/core/table/system/binlog_system_table.h +++ b/src/paimon/core/table/system/binlog_system_table.h @@ -29,6 +29,7 @@ namespace paimon { class FileSystem; class TableSchema; +/// System table for `T$binlog`, exposing changelog records with list-wrapped data columns. class BinlogSystemTable : public AuditLogSystemTable { public: static constexpr const char* kName = "binlog"; diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp new file mode 100644 index 00000000..411eb092 --- /dev/null +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/system/in_memory_system_table.h" + +#include +#include +#include +#include + +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/core/io/generic_row_to_arrow_array_converter.h" +#include "paimon/core/table/system/system_table_scan.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/read_context.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" + +namespace paimon { +namespace { + +class InMemorySystemTableBatchReader : public BatchReader { + public: + InMemorySystemTableBatchReader(std::shared_ptr table, + const std::shared_ptr& pool) + : table_(std::move(table)), arrow_pool_(GetArrowPool(pool)) {} + + Result NextBatch() override { + if (emitted_) { + return BatchReader::MakeEofBatch(); + } + emitted_ = true; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, table_->ArrowSchema()); + PAIMON_ASSIGN_OR_RAISE(std::vector rows, table_->BuildRows()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr converter, + GenericRowToArrowArrayConverter::Create(schema, arrow_pool_.get())); + return converter->NextBatch(rows); + } + + std::shared_ptr GetReaderMetrics() const override { + return std::make_shared(); + } + + void Close() override { + emitted_ = true; + } + + private: + std::shared_ptr table_; + std::unique_ptr arrow_pool_; + bool emitted_ = false; +}; + +class InMemorySystemTableRead : public TableRead { + public: + InMemorySystemTableRead(std::shared_ptr table, + const std::shared_ptr& memory_pool) + : TableRead(memory_pool), table_(std::move(table)) {} + + Result> CreateReader( + const std::vector>& splits) override { + if (splits.size() != 1) { + return Status::Invalid(table_->Name(), " system table expects a single split"); + } + for (const auto& split : splits) { + if (!std::dynamic_pointer_cast(split)) { + return Status::Invalid("unsupported split for ", table_->Name(), " system table"); + } + } + return std::make_unique(table_, GetMemoryPool()); + } + + Result> CreateReader( + const std::shared_ptr& split) override { + std::vector> splits = {split}; + return CreateReader(splits); + } + + private: + std::shared_ptr table_; +}; + +} // namespace + +InMemorySystemTable::InMemorySystemTable(std::string table_path) + : table_path_(std::move(table_path)) {} + +Result> InMemorySystemTable::NewScan( + const std::shared_ptr& /*context*/) const { + return std::make_unique(table_path_); +} + +Result> InMemorySystemTable::NewRead( + const std::shared_ptr& context) const { + return std::make_unique( + std::static_pointer_cast(shared_from_this()), + context->GetMemoryPool()); +} + +} // namespace paimon diff --git a/src/paimon/core/table/system/options_system_table.h b/src/paimon/core/table/system/in_memory_system_table.h similarity index 70% rename from src/paimon/core/table/system/options_system_table.h rename to src/paimon/core/table/system/in_memory_system_table.h index 2282721d..f9fcebdb 100644 --- a/src/paimon/core/table/system/options_system_table.h +++ b/src/paimon/core/table/system/in_memory_system_table.h @@ -21,28 +21,34 @@ #include #include +#include +#include "paimon/common/data/generic_row.h" #include "paimon/core/table/system/system_table.h" namespace paimon { -class TableSchema; -class OptionsSystemTable : public SystemTable { +/// Base class for system tables whose result can be materialized as a single in-memory +/// RecordBatch. +/// +/// It provides the common singleton split scan and one-shot batch reader. +class InMemorySystemTable : public SystemTable { public: - static constexpr const char* kName = "options"; + explicit InMemorySystemTable(std::string table_path); - OptionsSystemTable(std::string table_path, std::shared_ptr table_schema); - - std::string Name() const override; - Result> ArrowSchema() const override; Result> NewScan( const std::shared_ptr& context) const override; Result> NewRead( const std::shared_ptr& context) const override; + virtual Result> BuildRows() const = 0; + + protected: + const std::string& TablePath() const { + return table_path_; + } private: std::string table_path_; - std::shared_ptr table_schema_; }; } // namespace paimon diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp new file mode 100644 index 00000000..4545b357 --- /dev/null +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/system/metadata_system_tables.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/common/data/binary_string.h" +#include "paimon/common/data/generic_row.h" +#include "paimon/common/utils/date_time_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/tag/tag.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/consumer_manager.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" +#include "paimon/fs/file_system.h" +#include "paimon/status.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace paimon { +namespace { + +template +Result JsonString(const T& value) { + rapidjson::Document document; + auto json_value = RapidJsonUtil::SerializeValue(value, &document.GetAllocator()); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + if (!json_value.Accept(writer)) { + return Status::Invalid("failed to serialize metadata system table value"); + } + return std::string(buffer.GetString(), buffer.GetSize()); +} + +Result LocalDateTimePartsToTimestampMillis(const std::vector& parts) { + if (parts.size() < 6) { + return Status::Invalid("tag create time requires at least 6 date-time fields"); + } + + int64_t year = parts[0]; + int64_t month = parts[1]; + int64_t day = parts[2]; + int64_t hour = parts[3]; + int64_t minute = parts[4]; + int64_t second = parts[5]; + int64_t nanos = parts.size() > 6 ? parts[6] : 0; + auto is_leap_year = [](int64_t value) { + return value % 4 == 0 && (value % 100 != 0 || value % 400 == 0); + }; + int64_t days_in_month[] = {31, is_leap_year(year) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, + 31}; + if (month < 1 || month > 12 || day < 1 || day > days_in_month[month - 1] || hour < 0 || + hour > 23 || minute < 0 || minute > 59 || second < 0 || second > 59 || nanos < 0 || + nanos > 999999999) { + return Status::Invalid("invalid tag create time fields"); + } + + year -= month <= 2 ? 1 : 0; + int64_t era = (year >= 0 ? year : year - 399) / 400; + auto year_of_era = static_cast(year - era * 400); + auto month_prime = static_cast(month + (month > 2 ? -3 : 9)); + uint32_t day_of_year = (153 * month_prime + 2) / 5 + static_cast(day) - 1; + uint32_t day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + int64_t epoch_day = era * 146097 + static_cast(day_of_era) - 719468; + return epoch_day * DateTimeUtils::MILLIS_PER_DAY + hour * 3600000 + minute * 60000 + + second * 1000 + nanos / 1000000; +} + +Result> OptionalLocalDateTimePartsToTimestampMillis( + const std::optional>& parts) { + if (!parts) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(int64_t timestamp_millis, + LocalDateTimePartsToTimestampMillis(parts.value())); + return std::optional(timestamp_millis); +} + +std::optional OptionalDoubleToString(const std::optional& value) { + if (!value) { + return std::optional(); + } + return std::to_string(value.value()); +} + +VariantType OptionalInt64Value(const std::optional& value) { + if (!value) { + return NullType(); + } + return value.value(); +} + +VariantType StringValue(const std::string& value) { + return BinaryString::FromString(value, GetDefaultPool().get()); +} + +VariantType OptionalStringValue(const std::optional& value) { + if (!value) { + return NullType(); + } + return StringValue(value.value()); +} + +VariantType TimestampMillisValue(int64_t value) { + return Timestamp::FromEpochMillis(value); +} + +Result LocalTimestampMillisValue(int64_t epoch_millis) { + PAIMON_ASSIGN_OR_RAISE( + Timestamp local_timestamp, + DateTimeUtils::ToLocalTimestamp(Timestamp::FromEpochMillis(epoch_millis))); + return TimestampMillisValue(local_timestamp.GetMillisecond()); +} + +VariantType OptionalTimestampMillisValue(const std::optional& value) { + if (!value) { + return NullType(); + } + return TimestampMillisValue(value.value()); +} + +MetadataSystemTableContext CreateMetadataContext(std::shared_ptr fs, + std::string table_path, std::string branch) { + return { + std::move(fs), + std::move(table_path), + BranchManager::NormalizeBranch(branch), + }; +} + +} // namespace + +OptionsSystemTable::OptionsSystemTable(std::string table_path, + std::shared_ptr table_schema) + : InMemorySystemTable(std::move(table_path)), table_schema_(std::move(table_schema)) {} + +std::string OptionsSystemTable::Name() const { + return kName; +} + +Result> OptionsSystemTable::ArrowSchema() const { + return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false), + arrow::field("value", arrow::utf8(), /*nullable=*/false)}); +} + +Result> OptionsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::vector rows; + rows.reserve(table_schema_->Options().size()); + for (const auto& [key, value] : table_schema_->Options()) { + GenericRow row(schema->num_fields()); + row.SetField(0, std::string_view(key)); + row.SetField(1, std::string_view(value)); + rows.push_back(std::move(row)); + } + return rows; +} + +SnapshotsSystemTable::SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, + std::string branch) + : InMemorySystemTable(table_path), + context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} + +std::string SnapshotsSystemTable::Name() const { + return kName; +} + +Result> SnapshotsSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false), + arrow::field("schema_id", arrow::int64(), /*nullable=*/false), + arrow::field("commit_user", arrow::utf8(), /*nullable=*/false), + arrow::field("commit_identifier", arrow::int64(), /*nullable=*/false), + arrow::field("commit_kind", arrow::utf8(), /*nullable=*/false), + arrow::field("commit_time", arrow::timestamp(arrow::TimeUnit::MILLI), + /*nullable=*/false), + arrow::field("base_manifest_list", arrow::utf8(), /*nullable=*/false), + arrow::field("delta_manifest_list", arrow::utf8(), /*nullable=*/false), + arrow::field("changelog_manifest_list", arrow::utf8(), /*nullable=*/true), + arrow::field("total_record_count", arrow::int64(), /*nullable=*/true), + arrow::field("delta_record_count", arrow::int64(), /*nullable=*/true), + arrow::field("changelog_record_count", arrow::int64(), /*nullable=*/true), + arrow::field("watermark", arrow::int64(), /*nullable=*/true), + arrow::field("next_row_id", arrow::int64(), /*nullable=*/true), + }); +} + +Result> SnapshotsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + SnapshotManager snapshot_manager(context_.fs, context_.table_path, context_.branch); + PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, snapshot_manager.GetAllSnapshots()); + std::sort(snapshots.begin(), snapshots.end(), + [](const Snapshot& lhs, const Snapshot& rhs) { return lhs.Id() < rhs.Id(); }); + std::vector rows; + rows.reserve(snapshots.size()); + + for (const auto& snapshot : snapshots) { + GenericRow row(schema->num_fields()); + row.SetField(0, snapshot.Id()); + row.SetField(1, snapshot.SchemaId()); + row.SetField(2, StringValue(snapshot.CommitUser())); + row.SetField(3, snapshot.CommitIdentifier()); + row.SetField(4, StringValue(Snapshot::CommitKind::ToString(snapshot.GetCommitKind()))); + PAIMON_ASSIGN_OR_RAISE(VariantType commit_time, + LocalTimestampMillisValue(snapshot.TimeMillis())); + row.SetField(5, commit_time); + row.SetField(6, StringValue(snapshot.BaseManifestList())); + row.SetField(7, StringValue(snapshot.DeltaManifestList())); + row.SetField(8, OptionalStringValue(snapshot.ChangelogManifestList())); + row.SetField(9, OptionalInt64Value(snapshot.TotalRecordCount())); + row.SetField(10, OptionalInt64Value(snapshot.DeltaRecordCount())); + row.SetField(11, OptionalInt64Value(snapshot.ChangelogRecordCount())); + row.SetField(12, OptionalInt64Value(snapshot.Watermark())); + row.SetField(13, OptionalInt64Value(snapshot.NextRowId())); + rows.push_back(std::move(row)); + } + + return rows; +} + +SchemasSystemTable::SchemasSystemTable(std::shared_ptr fs, std::string table_path, + std::string branch) + : InMemorySystemTable(table_path), + context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} + +std::string SchemasSystemTable::Name() const { + return kName; +} + +Result> SchemasSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("schema_id", arrow::int64(), /*nullable=*/false), + arrow::field("fields", arrow::utf8(), /*nullable=*/false), + arrow::field("partition_keys", arrow::utf8(), /*nullable=*/false), + arrow::field("primary_keys", arrow::utf8(), /*nullable=*/false), + arrow::field("options", arrow::utf8(), /*nullable=*/false), + arrow::field("comment", arrow::utf8(), /*nullable=*/true), + arrow::field("update_time", arrow::timestamp(arrow::TimeUnit::MILLI), + /*nullable=*/false), + }); +} + +Result> SchemasSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + SchemaManager schema_manager(context_.fs, context_.table_path, context_.branch); + PAIMON_ASSIGN_OR_RAISE(std::vector schema_ids, schema_manager.ListAllIds()); + std::sort(schema_ids.begin(), schema_ids.end()); + std::vector rows; + rows.reserve(schema_ids.size()); + + for (int64_t id : schema_ids) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr table_schema, + schema_manager.ReadSchema(id)); + PAIMON_ASSIGN_OR_RAISE(std::string fields_json, JsonString(table_schema->Fields())); + PAIMON_ASSIGN_OR_RAISE(std::string partition_keys_json, + JsonString(table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::string primary_keys_json, + JsonString(table_schema->PrimaryKeys())); + PAIMON_ASSIGN_OR_RAISE(std::string options_json, JsonString(table_schema->Options())); + + GenericRow row(schema->num_fields()); + row.SetField(0, table_schema->Id()); + row.SetField(1, StringValue(fields_json)); + row.SetField(2, StringValue(partition_keys_json)); + row.SetField(3, StringValue(primary_keys_json)); + row.SetField(4, StringValue(options_json)); + row.SetField(5, OptionalStringValue(table_schema->Comment())); + PAIMON_ASSIGN_OR_RAISE(VariantType update_time, + LocalTimestampMillisValue(table_schema->TimeMillis())); + row.SetField(6, update_time); + rows.push_back(std::move(row)); + } + + return rows; +} + +TagsSystemTable::TagsSystemTable(std::shared_ptr fs, std::string table_path, + std::string branch) + : InMemorySystemTable(table_path), + context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} + +std::string TagsSystemTable::Name() const { + return kName; +} + +Result> TagsSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("tag_name", arrow::utf8(), /*nullable=*/false), + arrow::field("snapshot_id", arrow::int64(), /*nullable=*/false), + arrow::field("schema_id", arrow::int64(), /*nullable=*/false), + arrow::field("commit_time", arrow::timestamp(arrow::TimeUnit::MILLI), + /*nullable=*/false), + arrow::field("record_count", arrow::int64(), /*nullable=*/true), + arrow::field("create_time", arrow::timestamp(arrow::TimeUnit::MILLI), + /*nullable=*/true), + arrow::field("time_retained", arrow::utf8(), /*nullable=*/true), + }); +} + +Result> TagsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + TagManager tag_manager(context_.fs, context_.table_path, context_.branch); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_names, tag_manager.ListTagNames()); + std::vector rows; + rows.reserve(tag_names.size()); + + for (const auto& name : tag_names) { + PAIMON_ASSIGN_OR_RAISE(Tag tag, tag_manager.GetOrThrow(name)); + PAIMON_ASSIGN_OR_RAISE(std::optional tag_create_time, + OptionalLocalDateTimePartsToTimestampMillis(tag.TagCreateTime())); + GenericRow row(schema->num_fields()); + row.SetField(0, StringValue(name)); + row.SetField(1, tag.Id()); + row.SetField(2, tag.SchemaId()); + PAIMON_ASSIGN_OR_RAISE(VariantType commit_time, + LocalTimestampMillisValue(tag.TimeMillis())); + row.SetField(3, commit_time); + row.SetField(4, OptionalInt64Value(tag.TotalRecordCount())); + row.SetField(5, OptionalTimestampMillisValue(tag_create_time)); + row.SetField(6, OptionalStringValue(OptionalDoubleToString(tag.TagTimeRetained()))); + rows.push_back(std::move(row)); + } + + return rows; +} + +BranchesSystemTable::BranchesSystemTable(std::shared_ptr fs, std::string table_path, + std::string branch) + : InMemorySystemTable(table_path), + context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} + +std::string BranchesSystemTable::Name() const { + return kName; +} + +Result> BranchesSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("branch_name", arrow::utf8(), /*nullable=*/false), + arrow::field("create_time", arrow::timestamp(arrow::TimeUnit::MILLI), + /*nullable=*/false), + }); +} + +Result> BranchesSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + PAIMON_ASSIGN_OR_RAISE(std::vector branches, + BranchManager::ListBranches(context_.fs, context_.table_path)); + std::vector rows; + rows.reserve(branches.size()); + + for (const auto& name : branches) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr branch_status, + context_.fs->GetFileStatus(BranchManager::BranchPath(context_.table_path, name))); + GenericRow row(schema->num_fields()); + row.SetField(0, StringValue(name)); + PAIMON_ASSIGN_OR_RAISE(VariantType create_time, + LocalTimestampMillisValue(branch_status->GetModificationTime())); + row.SetField(1, create_time); + rows.push_back(std::move(row)); + } + + return rows; +} + +ConsumersSystemTable::ConsumersSystemTable(std::shared_ptr fs, std::string table_path, + std::string branch) + : InMemorySystemTable(table_path), + context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch))) {} + +std::string ConsumersSystemTable::Name() const { + return kName; +} + +Result> ConsumersSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("consumer_id", arrow::utf8(), /*nullable=*/false), + arrow::field("next_snapshot_id", arrow::int64(), /*nullable=*/false), + }); +} + +Result> ConsumersSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + ConsumerManager consumer_manager(context_.fs, context_.table_path, context_.branch); + PAIMON_ASSIGN_OR_RAISE(auto consumers, consumer_manager.Consumers()); + std::vector rows; + rows.reserve(consumers.size()); + + for (const auto& [id, snapshot_id] : consumers) { + GenericRow row(schema->num_fields()); + row.SetField(0, StringValue(id)); + row.SetField(1, snapshot_id); + rows.push_back(std::move(row)); + } + + return rows; +} + +} // namespace paimon diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h new file mode 100644 index 00000000..c2803538 --- /dev/null +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/core/table/system/in_memory_system_table.h" + +namespace paimon { +class FileSystem; +class TableSchema; + +/// System table for `T$options`, exposing the latest base table options as key/value rows. +class OptionsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "options"; + + OptionsSystemTable(std::string table_path, std::shared_ptr table_schema); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + std::shared_ptr table_schema_; +}; + +/// Shared table metadata location used by metadata system tables. +struct MetadataSystemTableContext { + std::shared_ptr fs; + std::string table_path; + std::string branch; +}; + +/// System table for `T$snapshots`, exposing snapshot commit history. +class SnapshotsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "snapshots"; + + SnapshotsSystemTable(std::shared_ptr fs, std::string table_path, + std::string branch); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + MetadataSystemTableContext context_; +}; + +/// System table for `T$schemas`, exposing schema evolution history. +class SchemasSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "schemas"; + + SchemasSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + MetadataSystemTableContext context_; +}; + +/// System table for `T$tags`, exposing tags and the snapshots they reference. +class TagsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "tags"; + + TagsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + MetadataSystemTableContext context_; +}; + +/// System table for `T$branches`, exposing table branches including `main`. +class BranchesSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "branches"; + + BranchesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + MetadataSystemTableContext context_; +}; + +/// System table for `T$consumers`, exposing persisted streaming consumer offsets. +class ConsumersSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "consumers"; + + ConsumersSystemTable(std::shared_ptr fs, std::string table_path, + std::string branch); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + MetadataSystemTableContext context_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/system/options_system_table.cpp b/src/paimon/core/table/system/options_system_table.cpp deleted file mode 100644 index 3d9c6859..00000000 --- a/src/paimon/core/table/system/options_system_table.cpp +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "paimon/core/table/system/options_system_table.h" - -#include -#include -#include -#include -#include - -#include "arrow/api.h" -#include "arrow/c/bridge.h" -#include "paimon/common/metrics/metrics_impl.h" -#include "paimon/common/utils/arrow/mem_utils.h" -#include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/system/system_table_scan.h" -#include "paimon/memory/memory_pool.h" -#include "paimon/read_context.h" -#include "paimon/result.h" -#include "paimon/status.h" -#include "paimon/table/source/table_read.h" - -namespace paimon { -namespace { - -std::shared_ptr OptionsSchema() { - return arrow::schema({arrow::field("key", arrow::utf8(), /*nullable=*/false), - arrow::field("value", arrow::utf8(), /*nullable=*/false)}); -} - -class OptionsBatchReader : public BatchReader { - public: - OptionsBatchReader(std::map options, - const std::shared_ptr& pool) - : arrow_pool_(GetArrowPool(pool)), options_(std::move(options)) {} - - Result NextBatch() override { - if (emitted_) { - return BatchReader::MakeEofBatch(); - } - emitted_ = true; - - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr key_array_builder, - arrow::MakeBuilder(arrow::utf8(), arrow_pool_.get())); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr value_array_builder, - arrow::MakeBuilder(arrow::utf8(), arrow_pool_.get())); - auto* key_builder = dynamic_cast(key_array_builder.get()); - auto* value_builder = dynamic_cast(value_array_builder.get()); - if (key_builder == nullptr || value_builder == nullptr) { - return Status::Invalid("cannot create string builders for options system table"); - } - for (const auto& [key, value] : options_) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Append(key)); - PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(value)); - } - std::shared_ptr key_array; - std::shared_ptr value_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Finish(&key_array)); - PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Finish(&value_array)); - auto struct_array = std::make_shared( - arrow::struct_(OptionsSchema()->fields()), key_array->length(), - std::vector>{key_array, value_array}); - - auto c_array = std::make_unique<::ArrowArray>(); - auto c_schema = std::make_unique<::ArrowSchema>(); - PAIMON_RETURN_NOT_OK_FROM_ARROW( - arrow::ExportArray(*struct_array, c_array.get(), c_schema.get())); - return std::make_pair(std::move(c_array), std::move(c_schema)); - } - - std::shared_ptr GetReaderMetrics() const override { - return std::make_shared(); - } - - void Close() override { - emitted_ = true; - } - - private: - std::unique_ptr arrow_pool_; - std::map options_; - bool emitted_ = false; -}; - -class OptionsTableRead : public TableRead { - public: - OptionsTableRead(std::map options, - const std::shared_ptr& memory_pool) - : TableRead(memory_pool), options_(std::move(options)) {} - - Result> CreateReader( - const std::vector>& splits) override { - if (splits.size() != 1) { - return Status::Invalid("options system table expects a single split"); - } - for (const auto& split : splits) { - if (!std::dynamic_pointer_cast(split)) { - return Status::Invalid("unsupported split for options system table"); - } - } - return std::make_unique(options_, GetMemoryPool()); - } - - Result> CreateReader( - const std::shared_ptr& split) override { - std::vector> splits = {split}; - return CreateReader(splits); - } - - private: - std::map options_; -}; - -} // namespace - -OptionsSystemTable::OptionsSystemTable(std::string table_path, - std::shared_ptr table_schema) - : table_path_(std::move(table_path)), table_schema_(std::move(table_schema)) {} - -std::string OptionsSystemTable::Name() const { - return kName; -} - -Result> OptionsSystemTable::ArrowSchema() const { - return OptionsSchema(); -} - -Result> OptionsSystemTable::NewScan( - const std::shared_ptr& /*context*/) const { - return std::make_unique(table_path_); -} - -Result> OptionsSystemTable::NewRead( - const std::shared_ptr& context) const { - return std::make_unique(table_schema_->Options(), context->GetMemoryPool()); -} - -} // namespace paimon diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 676d2a0f..52d60f61 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -19,10 +19,12 @@ #include "paimon/core/table/system/system_table.h" +#include #include #include #include #include +#include #include "paimon/catalog/identifier.h" #include "paimon/common/utils/path_util.h" @@ -31,36 +33,129 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" -#include "paimon/core/table/system/options_system_table.h" +#include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/status.h" namespace paimon { +namespace { -bool SystemTableLoader::IsSupported(const std::string& system_table_name) { - std::string normalized_name = StringUtils::ToLowerCase(system_table_name); - return normalized_name == OptionsSystemTable::kName || - normalized_name == AuditLogSystemTable::kName || - normalized_name == BinlogSystemTable::kName; -} +using SystemTableFactory = std::function>( + const std::shared_ptr&, const std::string&, const std::shared_ptr&, + const std::map&)>; -Result> SystemTableLoader::Load( - const std::string& system_table_name, const std::shared_ptr& fs, - const std::string& table_path, const std::shared_ptr& table_schema, +struct SystemTableRegistryEntry { + std::string name; + SystemTableFactory factory; +}; + +std::map MergeOptions( + const std::shared_ptr& table_schema, const std::map& dynamic_options) { - std::string normalized_name = StringUtils::ToLowerCase(system_table_name); - if (normalized_name == OptionsSystemTable::kName) { - return std::make_shared(table_path, table_schema); - } auto options = table_schema->Options(); for (const auto& [key, value] : dynamic_options) { options[key] = value; } - if (normalized_name == AuditLogSystemTable::kName) { - return std::make_shared(fs, table_path, table_schema, options); + return options; +} + +std::string LoadBranch(const std::map& options) { + auto branch_iter = options.find(Options::BRANCH); + return branch_iter == options.end() ? BranchManager::DEFAULT_MAIN_BRANCH : branch_iter->second; +} + +const std::vector& SystemTableRegistry() { + static const std::vector registry = { + {OptionsSystemTable::kName, + [](const std::shared_ptr& /*fs*/, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& /*dynamic_options*/) + -> Result> { + return std::make_shared(table_path, table_schema); + }}, + {AuditLogSystemTable::kName, + [](const std::shared_ptr& fs, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + return std::make_shared( + fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + }}, + {BinlogSystemTable::kName, + [](const std::shared_ptr& fs, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + return std::make_shared( + fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + }}, + {SnapshotsSystemTable::kName, + [](const std::shared_ptr& fs, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + auto options = MergeOptions(table_schema, dynamic_options); + return std::make_shared(fs, table_path, LoadBranch(options)); + }}, + {SchemasSystemTable::kName, + [](const std::shared_ptr& fs, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + auto options = MergeOptions(table_schema, dynamic_options); + return std::make_shared(fs, table_path, LoadBranch(options)); + }}, + {TagsSystemTable::kName, + [](const std::shared_ptr& fs, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + auto options = MergeOptions(table_schema, dynamic_options); + return std::make_shared(fs, table_path, LoadBranch(options)); + }}, + {BranchesSystemTable::kName, + [](const std::shared_ptr& fs, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + auto options = MergeOptions(table_schema, dynamic_options); + return std::make_shared(fs, table_path, LoadBranch(options)); + }}, + {ConsumersSystemTable::kName, + [](const std::shared_ptr& fs, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + auto options = MergeOptions(table_schema, dynamic_options); + return std::make_shared(fs, table_path, LoadBranch(options)); + }}, + }; + return registry; +} + +std::optional FindSystemTableFactory(const std::string& system_table_name) { + std::string normalized_name = StringUtils::ToLowerCase(system_table_name); + for (const auto& entry : SystemTableRegistry()) { + if (entry.name == normalized_name) { + return entry.factory; + } } - if (normalized_name == BinlogSystemTable::kName) { - return std::make_shared(fs, table_path, table_schema, options); + return std::nullopt; +} + +} // namespace + +bool SystemTableLoader::IsSupported(const std::string& system_table_name) { + return FindSystemTableFactory(system_table_name).has_value(); +} + +Result> SystemTableLoader::Load( + const std::string& system_table_name, const std::shared_ptr& fs, + const std::string& table_path, const std::shared_ptr& table_schema, + const std::map& dynamic_options) { + std::optional factory = FindSystemTableFactory(system_table_name); + if (factory) { + return factory.value()(fs, table_path, table_schema, dynamic_options); } return Status::NotImplemented("unsupported system table: ", system_table_name); } diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 16e38780..ed35b6c2 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -37,12 +37,19 @@ class TableScan; class TableRead; class TableSchema; +/// Parsed information for a table-scoped system table path. struct SystemTablePath { + /// Base data table path without the system table suffix. std::string table_path; + /// Optional branch parsed from identifiers such as `T$branch_dev$options`. std::optional branch; + /// System table name, for example `options` or `snapshots`. std::string system_table_name; }; +/// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. +/// +/// Implementations expose a read-only schema and create their own scan/read objects. class SystemTable : public std::enable_shared_from_this { public: virtual ~SystemTable() = default; @@ -55,6 +62,9 @@ class SystemTable : public std::enable_shared_from_this { const std::shared_ptr& context) const = 0; }; +/// Loads system table implementations from parsed table identifiers or table paths. +/// +/// The loader owns the registry that maps a system table name to its factory. class SystemTableLoader { public: static bool IsSupported(const std::string& system_table_name); diff --git a/src/paimon/core/table/system/system_table_scan.h b/src/paimon/core/table/system/system_table_scan.h index 33201d80..d42e7dd5 100644 --- a/src/paimon/core/table/system/system_table_scan.h +++ b/src/paimon/core/table/system/system_table_scan.h @@ -29,6 +29,7 @@ namespace paimon { class Plan; class Split; +/// Singleton split used by in-memory system tables. class SystemTableSplit : public Split { public: explicit SystemTableSplit(const std::string& table_path) : table_path_(table_path) {} @@ -41,6 +42,7 @@ class SystemTableSplit : public Split { std::string table_path_; }; +/// Scan implementation for system tables that materialize exactly one split. class SystemTableScan : public TableScan { public: explicit SystemTableScan(const std::string& table_path); diff --git a/src/paimon/core/table/system/system_table_schema.h b/src/paimon/core/table/system/system_table_schema.h index e735d0a8..14e3d19c 100644 --- a/src/paimon/core/table/system/system_table_schema.h +++ b/src/paimon/core/table/system/system_table_schema.h @@ -31,6 +31,7 @@ namespace paimon { +/// Read-only schema wrapper for system tables. class SystemTableSchema : public SystemSchema { public: explicit SystemTableSchema(std::shared_ptr schema); diff --git a/src/paimon/core/utils/branch_manager.cpp b/src/paimon/core/utils/branch_manager.cpp new file mode 100644 index 00000000..3ca52cf4 --- /dev/null +++ b/src/paimon/core/utils/branch_manager.cpp @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/branch_manager.h" + +#include +#include +#include +#include + +#include "paimon/fs/file_system.h" + +namespace paimon { + +Result> BranchManager::ListBranches(const std::shared_ptr& fs, + const std::string& table_root) { + std::vector branches = {DEFAULT_MAIN_BRANCH}; + std::string branch_dir = PathUtil::JoinPath(table_root, "branch"); + PAIMON_ASSIGN_OR_RAISE(bool is_exist, fs->Exists(branch_dir)); + if (!is_exist) { + return branches; + } + + std::vector> file_status_list; + PAIMON_RETURN_NOT_OK(fs->ListDir(branch_dir, &file_status_list)); + std::string branch_prefix = BRANCH_PREFIX; + for (const auto& file_status : file_status_list) { + if (!file_status->IsDir()) { + continue; + } + std::string dir_name = PathUtil::GetName(file_status->GetPath()); + if (StringUtils::StartsWith(dir_name, branch_prefix, /*start_pos=*/0)) { + branches.push_back(dir_name.substr(branch_prefix.length())); + } + } + std::sort(branches.begin(), branches.end()); + return branches; +} + +} // namespace paimon diff --git a/src/paimon/core/utils/branch_manager.h b/src/paimon/core/utils/branch_manager.h index ee31f738..5a4a5821 100644 --- a/src/paimon/core/utils/branch_manager.h +++ b/src/paimon/core/utils/branch_manager.h @@ -17,12 +17,20 @@ */ #pragma once +#include #include +#include #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/result.h" namespace paimon { +class FileSystem; +} // namespace paimon + +namespace paimon { +/// Utility methods for table branch paths and branch discovery. class BranchManager { public: BranchManager() = delete; @@ -31,10 +39,12 @@ class BranchManager { static constexpr char DEFAULT_MAIN_BRANCH[] = "main"; static constexpr char BRANCH_PREFIX[] = "branch-"; + /// Normalizes an empty branch name to `main`. static std::string NormalizeBranch(const std::string& branch) { return StringUtils::IsNullOrWhitespaceOnly(branch) ? DEFAULT_MAIN_BRANCH : branch; } + /// Returns the table root path for the selected branch. static std::string BranchPath(const std::string& table_root, const std::string& branch) { return IsMainBranch(branch) ? table_root @@ -42,8 +52,13 @@ class BranchManager { "/branch/" + std::string(BRANCH_PREFIX) + branch); } + /// Returns whether the branch is the default main branch. static bool IsMainBranch(const std::string& branch) { return branch == DEFAULT_MAIN_BRANCH; } + + /// Lists all branches for a table, including `main`. + static Result> ListBranches(const std::shared_ptr& fs, + const std::string& table_root); }; } // namespace paimon diff --git a/src/paimon/core/utils/consumer_manager.cpp b/src/paimon/core/utils/consumer_manager.cpp new file mode 100644 index 00000000..4bde69b1 --- /dev/null +++ b/src/paimon/core/utils/consumer_manager.cpp @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/consumer_manager.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/fs/file_system.h" +#include "rapidjson/document.h" + +namespace paimon { + +ConsumerManager::ConsumerManager(std::shared_ptr fs, std::string table_path, + std::string branch) + : fs_(std::move(fs)), + table_path_(std::move(table_path)), + branch_(BranchManager::NormalizeBranch(branch)) {} + +std::string ConsumerManager::ConsumerDirectory() const { + return PathUtil::JoinPath(BranchManager::BranchPath(table_path_, branch_), "consumer"); +} + +std::string ConsumerManager::ConsumerPath(const std::string& consumer_id) const { + return PathUtil::JoinPath(ConsumerDirectory(), std::string(kConsumerPrefix) + consumer_id); +} + +Result> ConsumerManager::ListConsumers() const { + std::vector consumers; + std::string consumer_dir = ConsumerDirectory(); + PAIMON_ASSIGN_OR_RAISE(bool exists, fs_->Exists(consumer_dir)); + if (!exists) { + return consumers; + } + + std::vector> file_status_list; + PAIMON_RETURN_NOT_OK(fs_->ListDir(consumer_dir, &file_status_list)); + std::string prefix = kConsumerPrefix; + for (const auto& file_status : file_status_list) { + if (file_status->IsDir()) { + continue; + } + std::string file_name = PathUtil::GetName(file_status->GetPath()); + if (StringUtils::StartsWith(file_name, prefix, /*start_pos=*/0)) { + consumers.push_back(file_name.substr(prefix.length())); + } + } + std::sort(consumers.begin(), consumers.end()); + return consumers; +} + +Result> ConsumerManager::GetNextSnapshotId( + const std::string& consumer_id) const { + constexpr int32_t kMaxRetryCount = 10; + constexpr int32_t kRetryIntervalMillis = 200; + Status last_error; + for (int32_t i = 0; i < kMaxRetryCount; ++i) { + std::string content; + Status read_status = fs_->ReadFile(ConsumerPath(consumer_id), &content); + if (!read_status.ok()) { + if (read_status.IsNotExist()) { + return std::optional(); + } + return read_status; + } + std::optional snapshot_id = StringUtils::StringToValue(content); + if (snapshot_id) { + return snapshot_id; + } + + rapidjson::Document document; + document.Parse(content.c_str()); + if (!document.HasParseError() && document.IsObject() && + document.HasMember("nextSnapshot") && document["nextSnapshot"].IsInt64()) { + return std::optional(document["nextSnapshot"].GetInt64()); + } + + last_error = + Status::Invalid("failed to parse consumer metadata: ", ConsumerPath(consumer_id)); + std::this_thread::sleep_for(std::chrono::milliseconds(kRetryIntervalMillis)); + } + return last_error; +} + +Result> ConsumerManager::Consumers() const { + std::map consumers; + PAIMON_ASSIGN_OR_RAISE(std::vector consumer_ids, ListConsumers()); + for (const auto& id : consumer_ids) { + PAIMON_ASSIGN_OR_RAISE(std::optional next_snapshot_id, GetNextSnapshotId(id)); + if (next_snapshot_id) { + consumers[id] = next_snapshot_id.value(); + } + } + return consumers; +} + +} // namespace paimon diff --git a/src/paimon/core/utils/consumer_manager.h b/src/paimon/core/utils/consumer_manager.h new file mode 100644 index 00000000..de473fd2 --- /dev/null +++ b/src/paimon/core/utils/consumer_manager.h @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { + +class FileSystem; + +/// Manager for table streaming consumer metadata files. +/// +/// Consumers are stored under the selected table branch as `consumer/consumer-*` files. +class ConsumerManager { + public: + /// File name prefix for persisted consumer state files. + static constexpr char kConsumerPrefix[] = "consumer-"; + + ConsumerManager(std::shared_ptr fs, std::string table_path, std::string branch); + + /// Returns the consumer metadata directory for the selected table branch. + std::string ConsumerDirectory() const; + + /// Returns the metadata file path for a specific consumer id. + std::string ConsumerPath(const std::string& consumer_id) const; + + /// Lists consumer ids found in the consumer metadata directory. + Result> ListConsumers() const; + + /// Reads the next snapshot id persisted for the given consumer id. + Result> GetNextSnapshotId(const std::string& consumer_id) const; + + /// Reads all consumers and their next snapshot ids. + Result> Consumers() const; + + private: + std::shared_ptr fs_; + std::string table_path_; + std::string branch_; +}; + +} // namespace paimon diff --git a/src/paimon/core/utils/consumer_manager_test.cpp b/src/paimon/core/utils/consumer_manager_test.cpp new file mode 100644 index 00000000..ae9252ba --- /dev/null +++ b/src/paimon/core/utils/consumer_manager_test.cpp @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/consumer_manager.h" + +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(ConsumerManagerTest, TestBranchConsumerPath) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = std::make_shared(); + std::string table_path = PathUtil::JoinPath(dir->Str(), "table"); + std::string consumer_dir = + PathUtil::JoinPath(BranchManager::BranchPath(table_path, "dev"), "consumer"); + ASSERT_OK(fs->Mkdirs(consumer_dir)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(consumer_dir, "consumer-c1"), + R"({"nextSnapshot":42})", /*overwrite=*/true)); + + ConsumerManager manager(fs, table_path, "dev"); + ASSERT_EQ(manager.ConsumerDirectory(), consumer_dir); + ASSERT_OK_AND_ASSIGN(std::vector consumers, manager.ListConsumers()); + ASSERT_EQ(consumers, (std::vector{"c1"})); + ASSERT_OK_AND_ASSIGN(std::optional next_snapshot_id, manager.GetNextSnapshotId("c1")); + ASSERT_EQ(next_snapshot_id, 42); + ASSERT_OK_AND_ASSIGN(auto all_consumers, manager.Consumers()); + ASSERT_EQ(all_consumers, (std::map{{"c1", 42}})); +} + +} // namespace paimon::test diff --git a/src/paimon/core/utils/tag_manager.cpp b/src/paimon/core/utils/tag_manager.cpp index a7c28a04..25cdba2e 100644 --- a/src/paimon/core/utils/tag_manager.cpp +++ b/src/paimon/core/utils/tag_manager.cpp @@ -18,12 +18,14 @@ #include "paimon/core/utils/tag_manager.h" +#include #include -#include #include +#include #include "fmt/format.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/tag/tag.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/fs/file_system.h" @@ -55,8 +57,35 @@ Result> TagManager::Get(const std::string& tag_name) const { return std::optional(std::move(tag)); } +Result> TagManager::ListTagNames() const { + std::vector tag_names; + std::string tag_dir = TagDirectory(); + PAIMON_ASSIGN_OR_RAISE(bool is_exist, fs_->Exists(tag_dir)); + if (!is_exist) { + return tag_names; + } + + std::vector> file_status_list; + PAIMON_RETURN_NOT_OK(fs_->ListDir(tag_dir, &file_status_list)); + std::string tag_prefix = TAG_PREFIX; + for (const auto& file_status : file_status_list) { + if (file_status->IsDir()) { + continue; + } + std::string file_name = PathUtil::GetName(file_status->GetPath()); + if (StringUtils::StartsWith(file_name, tag_prefix, /*start_pos=*/0)) { + tag_names.push_back(file_name.substr(tag_prefix.length())); + } + } + std::sort(tag_names.begin(), tag_names.end()); + return tag_names; +} + std::string TagManager::TagPath(const std::string& tag_name) const { - return PathUtil::JoinPath(BranchManager::BranchPath(root_path_, branch_), - "/tag/" + std::string(TAG_PREFIX) + tag_name); + return PathUtil::JoinPath(TagDirectory(), std::string(TAG_PREFIX) + tag_name); +} + +std::string TagManager::TagDirectory() const { + return PathUtil::JoinPath(BranchManager::BranchPath(root_path_, branch_), "tag"); } } // namespace paimon diff --git a/src/paimon/core/utils/tag_manager.h b/src/paimon/core/utils/tag_manager.h index 27742418..d1b937e8 100644 --- a/src/paimon/core/utils/tag_manager.h +++ b/src/paimon/core/utils/tag_manager.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "paimon/core/tag/tag.h" @@ -41,8 +42,12 @@ class TagManager { Result> Get(const std::string& tag_name) const; + Result> ListTagNames() const; + std::string TagPath(const std::string& tag_name) const; + std::string TagDirectory() const; + private: std::shared_ptr fs_; std::string root_path_; diff --git a/src/paimon/core/utils/tag_manager_test.cpp b/src/paimon/core/utils/tag_manager_test.cpp index 8909b1a1..0a36f014 100644 --- a/src/paimon/core/utils/tag_manager_test.cpp +++ b/src/paimon/core/utils/tag_manager_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/utils/tag_manager.h" #include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/testing/utils/testharness.h" @@ -43,4 +44,46 @@ TEST(TagManagerTest, TestTagPath) { TagManager(nullptr, "/root", "data").TagPath("data")); } +TEST(TagManagerTest, TestListTagNames) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = std::make_shared(); + std::string table_path = PathUtil::JoinPath(dir->Str(), "table"); + std::string tag_dir = PathUtil::JoinPath(table_path, "tag"); + ASSERT_OK(fs->Mkdirs(tag_dir)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(tag_dir, "tag-2"), "", /*overwrite=*/true)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(tag_dir, "tag-1"), "", /*overwrite=*/true)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(tag_dir, "snapshot-1"), "", /*overwrite=*/true)); + ASSERT_OK(fs->Mkdirs(PathUtil::JoinPath(tag_dir, "tag-dir"))); + + TagManager manager(fs, table_path); + ASSERT_OK_AND_ASSIGN(std::vector tag_names, manager.ListTagNames()); + ASSERT_EQ(tag_names, (std::vector{"1", "2"})); +} + +TEST(TagManagerTest, TestListBranchTagNames) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = std::make_shared(); + std::string table_path = PathUtil::JoinPath(dir->Str(), "table"); + std::string tag_dir = PathUtil::JoinPath(BranchManager::BranchPath(table_path, "dev"), "tag"); + ASSERT_OK(fs->Mkdirs(tag_dir)); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(tag_dir, "tag-dev"), "", /*overwrite=*/true)); + + TagManager manager(fs, table_path, "dev"); + ASSERT_EQ(manager.TagDirectory(), tag_dir); + ASSERT_OK_AND_ASSIGN(std::vector tag_names, manager.ListTagNames()); + ASSERT_EQ(tag_names, (std::vector{"dev"})); +} + +TEST(TagManagerTest, TestListTagNamesReturnsEmptyForMissingDirectory) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = std::make_shared(); + TagManager manager(fs, PathUtil::JoinPath(dir->Str(), "table")); + + ASSERT_OK_AND_ASSIGN(std::vector tag_names, manager.ListTagNames()); + ASSERT_TRUE(tag_names.empty()); +} + } // namespace paimon::test diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index d00a0d70..5a9d1e4b 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -41,6 +42,7 @@ #include "paimon/common/reader/concat_batch_reader.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/io/data_file_meta.h" @@ -49,6 +51,7 @@ #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/deletion_file.h" #include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/core/tag/tag.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/defs.h" @@ -616,6 +619,191 @@ TEST(SystemTableReadInteTest, TestReadBranchOptionsSystemTable) { ASSERT_EQ(CollectStringMap(result), expected) << result->ToString(); } +TEST(SystemTableReadInteTest, TestReadMetadataSystemTables) { + arrow::FieldVector fields = { + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"pk"}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["a", 1]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["b", 2]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_2), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto snapshots_result, + ReadSystemTable(table_path + "$snapshots", options)); + auto snapshots_array = SingleStructChunk(snapshots_result); + ASSERT_EQ(StructFieldNames(snapshots_array), + (std::vector{ + "snapshot_id", "schema_id", "commit_user", "commit_identifier", "commit_kind", + "commit_time", "base_manifest_list", "delta_manifest_list", + "changelog_manifest_list", "total_record_count", "delta_record_count", + "changelog_record_count", "watermark", "next_row_id"})); + ASSERT_EQ(snapshots_array->length(), 2); + auto snapshot_id_array = + std::dynamic_pointer_cast(snapshots_array->field(0)); + auto commit_kind_array = + std::dynamic_pointer_cast(snapshots_array->field(4)); + auto commit_time_array = + std::dynamic_pointer_cast(snapshots_array->field(5)); + ASSERT_TRUE(snapshot_id_array); + ASSERT_TRUE(commit_kind_array); + ASSERT_TRUE(commit_time_array); + ASSERT_EQ(snapshot_id_array->Value(0), 1); + ASSERT_EQ(snapshot_id_array->Value(1), 2); + ASSERT_EQ(commit_kind_array->GetString(0), "APPEND"); + ASSERT_EQ(commit_kind_array->GetString(1), "APPEND"); + + ASSERT_OK_AND_ASSIGN(auto schemas_result, ReadSystemTable(table_path + "$schemas", options)); + auto schemas_array = SingleStructChunk(schemas_result); + ASSERT_EQ(StructFieldNames(schemas_array), + (std::vector{"schema_id", "fields", "partition_keys", "primary_keys", + "options", "comment", "update_time"})); + ASSERT_EQ(schemas_array->length(), 1); + auto schema_id_array = std::dynamic_pointer_cast(schemas_array->field(0)); + auto primary_keys_array = + std::dynamic_pointer_cast(schemas_array->field(3)); + auto update_time_array = + std::dynamic_pointer_cast(schemas_array->field(6)); + ASSERT_TRUE(schema_id_array); + ASSERT_TRUE(primary_keys_array); + ASSERT_TRUE(update_time_array); + ASSERT_EQ(schema_id_array->Value(0), 0); + ASSERT_EQ(primary_keys_array->GetString(0), R"(["pk"])"); + + ASSERT_OK_AND_ASSIGN(auto branches_result, ReadSystemTable(table_path + "$branches", options)); + auto branches_array = SingleStructChunk(branches_result); + ASSERT_EQ(StructFieldNames(branches_array), + (std::vector{"branch_name", "create_time"})); + ASSERT_EQ(branches_array->length(), 1); + auto branch_name_array = + std::dynamic_pointer_cast(branches_array->field(0)); + ASSERT_TRUE(branch_name_array); + ASSERT_EQ(branch_name_array->GetString(0), "main"); + auto branch_create_time_array = + std::dynamic_pointer_cast(branches_array->field(1)); + ASSERT_TRUE(branch_create_time_array); +} + +TEST(SystemTableReadInteTest, TestReadTagBranchAndConsumerSystemTables) { + const char* old_tz = std::getenv("TZ"); + std::optional old_timezone; + if (old_tz != nullptr) { + old_timezone = old_tz; + } + setenv("TZ", "Asia/Shanghai", /*overwrite=*/1); + tzset(); + ScopeGuard timezone_guard([old_timezone]() { + if (old_timezone) { + setenv("TZ", old_timezone->c_str(), /*overwrite=*/1); + } else { + unsetenv("TZ"); + } + tzset(); + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string source_path = + GetDataDir() + "/parquet/append_table_with_rt_branch.db/append_table_with_rt_branch"; + std::string table_path = PathUtil::JoinPath(dir->Str(), "metadata_table"); + ASSERT_TRUE(TestUtil::CopyDirectory(std::filesystem::path(source_path), + std::filesystem::path(table_path))); + + auto fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(Tag tag, + Tag::FromPath(fs, GetDataDir() + "/orc/append_table_with_tag.db/" + "append_table_with_tag/tag/tag-1")); + ASSERT_OK_AND_ASSIGN(std::string tag_json, tag.ToJsonString()); + ASSERT_OK(fs->Mkdirs(PathUtil::JoinPath(table_path, "tag"))); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "tag/tag-release"), tag_json, + /*overwrite=*/true)); + + ASSERT_OK(fs->Mkdirs(PathUtil::JoinPath(table_path, "consumer"))); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "consumer/consumer-c1"), + R"({"nextSnapshot":3})", + /*overwrite=*/true)); + + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}}; + + ASSERT_OK_AND_ASSIGN(auto branches_result, ReadSystemTable(table_path + "$branches", options)); + auto branches_array = SingleStructChunk(branches_result); + ASSERT_EQ(branches_array->length(), 2); + auto branch_name_array = + std::dynamic_pointer_cast(branches_array->field(0)); + ASSERT_TRUE(branch_name_array); + ASSERT_EQ(branch_name_array->GetString(0), "main"); + ASSERT_EQ(branch_name_array->GetString(1), "rt"); + + ASSERT_OK_AND_ASSIGN(auto tags_result, ReadSystemTable(table_path + "$tags", options)); + auto tags_array = SingleStructChunk(tags_result); + ASSERT_EQ(StructFieldNames(tags_array), + (std::vector{"tag_name", "snapshot_id", "schema_id", "commit_time", + "record_count", "create_time", "time_retained"})); + ASSERT_EQ(tags_array->length(), 1); + auto tag_name_array = std::dynamic_pointer_cast(tags_array->field(0)); + auto tag_snapshot_array = std::dynamic_pointer_cast(tags_array->field(1)); + auto tag_commit_time_array = + std::dynamic_pointer_cast(tags_array->field(3)); + auto tag_record_count_array = + std::dynamic_pointer_cast(tags_array->field(4)); + auto tag_create_time_array = + std::dynamic_pointer_cast(tags_array->field(5)); + auto tag_time_retained_array = + std::dynamic_pointer_cast(tags_array->field(6)); + ASSERT_TRUE(tag_name_array); + ASSERT_TRUE(tag_snapshot_array); + ASSERT_TRUE(tag_commit_time_array); + ASSERT_TRUE(tag_record_count_array); + ASSERT_TRUE(tag_create_time_array); + ASSERT_TRUE(tag_time_retained_array); + ASSERT_EQ(tag_name_array->GetString(0), "release"); + ASSERT_EQ(tag_snapshot_array->Value(0), tag.Id()); + ASSERT_OK_AND_ASSIGN( + Timestamp tag_commit_time, + DateTimeUtils::ToLocalTimestamp(Timestamp::FromEpochMillis(tag.TimeMillis()))); + ASSERT_EQ(tag_commit_time_array->Value(0), tag_commit_time.GetMillisecond()); + ASSERT_EQ(tag_record_count_array->Value(0), tag.TotalRecordCount().value()); + ASSERT_FALSE(tag_create_time_array->IsNull(0)); + ASSERT_EQ(tag_create_time_array->Value(0), 1770185290000); + ASSERT_EQ(tag_time_retained_array->GetString(0), "3.000000"); + + ASSERT_OK_AND_ASSIGN(auto consumers_result, + ReadSystemTable(table_path + "$consumers", options)); + auto consumers_array = SingleStructChunk(consumers_result); + ASSERT_EQ(StructFieldNames(consumers_array), + (std::vector{"consumer_id", "next_snapshot_id"})); + ASSERT_EQ(consumers_array->length(), 1); + auto consumer_id_array = + std::dynamic_pointer_cast(consumers_array->field(0)); + auto next_snapshot_array = + std::dynamic_pointer_cast(consumers_array->field(1)); + ASSERT_TRUE(consumer_id_array); + ASSERT_TRUE(next_snapshot_array); + ASSERT_EQ(consumer_id_array->GetString(0), "c1"); + ASSERT_EQ(next_snapshot_array->Value(0), 3); +} + TEST(SystemTableReadInteTest, TestReadAuditLogSystemTable) { arrow::FieldVector fields = { arrow::field("pk", arrow::utf8()), From 3ae0723cd836e81eff6edbddceb9992a8e20d994 Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Mon, 25 May 2026 19:06:45 +0800 Subject: [PATCH 002/138] fix: add break for CHAR_END_SUBTYPE From d5d94d8ce93b24c16ddb856e5b32eb9ae2abc3db Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Mon, 25 May 2026 21:08:00 +0800 Subject: [PATCH 003/138] fix: Fix MemorySegment::Compare to use big-endian byte-order comparison semantics From 5e25655f9f4325a5bf58b534b87f6300c5acc9cd Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Tue, 26 May 2026 10:46:21 +0800 Subject: [PATCH 004/138] chore(license): refine build support notices From 184cda7113708cd00b03178ee633ef809834eec4 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Tue, 26 May 2026 15:49:26 +0800 Subject: [PATCH 005/138] fix: Fix undefined behavior in BinarySection bitmask constants caused by signed left shift From 2b5426a9bb52a6d292a7c715f815cd6efd52d78e Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Wed, 27 May 2026 11:23:32 +0800 Subject: [PATCH 006/138] refactor: ColumnarArray: change constructor parameter to raw pointer to clarify non-owning semantics From 566586b261356252138213a6fb4e85071bb69466 Mon Sep 17 00:00:00 2001 From: Socrates Date: Wed, 27 May 2026 16:21:24 +0800 Subject: [PATCH 007/138] feat: Support macOS builds with AppleClang and Homebrew Clang --- CMakeLists.txt | 74 +++++++++++------- README.md | 2 +- build_support/asan_symbolize.py | 2 +- build_support/iwyu/iwyu.sh | 6 +- build_support/iwyu/iwyu_tool.py | 4 +- build_support/run_clang_format.py | 2 +- build_support/run_clang_tidy.py | 2 +- cmake_modules/BuildUtils.cmake | 51 +++++++++++-- cmake_modules/DefineOptions.cmake | 4 + cmake_modules/SetupCxxFlags.cmake | 18 +++++ cmake_modules/ThirdpartyToolchain.cmake | 69 +++++++++++------ cmake_modules/arrow.diff | 12 +++ docs/source/building.rst | 30 +++++++- src/paimon/CMakeLists.txt | 12 +-- src/paimon/common/file_index/CMakeLists.txt | 2 +- src/paimon/common/global_index/CMakeLists.txt | 2 +- src/paimon/common/logging/logging.cpp | 13 +++- src/paimon/format/avro/CMakeLists.txt | 8 +- src/paimon/format/blob/CMakeLists.txt | 8 +- src/paimon/format/orc/CMakeLists.txt | 8 +- src/paimon/format/orc/orc_format_writer.cpp | 29 +++++-- .../orc/orc_input_output_stream_test.cpp | 9 ++- .../format/orc/predicate_converter_test.cpp | 40 ++++++---- .../format/orc/predicate_pushdown_test.cpp | 76 +++++++++++-------- src/paimon/format/parquet/CMakeLists.txt | 8 +- src/paimon/global_index/lucene/CMakeLists.txt | 8 +- src/paimon/global_index/lumina/CMakeLists.txt | 8 +- src/paimon/testing/utils/CMakeLists.txt | 4 +- .../testing/utils/dict_array_converter.h | 41 +++++----- third_party/jindosdk-nextarch/CMakeLists.txt | 3 +- 30 files changed, 360 insertions(+), 195 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b6aca77a..5cc1ca1d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,7 +57,6 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON) option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON) option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF) option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF) - if(PAIMON_ENABLE_ORC) add_definitions(-DPAIMON_ENABLE_ORC) endif() @@ -286,9 +285,6 @@ set(PAIMON_SHARED_PRIVATE_LINK_LIBS ${PAIMON_STATIC_LINK_LIBS}) add_subdirectory(third_party/roaring_bitmap EXCLUDE_FROM_ALL) add_subdirectory(third_party/xxhash EXCLUDE_FROM_ALL) -list(APPEND PAIMON_LINK_LIBS ${CMAKE_DL_LIBS}) -list(APPEND PAIMON_SHARED_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS}) - if(PAIMON_ENABLE_LUCENE) set(PAIMON_DICT_DEST "share/paimon/dict") @@ -325,8 +321,11 @@ add_compile_definitions("GLOG_USE_GLOG_EXPORT") set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -set(PAIMON_VERSION_SCRIPT_FLAGS - "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/paimon/symbols.map") +set(PAIMON_VERSION_SCRIPT_FLAGS) +if(NOT APPLE) + set(PAIMON_VERSION_SCRIPT_FLAGS + "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/paimon/symbols.map") +endif() set(ENV{PAIMON_TEST_DATA} "${CMAKE_SOURCE_DIR}/test/test_data") @@ -354,42 +353,59 @@ if(PAIMON_BUILD_TESTS) include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) include_directories("${CMAKE_SOURCE_DIR}/test/") - set(TEST_STATIC_LINK_LIBS - "-Wl,--whole-archive" + paimon_link_libraries_whole_archive( + TEST_WHOLE_ARCHIVE_LINK_LIBS paimon_file_index_static paimon_global_index_static paimon_local_file_system_static - paimon_mock_file_format_static - "-Wl,--no-whole-archive" - "-Wl,--no-as-needed" - paimon_parquet_file_format_shared - paimon_blob_file_format_shared - "-Wl,--as-needed") + paimon_mock_file_format_static) + paimon_link_libraries_no_as_needed( + TEST_PLUGIN_LINK_LIBS paimon_parquet_file_format_shared + paimon_blob_file_format_shared) + set(TEST_STATIC_LINK_LIBS ${TEST_WHOLE_ARCHIVE_LINK_LIBS} ${TEST_PLUGIN_LINK_LIBS}) + paimon_link_libraries_whole_archive(PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS + paimon_local_file_system_static) + paimon_link_libraries_no_as_needed(PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS + paimon_local_file_system_shared) + paimon_link_libraries_whole_archive(PAIMON_BLOB_FILE_FORMAT_STATIC_LINK_LIBS + paimon_blob_file_format_static) + paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS + paimon_parquet_file_format_static) if(PAIMON_ENABLE_ORC) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_orc_file_format_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS + paimon_orc_file_format_static) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_orc_file_format_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() if(PAIMON_ENABLE_AVRO) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_avro_file_format_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_whole_archive(PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS + paimon_avro_file_format_static) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_avro_file_format_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() if(PAIMON_ENABLE_JINDO) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_jindo_file_system_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_whole_archive(PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS + paimon_jindo_file_system_static) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_jindo_file_system_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() if(PAIMON_ENABLE_LUMINA) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_lumina_index_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_whole_archive(PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS + paimon_lumina_index_static) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_lumina_index_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() if(PAIMON_ENABLE_LUCENE) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--no-as-needed") - list(APPEND TEST_STATIC_LINK_LIBS paimon_lucene_index_shared) - list(APPEND TEST_STATIC_LINK_LIBS "-Wl,--as-needed") + paimon_link_libraries_whole_archive(PAIMON_LUCENE_INDEX_STATIC_LINK_LIBS + paimon_lucene_index_static) + paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS + paimon_lucene_index_shared) + list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() endif() diff --git a/README.md b/README.md index 57e32d47..81d3e82f 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Paimon-cpp currently provides: - **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index capabilities including bitmap index, B-tree index, DiskANN-based vector search with Lumina, and Lucene-based full-text search. - **Compatibility**: compatibility with Apache Paimon Java format and communication protocols, including commit messages, data splits, and manifests. -The current implementation supports the `x86_64` architecture. +Note: Linux x86_64 and macOS arm64 builds are currently verified. ## Building diff --git a/build_support/asan_symbolize.py b/build_support/asan_symbolize.py index bffb75a1..854090ae 100755 --- a/build_support/asan_symbolize.py +++ b/build_support/asan_symbolize.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 #===- lib/asan/scripts/asan_symbolize.py -----------------------------------===# # # The LLVM Compiler Infrastructure diff --git a/build_support/iwyu/iwyu.sh b/build_support/iwyu/iwyu.sh index 09b48039..7fb16f03 100755 --- a/build_support/iwyu/iwyu.sh +++ b/build_support/iwyu/iwyu.sh @@ -47,7 +47,7 @@ affected_files() { include-what-you-use --version if [[ "${1:-}" == "all" ]]; then - python $ROOT/build_support/iwyu/iwyu_tool.py -p ${IWYU_COMPILATION_DATABASE_PATH:-.} \ + ${PYTHON:-python3} $ROOT/build_support/iwyu/iwyu_tool.py -p ${IWYU_COMPILATION_DATABASE_PATH:-.} \ -- $IWYU_ARGS #| awk -f $ROOT/build_support/iwyu/iwyu-filter.awk elif [[ "${1:-}" == "match" ]]; then @@ -60,7 +60,7 @@ elif [[ "${1:-}" == "match" ]]; then done echo "Running IWYU on $IWYU_FILE_LIST" - python $ROOT/build_support/iwyu/iwyu_tool.py \ + ${PYTHON:-python3} $ROOT/build_support/iwyu/iwyu_tool.py \ -p ${IWYU_COMPILATION_DATABASE_PATH:-.} $IWYU_FILE_LIST -- \ $IWYU_ARGS | awk -f $ROOT/build_support/iwyu/iwyu-filter.awk else @@ -77,7 +77,7 @@ else IWYU_FILE_LIST="$IWYU_FILE_LIST $ROOT/$p" done - python $ROOT/build_support/iwyu/iwyu_tool.py \ + ${PYTHON:-python3} $ROOT/build_support/iwyu/iwyu_tool.py \ -p ${IWYU_COMPILATION_DATABASE_PATH:-.} $IWYU_FILE_LIST -- \ $IWYU_ARGS | awk -f $ROOT/build_support/iwyu/iwyu-filter.awk > $IWYU_LOG fi diff --git a/build_support/iwyu/iwyu_tool.py b/build_support/iwyu/iwyu_tool.py index 1429e0c0..87cf0307 100755 --- a/build_support/iwyu/iwyu_tool.py +++ b/build_support/iwyu/iwyu_tool.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This file has been imported into the apache source tree from # the IWYU source tree as of version 0.8 @@ -65,7 +65,7 @@ -DCMAKE_C_COMPILER="%VCINSTALLDIR%/VC/bin/cl.exe" \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -G Ninja ... - $ python iwyu_tool.py -p . + $ python3 iwyu_tool.py -p . See iwyu_tool.py -h for more details on command-line arguments. """ diff --git a/build_support/run_clang_format.py b/build_support/run_clang_format.py index 5b632060..fd653a53 100755 --- a/build_support/run_clang_format.py +++ b/build_support/run_clang_format.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/build_support/run_clang_tidy.py b/build_support/run_clang_tidy.py index 611a7e21..1cc1b216 100755 --- a/build_support/run_clang_tidy.py +++ b/build_support/run_clang_tidy.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index 5f93160e..cbc35748 100644 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -18,6 +18,43 @@ # Borrowed the file from Apache Arrow: # https://github.com/apache/arrow/blob/apache-arrow-17.0.0/cpp/cmake_modules/BuildUtils.cmake +function(paimon_link_libraries_whole_archive OUT_VAR) + set(_paimon_whole_archive_libs) + if(APPLE) + foreach(_paimon_lib IN LISTS ARGN) + list(APPEND _paimon_whole_archive_libs + "-Wl,-force_load,$" ${_paimon_lib}) + endforeach() + else() + list(APPEND + _paimon_whole_archive_libs + "-Wl,--whole-archive" + ${ARGN} + "-Wl,--no-whole-archive") + endif() + set(${OUT_VAR} + ${_paimon_whole_archive_libs} + PARENT_SCOPE) +endfunction() + +function(paimon_link_libraries_no_as_needed OUT_VAR) + set(_paimon_link_libs) + foreach(_paimon_lib IN LISTS ARGN) + if(APPLE) + list(APPEND _paimon_link_libs ${_paimon_lib}) + else() + list(APPEND + _paimon_link_libs + "-Wl,--no-as-needed" + ${_paimon_lib} + "-Wl,--as-needed") + endif() + endforeach() + set(${OUT_VAR} + ${_paimon_link_libs} + PARENT_SCOPE) +endfunction() + function(add_paimon_lib LIB_NAME) set(options BUILD_SHARED BUILD_STATIC) set(one_value_args SHARED_LINK_FLAGS) @@ -142,12 +179,14 @@ function(add_paimon_lib LIB_NAME) target_link_libraries(${LIB_NAME}_shared PUBLIC "$") - target_link_options(${LIB_NAME}_shared - PRIVATE - -Wl,--exclude-libs,ALL - -Wl,-Bsymbolic - -Wl,-z,defs - -Wl,--gc-sections) + if(NOT APPLE) + target_link_options(${LIB_NAME}_shared + PRIVATE + -Wl,--exclude-libs,ALL + -Wl,-Bsymbolic + -Wl,-z,defs + -Wl,--gc-sections) + endif() install(TARGETS ${LIB_NAME}_shared ${INSTALL_IS_OPTIONAL} EXPORT PaimonTargets diff --git a/cmake_modules/DefineOptions.cmake b/cmake_modules/DefineOptions.cmake index 45cf4476..6b266bec 100644 --- a/cmake_modules/DefineOptions.cmake +++ b/cmake_modules/DefineOptions.cmake @@ -98,6 +98,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(PAIMON_USE_CCACHE "Use ccache when compiling (if available)" ON) + define_option(PAIMON_USE_APPLE_LIBCXX_WITH_CLANG + "Use Apple SDK libc++ headers when building with upstream Clang on macOS" + ON) + #---------------------------------------------------------------------- set_option_category("Test") diff --git a/cmake_modules/SetupCxxFlags.cmake b/cmake_modules/SetupCxxFlags.cmake index 043c043b..4df2f560 100644 --- a/cmake_modules/SetupCxxFlags.cmake +++ b/cmake_modules/SetupCxxFlags.cmake @@ -196,6 +196,24 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STRE # the default standard library which does not support C++11. libc++ is the # default from 10.9 onward. set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++") + elseif(APPLE AND PAIMON_USE_APPLE_LIBCXX_WITH_CLANG) + execute_process(COMMAND xcrun --show-sdk-path + OUTPUT_VARIABLE PAIMON_MACOS_SDK_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(NOT PAIMON_MACOS_SDK_PATH) + message(FATAL_ERROR "PAIMON_USE_APPLE_LIBCXX_WITH_CLANG is enabled, but xcrun could not find the macOS SDK" + ) + endif() + set(PAIMON_MACOS_LIBCXX_INCLUDE_DIR "${PAIMON_MACOS_SDK_PATH}/usr/include/c++/v1") + if(NOT EXISTS "${PAIMON_MACOS_LIBCXX_INCLUDE_DIR}/cstdlib") + message(FATAL_ERROR "PAIMON_USE_APPLE_LIBCXX_WITH_CLANG is enabled, but libc++ headers were not found at ${PAIMON_MACOS_LIBCXX_INCLUDE_DIR}" + ) + endif() + message(STATUS "Using Apple libc++ headers with Clang: ${PAIMON_MACOS_LIBCXX_INCLUDE_DIR}" + ) + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -nostdinc++ -isystem ${PAIMON_MACOS_LIBCXX_INCLUDE_DIR} -include cstdlib" + ) endif() endif() diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index ec93aa50..f305d140 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -818,7 +818,7 @@ macro(build_lucene) boost_chrono boost_atomic pthread - dl) + ${CMAKE_DL_LIBS}) add_dependencies(lucene lucene_ep) endmacro() @@ -1296,7 +1296,8 @@ macro(build_jindosdk_nextarch) PROPERTIES IMPORTED_LOCATION "${JINDOSDK_NEXTARCH_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${JINDOSDK_NEXTARCH_INCLUDE_DIR}") - target_link_libraries(jindosdk::nextarch INTERFACE jindosdk::c_sdk pthread dl) + target_link_libraries(jindosdk::nextarch INTERFACE jindosdk::c_sdk pthread + ${CMAKE_DL_LIBS}) list(APPEND JINDOSDK_INCLUDE_DIR ${JINDOSDK_NEXTARCH_INCLUDE_DIR}) add_dependencies(jindosdk::nextarch jindosdk-nextarch_ep) @@ -1318,6 +1319,26 @@ macro(build_protobuf) get_target_property(THIRDPARTY_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) get_filename_component(THIRDPARTY_ZLIB_ROOT "${THIRDPARTY_ZLIB_INCLUDE_DIR}" DIRECTORY) + get_target_property(THIRDPARTY_ZLIB_LIBRARY zlib IMPORTED_LOCATION) + set(PROTOBUF_ZLIB_LIBRARY_ARGS) + foreach(_PAIMON_ZLIB_LOCATION_PROPERTY + IMPORTED_LOCATION + IMPORTED_LOCATION_NOCONFIG + IMPORTED_LOCATION_RELEASE + IMPORTED_LOCATION_DEBUG + IMPORTED_LOCATION_RELWITHDEBINFO + IMPORTED_LOCATION_MINSIZEREL) + if(NOT THIRDPARTY_ZLIB_LIBRARY AND TARGET ZLIB::ZLIB) + get_target_property(THIRDPARTY_ZLIB_LIBRARY ZLIB::ZLIB + ${_PAIMON_ZLIB_LOCATION_PROPERTY}) + endif() + endforeach() + unset(_PAIMON_ZLIB_LOCATION_PROPERTY) + if(THIRDPARTY_ZLIB_LIBRARY) + set(PROTOBUF_ZLIB_LIBRARY_ARGS + "-DZLIB_LIBRARY=${THIRDPARTY_ZLIB_LIBRARY}" + "-DZLIB_LIBRARY_RELEASE=${THIRDPARTY_ZLIB_LIBRARY}") + endif() # Strip lto flags (which may be added by dh_auto_configure) # See https://github.com/protocolbuffers/protobuf/issues/7092 @@ -1337,6 +1358,7 @@ macro(build_protobuf) "-DZLIB_ROOT=${THIRDPARTY_ZLIB_ROOT}" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX=) + list(APPEND PROTOBUF_CMAKE_ARGS ${PROTOBUF_ZLIB_LIBRARY_ARGS}) set(PROTOBUF_CONFIGURE SOURCE_SUBDIR "cmake" CMAKE_ARGS ${PROTOBUF_CMAKE_ARGS}) externalproject_add(protobuf_ep @@ -1441,6 +1463,14 @@ macro(build_orc) message(STATUS "PAIMON_RPATH value: ${PAIMON_RPATH}") set(ORC_RPATH ${PAIMON_RPATH}) message(STATUS "ORC_RPATH value: ${ORC_RPATH}") + set(ORC_LINKER_FLAGS) + if(NOT "${ORC_RPATH}" STREQUAL "") + list(APPEND + ORC_LINKER_FLAGS + "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" + "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" + "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}") + endif() string(REPLACE "-Werror" "" EP_CXX_FLAGS ${EP_CXX_FLAGS}) @@ -1466,9 +1496,7 @@ macro(build_orc) "-DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS}" "-DCMAKE_C_FLAGS=${ORC_CMAKE_C_FLAGS}" "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ORC_CMAKE_CXX_FLAGS}" - "-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" - "-DCMAKE_SHARED_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" - "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}" + ${ORC_LINKER_FLAGS} "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}" "-DLZ4_HOME=${ORC_LZ4_ROOT}" "-DZSTD_HOME=${ORC_ZSTD_ROOT}" @@ -1572,6 +1600,7 @@ macro(build_arrow) "-DCMAKE_CXX_FLAGS=${ARROW_CMAKE_CXX_FLAGS}" "-DCMAKE_C_FLAGS=${ARROW_CMAKE_C_FLAGS}" "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ARROW_CMAKE_CXX_FLAGS}" + -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_DEPENDENCY_USE_SHARED=OFF -DARROW_BUILD_SHARED=OFF -DARROW_BUILD_STATIC=ON @@ -1595,6 +1624,12 @@ macro(build_arrow) -DARROW_WITH_ZSTD=ON -DARROW_WITH_BZ2=OFF -DARROW_WITH_BROTLI=ON + -Dzstd_SOURCE=SYSTEM + -DSnappy_SOURCE=SYSTEM + -Dlz4_SOURCE=SYSTEM + -DZLIB_SOURCE=SYSTEM + -Dre2_SOURCE=SYSTEM + -Dzstd_ROOT=${ARROW_ZSTD_ROOT} -DZSTD_ROOT=${ARROW_ZSTD_ROOT} -DZLIB_ROOT=${ARROW_ZLIB_ROOT} -DSnappy_ROOT=${ARROW_SNAPPY_ROOT} @@ -1626,40 +1661,30 @@ macro(build_arrow) add_library(arrow STATIC IMPORTED) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libarrow.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_library(arrow_dataset STATIC IMPORTED) set_target_properties(arrow_dataset PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libarrow_dataset.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_library(arrow_acero STATIC IMPORTED) set_target_properties(arrow_acero PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libarrow_acero.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_library(parquet STATIC IMPORTED) set_target_properties(parquet PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libparquet.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_library(arrow_bundled_dependencies STATIC IMPORTED) set_target_properties(arrow_bundled_dependencies PROPERTIES IMPORTED_LOCATION "${ARROW_PREFIX}/lib/libarrow_bundled_dependencies.a" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${ARROW_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") add_dependencies(arrow arrow_ep) add_dependencies(parquet arrow_ep) @@ -1794,9 +1819,7 @@ macro(build_tbb) add_library(tbb STATIC IMPORTED) set_target_properties(tbb PROPERTIES IMPORTED_LOCATION "${TBB_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${TBB_INCLUDE_DIR}" - INTERFACE_LINK_DIRECTORIES - "${TBB_BUILD_DIR}/${LOWERCASE_BUILD_TYPE}") + INTERFACE_INCLUDE_DIRECTORIES "${TBB_INCLUDE_DIR}") add_dependencies(tbb tbb_ep) endmacro(build_tbb) diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff index e539d1f8..0d55f2be 100644 --- a/cmake_modules/arrow.diff +++ b/cmake_modules/arrow.diff @@ -211,3 +211,15 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/Thi # and crosscompiling emulator (for try_run() ) if(CMAKE_CROSSCOMPILING_EMULATOR) string(REPLACE ";" ${EP_LIST_SEPARATOR} EP_CMAKE_CROSSCOMPILING_EMULATOR +diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake +--- a/cpp/cmake_modules/BuildUtils.cmake ++++ b/cpp/cmake_modules/BuildUtils.cmake +@@ -112,7 +112,7 @@ function(arrow_create_merged_static_lib output_target) + execute_process(COMMAND ${LIBTOOL_MACOS} -V + OUTPUT_VARIABLE LIBTOOL_V_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE) +- if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools-([0-9.]+).*") ++ if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools(_ld)?-([0-9.]+).*") + message(FATAL_ERROR "libtool found appears to be the incompatible GNU libtool: ${LIBTOOL_MACOS}" + ) + endif() diff --git a/docs/source/building.rst b/docs/source/building.rst index 8b106c06..8608107a 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -27,13 +27,14 @@ System setup ============ Paimon uses CMake as a build configuration system. We recommend building -out-of-source. For example, you could create ``paimon-cpp/build-release`` -and invoke ``cmake $CMAKE_ARGS ..`` from this directory. +out-of-source. For example, you could create ``paimon-cpp/build`` and invoke +``cmake $CMAKE_ARGS ..`` from this directory. Building requires: * A C++17-enabled compiler. On Linux, gcc 8 and higher should be - sufficient. Windows and MacOS are not supported for now. + sufficient. On macOS, use AppleClang from Xcode Command Line Tools or + LLVM clang from Homebrew. Windows is not supported for now. * At least 2GB of RAM for a minimal build, 8GB for a minimal debug build with tests and 16GB for a full build. @@ -45,6 +46,29 @@ On Ubuntu/Debian you can install the requirements with: build-essential \ cmake +On macOS you can install the requirements with: + +.. code-block:: shell + + xcode-select --install + brew install cmake + +The same CMake build options apply on Linux and macOS. If you prefer upstream +LLVM clang instead of AppleClang on macOS, install LLVM and pass the Homebrew +compiler paths when configuring: + +.. code-block:: shell + + brew install llvm + cmake -B build \ + -DCMAKE_C_COMPILER="$(brew --prefix llvm)/bin/clang" \ + -DCMAKE_CXX_COMPILER="$(brew --prefix llvm)/bin/clang++" + +When building with upstream Clang on macOS, Paimon uses Apple SDK libc++ +headers by default to avoid incompatibilities in bundled third-party +dependencies. Pass ``-DPAIMON_USE_APPLE_LIBCXX_WITH_CLANG=OFF`` to disable +this behavior. + We also provide a docker template to help you get started quickly. See in ``.devcontainer`` folder for more details. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 5673a55d..c550226c 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -357,7 +357,7 @@ add_paimon_lib(paimon arrow tbb glog - dl + ${CMAKE_DL_LIBS} fmt roaring_bitmap xxhash @@ -549,9 +549,7 @@ if(PAIMON_BUILD_TESTS) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - "-Wl,--no-whole-archive" + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN}) add_paimon_test(core_test @@ -739,10 +737,8 @@ if(PAIMON_BUILD_TESTS) # fs/jindo/jindo_file_system_test.cpp STATIC_LINK_LIBS paimon_shared - "-Wl,--whole-archive" - paimon_local_file_system_static - # paimon_jindo_file_system_static - "-Wl,--no-whole-archive" + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} + # ${PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS} test_utils_static ${GTEST_LINK_TOOLCHAIN}) diff --git a/src/paimon/common/file_index/CMakeLists.txt b/src/paimon/common/file_index/CMakeLists.txt index 9a1419e3..0bd1f167 100644 --- a/src/paimon/common/file_index/CMakeLists.txt +++ b/src/paimon/common/file_index/CMakeLists.txt @@ -44,7 +44,7 @@ add_paimon_lib(paimon_file_index arrow fmt xxhash - dl + ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/common/global_index/CMakeLists.txt b/src/paimon/common/global_index/CMakeLists.txt index 7805e3a6..c2f9fb51 100644 --- a/src/paimon/common/global_index/CMakeLists.txt +++ b/src/paimon/common/global_index/CMakeLists.txt @@ -40,7 +40,7 @@ add_paimon_lib(paimon_global_index STATIC_LINK_LIBS arrow fmt - dl + ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/common/logging/logging.cpp b/src/paimon/common/logging/logging.cpp index 5e8a3f18..252e7a52 100644 --- a/src/paimon/common/logging/logging.cpp +++ b/src/paimon/common/logging/logging.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,16 @@ namespace paimon { +static const char* GetProgramName() { +#if defined(__APPLE__) + return getprogname(); +#elif defined(__GLIBC__) + return program_invocation_name; +#else + return "paimon-cpp"; +#endif +} + static std::optional& getLoggerCreator() { static std::optional _loggerCreator; return _loggerCreator; @@ -86,7 +97,7 @@ std::unique_ptr Logger::GetLogger(const std::string& path) { } std::unique_lock ulock(getRegistryLock()); if (!google::IsGoogleLoggingInitialized()) { - google::InitGoogleLogging(program_invocation_name); + google::InitGoogleLogging(GetProgramName()); } return std::make_unique(); } diff --git a/src/paimon/format/avro/CMakeLists.txt b/src/paimon/format/avro/CMakeLists.txt index 8aa593b0..00c78477 100644 --- a/src/paimon/format/avro/CMakeLists.txt +++ b/src/paimon/format/avro/CMakeLists.txt @@ -40,7 +40,7 @@ if(PAIMON_ENABLE_AVRO) fmt avro tbb - dl + ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared @@ -61,10 +61,8 @@ if(PAIMON_ENABLE_AVRO) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_avro_file_format_static - "-Wl,--no-whole-archive" + ${PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN} EXTRA_LINK_LIBS avro) diff --git a/src/paimon/format/blob/CMakeLists.txt b/src/paimon/format/blob/CMakeLists.txt index 4400ba04..440effff 100644 --- a/src/paimon/format/blob/CMakeLists.txt +++ b/src/paimon/format/blob/CMakeLists.txt @@ -27,7 +27,7 @@ add_paimon_lib(paimon_blob_file_format STATIC_LINK_LIBS arrow fmt - dl + ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared @@ -47,9 +47,7 @@ if(PAIMON_BUILD_TESTS) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_blob_file_format_static - "-Wl,--no-whole-archive" + ${PAIMON_BLOB_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN}) endif() diff --git a/src/paimon/format/orc/CMakeLists.txt b/src/paimon/format/orc/CMakeLists.txt index 25b44a5a..8bb2e7dc 100644 --- a/src/paimon/format/orc/CMakeLists.txt +++ b/src/paimon/format/orc/CMakeLists.txt @@ -40,7 +40,7 @@ if(PAIMON_ENABLE_ORC) fmt orc::orc tbb - dl + ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared @@ -63,10 +63,8 @@ if(PAIMON_ENABLE_ORC) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_orc_file_format_static - "-Wl,--no-whole-archive" + ${PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN} EXTRA_LINK_LIBS orc::orc) diff --git a/src/paimon/format/orc/orc_format_writer.cpp b/src/paimon/format/orc/orc_format_writer.cpp index 870d755d..ff6cd8a7 100644 --- a/src/paimon/format/orc/orc_format_writer.cpp +++ b/src/paimon/format/orc/orc_format_writer.cpp @@ -35,6 +35,7 @@ #include "orc/Vector.hh" #include "orc/Writer.hh" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/options/memory_size.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/string_utils.h" @@ -207,6 +208,18 @@ std::shared_ptr OrcFormatWriter::GetWriterMetrics() const { return metrics_; } +namespace { + +Result GetMemorySizeOption(const std::map& options, + const std::string& key, uint64_t default_value) { + PAIMON_ASSIGN_OR_RAISE(std::string value, OptionsUtils::GetValueFromMap( + options, key, std::to_string(default_value))); + PAIMON_ASSIGN_OR_RAISE(int64_t bytes, MemorySize::ParseBytes(value)); + return static_cast(bytes); +} + +} // namespace + Result<::orc::WriterOptions> OrcFormatWriter::PrepareWriterOptions( const std::map& options, const std::string& file_compression, const std::shared_ptr& data_type) { @@ -220,15 +233,15 @@ Result<::orc::WriterOptions> OrcFormatWriter::PrepareWriterOptions( } } ::orc::WriterOptions writer_options; - PAIMON_ASSIGN_OR_RAISE(size_t stripe_size, OptionsUtils::GetValueFromMap( - options, ORC_STRIPE_SIZE, DEFAULT_STRIPE_SIZE)); + PAIMON_ASSIGN_OR_RAISE(uint64_t stripe_size, + GetMemorySizeOption(options, ORC_STRIPE_SIZE, DEFAULT_STRIPE_SIZE)); writer_options.setStripeSize(stripe_size); PAIMON_ASSIGN_OR_RAISE(::orc::CompressionKind compression, ToOrcCompressionKind(StringUtils::ToLowerCase(file_compression))); writer_options.setCompression(compression); - PAIMON_ASSIGN_OR_RAISE(size_t compression_block_size, OptionsUtils::GetValueFromMap( - options, ORC_COMPRESSION_BLOCK_SIZE, - DEFAULT_COMPRESSION_BLOCK_SIZE)); + PAIMON_ASSIGN_OR_RAISE( + uint64_t compression_block_size, + GetMemorySizeOption(options, ORC_COMPRESSION_BLOCK_SIZE, DEFAULT_COMPRESSION_BLOCK_SIZE)); writer_options.setCompressionBlockSize(compression_block_size); PAIMON_ASSIGN_OR_RAISE( double dictionary_key_threshold, @@ -237,9 +250,9 @@ Result<::orc::WriterOptions> OrcFormatWriter::PrepareWriterOptions( writer_options.setDictionaryKeySizeThreshold(dictionary_key_threshold); // always use tight numeric vector writer_options.setUseTightNumericVector(true); - PAIMON_ASSIGN_OR_RAISE(size_t row_index_stride, - OptionsUtils::GetValueFromMap(options, ORC_ROW_INDEX_STRIDE, - DEFAULT_ROW_INDEX_STRIDE)); + PAIMON_ASSIGN_OR_RAISE(uint64_t row_index_stride, + OptionsUtils::GetValueFromMap(options, ORC_ROW_INDEX_STRIDE, + DEFAULT_ROW_INDEX_STRIDE)); writer_options.setRowIndexStride(row_index_stride); // In order to avoid issue like https://github.com/alibaba/paimon-cpp/issues/42, we explicitly // set GMT timezone. diff --git a/src/paimon/format/orc/orc_input_output_stream_test.cpp b/src/paimon/format/orc/orc_input_output_stream_test.cpp index 1caa5f10..7db0e1bb 100644 --- a/src/paimon/format/orc/orc_input_output_stream_test.cpp +++ b/src/paimon/format/orc/orc_input_output_stream_test.cpp @@ -32,6 +32,7 @@ #include "orc/Type.hh" #include "orc/Vector.hh" #include "orc/Writer.hh" +#include "paimon/common/utils/path_util.h" #include "paimon/format/orc/orc_format_defs.h" #include "paimon/format/orc/orc_input_stream_impl.h" #include "paimon/format/orc/orc_output_stream_impl.h" @@ -55,7 +56,8 @@ TEST(OrcInputOutputStreamTest, TestInOutStream) { file_system->Create(file_name, /*overwrite=*/true)); ASSERT_OK_AND_ASSIGN(std::unique_ptr out_stream, OrcOutputStreamImpl::Create(out)); - ASSERT_EQ(out_stream->getName(), file_name); + ASSERT_OK_AND_ASSIGN(auto normalized_file_name, PathUtil::NormalizePath(file_name)); + ASSERT_EQ(out_stream->getName(), normalized_file_name); ASSERT_EQ(out_stream->getNaturalWriteSize(), 128 * 1024); ASSERT_EQ(out_stream->getLength(), 0); @@ -86,7 +88,8 @@ TEST(OrcInputOutputStreamTest, TestSimple) { file_system->Create(file_name, /*overwrite=*/true)); ASSERT_OK_AND_ASSIGN(std::unique_ptr out_stream, OrcOutputStreamImpl::Create(out)); - ASSERT_EQ(out_stream->getName(), file_name); + ASSERT_OK_AND_ASSIGN(auto normalized_file_name, PathUtil::NormalizePath(file_name)); + ASSERT_EQ(out_stream->getName(), normalized_file_name); ASSERT_EQ(out_stream->getNaturalWriteSize(), 128 * 1024); ASSERT_EQ(out_stream->getLength(), 0); @@ -133,7 +136,7 @@ TEST(OrcInputOutputStreamTest, TestSimple) { ASSERT_OK_AND_ASSIGN(auto in_stream, OrcInputStreamImpl::Create(input_stream, DEFAULT_NATURAL_READ_SIZE)); auto length = file_system->GetFileStatus(file_name).value()->GetLen(); - ASSERT_EQ(in_stream->getName(), file_name); + ASSERT_EQ(in_stream->getName(), normalized_file_name); ASSERT_EQ(in_stream->getLength(), length); ASSERT_EQ(in_stream->getNaturalReadSize(), 1024 * 1024); diff --git a/src/paimon/format/orc/predicate_converter_test.cpp b/src/paimon/format/orc/predicate_converter_test.cpp index c5726e5b..f9ec5557 100644 --- a/src/paimon/format/orc/predicate_converter_test.cpp +++ b/src/paimon/format/orc/predicate_converter_test.cpp @@ -33,6 +33,16 @@ namespace paimon::orc::test { +namespace { + +// Use an explicit int64_t literal for BIGINT predicates. On macOS arm64, `long` and +// `int64_t` are distinct types, so `Literal(5l)` may instantiate `Literal`. +Literal BigIntLiteral(int64_t value) { + return Literal(value); +} + +} // namespace + TEST(PredicateConverterTest, TestSimple) { std::string orc_schema = "struct"; @@ -51,7 +61,7 @@ TEST(PredicateConverterTest, TestSimple) { } { auto predicate = PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) = 5), expr = leaf-0", search_arg->toString()); } @@ -72,19 +82,19 @@ TEST(PredicateConverterTest, TestSimple) { } { auto predicate = PredicateBuilder::NotEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) = 5), expr = (not leaf-0)", search_arg->toString()); } { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) <= 5), expr = (not leaf-0)", search_arg->toString()); } { auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) < 5), expr = (not leaf-0)", search_arg->toString()); } @@ -97,20 +107,20 @@ TEST(PredicateConverterTest, TestSimple) { } { auto predicate = PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) < 5), expr = leaf-0", search_arg->toString()); } { auto predicate = PredicateBuilder::LessOrEqual(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)); + FieldType::BIGINT, BigIntLiteral(5)); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) <= 5), expr = leaf-0", search_arg->toString()); } { - auto predicate = - PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - {Literal(1l), Literal(3l), Literal(5l)}); + auto predicate = PredicateBuilder::In( + /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(3), BigIntLiteral(5)}); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) in [1, 3, 5]), expr = leaf-0", search_arg->toString()); } @@ -147,9 +157,9 @@ TEST(PredicateConverterTest, TestSimple) { ASSERT_EQ("expr = YES", search_arg->toString()); } { - auto predicate = - PredicateBuilder::NotIn(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - {Literal(1l), Literal(3l), Literal(5l)}); + auto predicate = PredicateBuilder::NotIn( + /*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(3), BigIntLiteral(5)}); ASSERT_OK_AND_ASSIGN(auto search_arg, PredicateConverter::Convert(*orc_type, predicate)); ASSERT_EQ("leaf-0 = (column(id=1) in [1, 3, 5]), expr = (not leaf-0)", search_arg->toString()); @@ -189,7 +199,7 @@ TEST(PredicateConverterTest, TestCompound) { auto predicate, PredicateBuilder::And({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + BigIntLiteral(3)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -223,7 +233,7 @@ TEST(PredicateConverterTest, TestCompound) { auto predicate, PredicateBuilder::Or({ PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT, - Literal(3l)), + BigIntLiteral(3)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0))), PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::STRING, @@ -247,7 +257,7 @@ TEST(PredicateConverterTest, TestCompound) { {PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BOOLEAN, Literal(true)), PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"f0", - FieldType::BIGINT, Literal(3l))}) + FieldType::BIGINT, BigIntLiteral(3))}) .value(), PredicateBuilder::And( {PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", diff --git a/src/paimon/format/orc/predicate_pushdown_test.cpp b/src/paimon/format/orc/predicate_pushdown_test.cpp index 643d435f..76d1e740 100644 --- a/src/paimon/format/orc/predicate_pushdown_test.cpp +++ b/src/paimon/format/orc/predicate_pushdown_test.cpp @@ -55,6 +55,16 @@ class Predicate; namespace paimon::orc::test { +namespace { + +// Use an explicit int64_t literal for BIGINT predicates. On macOS arm64, `long` and +// `int64_t` are distinct types, so `Literal(5l)` may instantiate `Literal`. +Literal BigIntLiteral(int64_t value) { + return Literal(value); +} + +} // namespace + class PredicatePushdownTest : public ::testing::Test { public: void SetUp() override { @@ -175,48 +185,48 @@ TEST_F(PredicatePushdownTest, TestIntDoubleData) { { // f2 != 4, has data auto predicate = PredicateBuilder::NotEqual(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(4l)); + FieldType::BIGINT, BigIntLiteral(4)); CheckResult(read_schema, predicate, expected_array); } { // f2 == 6, has data auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)); + FieldType::BIGINT, BigIntLiteral(6)); CheckResult(read_schema, predicate, expected_array); } { // f2 == 1, no data auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(1l)); + FieldType::BIGINT, BigIntLiteral(1)); CheckResult(read_schema, predicate, /*expected_array=*/ nullptr); } { // f2 in [1,2,3], no data - auto predicate = - PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(1l), Literal(2l), Literal(3l)}); + auto predicate = PredicateBuilder::In( + /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(2), BigIntLiteral(3)}); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f2 not in [1,2,3], has data - auto predicate = - PredicateBuilder::NotIn(/*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(1l), Literal(2l), Literal(3l)}); + auto predicate = PredicateBuilder::NotIn( + /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(2), BigIntLiteral(3)}); CheckResult(read_schema, predicate, expected_array); } { // f2 in [2,3,4], has data - auto predicate = - PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(2l), Literal(3l), Literal(4l)}); + auto predicate = PredicateBuilder::In( + /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, + {BigIntLiteral(2), BigIntLiteral(3), BigIntLiteral(4)}); CheckResult(read_schema, predicate, expected_array); } { // f2 not in [2,3,4], has data - auto predicate = - PredicateBuilder::NotIn(/*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(2l), Literal(3l), Literal(4l)}); + auto predicate = PredicateBuilder::NotIn( + /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, + {BigIntLiteral(2), BigIntLiteral(3), BigIntLiteral(4)}); CheckResult(read_schema, predicate, expected_array); } } @@ -363,26 +373,28 @@ TEST_F(PredicatePushdownTest, TestPredicatePushdownWithAllDataNull) { // other predicate, always return IS_NULL (no data) { // f4 in [1,2], no data - auto predicate = PredicateBuilder::In(/*field_index=*/4, /*field_name=*/"f4", - FieldType::BIGINT, {Literal(1l), Literal(2l)}); + auto predicate = + PredicateBuilder::In(/*field_index=*/4, /*field_name=*/"f4", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(2)}); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f4 not in [1,2], no data - auto predicate = PredicateBuilder::NotIn(/*field_index=*/4, /*field_name=*/"f4", - FieldType::BIGINT, {Literal(1l), Literal(2l)}); + auto predicate = + PredicateBuilder::NotIn(/*field_index=*/4, /*field_name=*/"f4", FieldType::BIGINT, + {BigIntLiteral(1), BigIntLiteral(2)}); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f4 >= 3, no data auto predicate = PredicateBuilder::GreaterOrEqual(/*field_index=*/4, /*field_name=*/"f4", - FieldType::BIGINT, Literal(3l)); + FieldType::BIGINT, BigIntLiteral(3)); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f4 <= 3, no data auto predicate = PredicateBuilder::LessOrEqual(/*field_index=*/4, /*field_name=*/"f4", - FieldType::BIGINT, Literal(3l)); + FieldType::BIGINT, BigIntLiteral(3)); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } } @@ -436,14 +448,14 @@ TEST_F(PredicatePushdownTest, TestPredicatePushdownWithNullLiteral) { // f2 in [1,null,2], no data auto predicate = PredicateBuilder::In( /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(1l), Literal(FieldType::BIGINT), Literal(2l)}); + {BigIntLiteral(1), Literal(FieldType::BIGINT), BigIntLiteral(2)}); CheckResult(read_schema, predicate, /*expected_array=*/nullptr); } { // f2 in [1,null,2,4], has data auto predicate = PredicateBuilder::In( /*field_index=*/2, /*field_name=*/"f2", FieldType::BIGINT, - {Literal(1l), Literal(FieldType::BIGINT), Literal(2l), Literal(4l)}); + {BigIntLiteral(1), Literal(FieldType::BIGINT), BigIntLiteral(2), BigIntLiteral(4)}); CheckResult(read_schema, predicate, expected_array); } } @@ -457,7 +469,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::And( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0))), PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BOOLEAN, @@ -471,7 +483,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::And( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0))), PredicateBuilder::IsNull(/*field_index=*/3, /*field_name=*/"f3", @@ -485,7 +497,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::And( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0))), PredicateBuilder::IsNull(/*field_index=*/5, /*field_name=*/"f5", @@ -499,7 +511,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::And( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0))), PredicateBuilder::IsNull(/*field_index=*/5, /*field_name=*/"f5", @@ -513,7 +525,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::Or( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0)))})); ASSERT_TRUE(predicate); @@ -525,7 +537,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::Or( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(6l)), + FieldType::BIGINT, BigIntLiteral(6)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0)))})); ASSERT_TRUE(predicate); @@ -536,7 +548,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { ASSERT_OK_AND_ASSIGN( auto predicate, PredicateBuilder::Or({PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(2l)), + FieldType::BIGINT, BigIntLiteral(2)), PredicateBuilder::IsNull(/*field_index=*/5, /*field_name=*/"f5", FieldType::BINARY)})); ASSERT_TRUE(predicate); @@ -548,7 +560,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::Or( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(2l)), + FieldType::BIGINT, BigIntLiteral(2)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(4.0))), PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"f3", FieldType::BOOLEAN, @@ -562,7 +574,7 @@ TEST_F(PredicatePushdownTest, TestCompoundPredicate) { auto predicate, PredicateBuilder::Or( {PredicateBuilder::LessThan(/*field_index=*/2, /*field_name=*/"f2", - FieldType::BIGINT, Literal(2l)), + FieldType::BIGINT, BigIntLiteral(2)), PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f1", FieldType::FLOAT, Literal(static_cast(5.0))), PredicateBuilder::IsNull(/*field_index=*/3, /*field_name=*/"f3", diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 3dedd91c..2b993688 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -37,7 +37,7 @@ add_paimon_lib(paimon_parquet_file_format arrow glog fmt - dl + ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared @@ -59,10 +59,8 @@ if(PAIMON_BUILD_TESTS) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_parquet_file_format_static - "-Wl,--no-whole-archive" + ${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} parquet ${GTEST_LINK_TOOLCHAIN}) endif() diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index 8ffcc59d..3e7b0525 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -38,7 +38,7 @@ if(PAIMON_ENABLE_LUCENE) lucene arrow fmt - dl + ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared @@ -64,10 +64,8 @@ if(PAIMON_ENABLE_LUCENE) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_lucene_index_static - "-Wl,--no-whole-archive" + ${PAIMON_LUCENE_INDEX_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN}) endif() diff --git a/src/paimon/global_index/lumina/CMakeLists.txt b/src/paimon/global_index/lumina/CMakeLists.txt index f85af649..326f6660 100644 --- a/src/paimon/global_index/lumina/CMakeLists.txt +++ b/src/paimon/global_index/lumina/CMakeLists.txt @@ -27,7 +27,7 @@ if(PAIMON_ENABLE_LUMINA) arrow glog fmt - dl + ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS lumina::interface @@ -46,10 +46,8 @@ if(PAIMON_ENABLE_LUMINA) STATIC_LINK_LIBS paimon_shared test_utils_static - "-Wl,--whole-archive" - paimon_local_file_system_static - paimon_lumina_index_static - "-Wl,--no-whole-archive" + ${PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS} + ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} lumina::interface ${GTEST_LINK_TOOLCHAIN}) endif() diff --git a/src/paimon/testing/utils/CMakeLists.txt b/src/paimon/testing/utils/CMakeLists.txt index 453f963f..e0b9c80b 100644 --- a/src/paimon/testing/utils/CMakeLists.txt +++ b/src/paimon/testing/utils/CMakeLists.txt @@ -34,9 +34,7 @@ if(PAIMON_BUILD_TESTS) data_generator_test.cpp STATIC_LINK_LIBS paimon_shared - "-Wl,--whole-archive" - paimon_local_file_system_shared - "-Wl,--no-whole-archive" + ${PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS} test_utils_static ${GTEST_LINK_TOOLCHAIN}) diff --git a/src/paimon/testing/utils/dict_array_converter.h b/src/paimon/testing/utils/dict_array_converter.h index e24b70ec..14ee83be 100644 --- a/src/paimon/testing/utils/dict_array_converter.h +++ b/src/paimon/testing/utils/dict_array_converter.h @@ -30,7 +30,8 @@ class DictArrayConverter { DictArrayConverter() = delete; ~DictArrayConverter() = delete; - // deep copy dictionary array to string array/binary array + // Decode dictionary string arrays to plain StringArray so test comparisons are stable across + // Arrow dictionary index types and string/large_string dictionary values. static Result> ConvertDictArray( const std::shared_ptr& array, arrow::MemoryPool* pool) { arrow::Type::type kind = array->type_id(); @@ -93,21 +94,15 @@ class DictArrayConverter { auto dict_type = arrow::internal::checked_pointer_cast( dict_array->type()); auto value_type_id = dict_type->value_type()->id(); - auto index_type_id = dict_type->index_type()->id(); - if (value_type_id == arrow::Type::type::STRING && - index_type_id == arrow::Type::type::INT32) { - return ConvertDictionaryArrayToBinaryArray< - arrow::StringArray, arrow::Int32Array, arrow::StringBuilder>(dict_array, - pool); - } else if (value_type_id == arrow::Type::type::LARGE_STRING && - index_type_id == arrow::Type::type::INT64) { - return ConvertDictionaryArrayToBinaryArray< - arrow::LargeStringArray, arrow::Int64Array, arrow::StringBuilder>( - dict_array, pool); + if (value_type_id == arrow::Type::type::STRING) { + return ConvertDictionaryArrayToStringArray(dict_array, + pool); + } else if (value_type_id == arrow::Type::type::LARGE_STRING) { + return ConvertDictionaryArrayToStringArray(dict_array, + pool); } else { return Status::Invalid( - "only support [STRING, INT32] or [LARGE_STRING, INT64] for " - "DictionaryArray"); + "only support STRING or LARGE_STRING value type for DictionaryArray"); } } default: { @@ -117,23 +112,25 @@ class DictArrayConverter { } private: - template - static Result> ConvertDictionaryArrayToBinaryArray( + template + static Result> ConvertDictionaryArrayToStringArray( const std::shared_ptr& dict_array, arrow::MemoryPool* pool) { auto dictionary = std::dynamic_pointer_cast(dict_array->dictionary()); - auto indices = std::dynamic_pointer_cast(dict_array->indices()); - auto string_builder = std::make_shared(pool); + if (!dictionary) { + return Status::Invalid("dictionary value array type does not match dictionary type"); + } + + arrow::StringBuilder string_builder(pool); for (int64_t i = 0; i < dict_array->length(); ++i) { if (dict_array->IsNull(i)) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder->AppendNull()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder.AppendNull()); } else { - int64_t dict_index = indices->Value(i); PAIMON_RETURN_NOT_OK_FROM_ARROW( - string_builder->Append(dictionary->GetString(dict_index))); + string_builder.Append(dictionary->GetString(dict_array->GetValueIndex(i)))); } } std::shared_ptr string_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder->Finish(&string_array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(string_builder.Finish(&string_array)); return string_array; } }; diff --git a/third_party/jindosdk-nextarch/CMakeLists.txt b/third_party/jindosdk-nextarch/CMakeLists.txt index 1f921097..2ff20461 100644 --- a/third_party/jindosdk-nextarch/CMakeLists.txt +++ b/third_party/jindosdk-nextarch/CMakeLists.txt @@ -28,7 +28,8 @@ find_package(jindosdk_c REQUIRED) file(GLOB_RECURSE JINDOSDK_SOURCES src/*.cpp) add_library(jindosdk-nextarch STATIC ${JINDOSDK_SOURCES}) -target_link_libraries(jindosdk-nextarch PUBLIC pthread dl JINDOSDK::JINDOSDK) +target_link_libraries(jindosdk-nextarch PUBLIC pthread ${CMAKE_DL_LIBS} + JINDOSDK::JINDOSDK) target_include_directories(jindosdk-nextarch PUBLIC $ $) From e98acc955176c6a34caaef362728df3884d257ce Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Thu, 28 May 2026 14:11:59 +0800 Subject: [PATCH 008/138] fix: fix undefined behavior in bitwise left-shift operations across codebase From 002daa460e96a3bbaf58360e3830086194f43b99 Mon Sep 17 00:00:00 2001 From: liangjie Date: Thu, 28 May 2026 16:45:24 +0800 Subject: [PATCH 009/138] feat: optimize parquet reads with page-level filtering --- cmake_modules/arrow.diff | 187 +++++ .../arrow/arrow_input_stream_adapter.cpp | 1 + .../operation/key_value_file_store_scan.cpp | 1 + src/paimon/format/parquet/CMakeLists.txt | 10 +- .../format/parquet/column_index_filter.cpp | 718 +++++++++++++++++ .../format/parquet/column_index_filter.h | 177 +++++ .../parquet/column_index_filter_test.cpp | 486 ++++++++++++ .../format/parquet/file_reader_wrapper.cpp | 547 ++++++++++--- .../format/parquet/file_reader_wrapper.h | 97 ++- .../parquet/file_reader_wrapper_test.cpp | 139 +++- .../page_filtered_row_group_reader.cpp | 369 +++++++++ .../parquet/page_filtered_row_group_reader.h | 109 +++ .../page_filtered_row_group_reader_test.cpp | 725 ++++++++++++++++++ .../parquet/parquet_file_batch_reader.cpp | 318 +++++--- .../parquet/parquet_file_batch_reader.h | 10 + .../format/parquet/parquet_format_defs.h | 10 + .../format/parquet/parquet_writer_builder.cpp | 9 + src/paimon/format/parquet/row_ranges.cpp | 137 ++++ src/paimon/format/parquet/row_ranges.h | 108 +++ test/inte/append_compaction_inte_test.cpp | 2 +- test/inte/scan_and_read_inte_test.cpp | 1 + test/inte/write_and_read_inte_test.cpp | 233 ++++++ test/inte/write_inte_test.cpp | 8 +- 23 files changed, 4190 insertions(+), 212 deletions(-) create mode 100644 src/paimon/format/parquet/column_index_filter.cpp create mode 100644 src/paimon/format/parquet/column_index_filter.h create mode 100644 src/paimon/format/parquet/column_index_filter_test.cpp create mode 100644 src/paimon/format/parquet/page_filtered_row_group_reader.cpp create mode 100644 src/paimon/format/parquet/page_filtered_row_group_reader.h create mode 100644 src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp create mode 100644 src/paimon/format/parquet/row_ranges.cpp create mode 100644 src/paimon/format/parquet/row_ranges.h diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff index 0d55f2be..f61b61ca 100644 --- a/cmake_modules/arrow.diff +++ b/cmake_modules/arrow.diff @@ -196,6 +196,193 @@ index 4d3acb491e..3906ff3c59 100644 int64_t pagesize_; ParquetDataPageVersion parquet_data_page_version_; ParquetVersion::type parquet_version_; + +--- a/cpp/src/parquet/file_reader.h ++++ b/cpp/src/parquet/file_reader.h +@@ -210,6 +210,17 @@ + ::arrow::Future<> WhenBuffered(const std::vector& row_groups, + const std::vector& column_indices) const; + ++ /// Pre-buffer arbitrary byte ranges (e.g., page-level ranges from OffsetIndex). ++ /// Unlike PreBuffer(), this does NOT set the column bitmap, so ++ /// GetColumnPageReader will use CachedInputStream (page-level cache path). ++ void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges, ++ const ::arrow::io::IOContext& ctx, ++ const ::arrow::io::CacheOptions& options); ++ ++ /// Wait for arbitrary byte ranges to be pre-buffered. ++ ::arrow::Future<> WhenBufferedRanges( ++ const std::vector<::arrow::io::ReadRange>& ranges) const; ++ + private: + // Holds a pointer to an instance of Contents implementation + std::unique_ptr contents_; + +--- a/cpp/src/parquet/file_reader.cc ++++ b/cpp/src/parquet/file_reader.cc +@@ -207,6 +207,100 @@ + return {col_start, col_length}; + } + ++// CachedInputStream: InputStream adapter that reads through ReadRangeCache with ++// zero-cost skip for non-cached pages. Used for page-level caching where only ++// specific pages are pre-buffered. ++// ++// Key behavior: ++// - Read(): On cache hit, returns cached data. On cache miss, returns zero-filled ++// buffer (zero I/O). This makes InputStream::Advance() (which calls Read() and ++// discards) effectively free for skipped pages. ++// - Peek(): Always falls back to source on cache miss, because PageReader uses ++// Peek() to read Thrift page headers (~30 bytes) which must have real data. ++class CachedInputStream : public ::arrow::io::InputStream { ++ public: ++ CachedInputStream( ++ std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache, ++ std::shared_ptr source, ++ int64_t offset, int64_t length) ++ : cache_(std::move(cache)), ++ source_(std::move(source)), ++ base_offset_(offset), ++ length_(length) {} ++ ++ ::arrow::Status Close() override { ++ closed_ = true; ++ return ::arrow::Status::OK(); ++ } ++ ++ bool closed() const override { return closed_; } ++ ++ ::arrow::Result Tell() const override { return position_; } ++ ++ ::arrow::Result Peek(int64_t nbytes) override { ++ int64_t to_read = std::min(nbytes, length_ - position_); ++ if (to_read <= 0) { ++ return std::string_view(); ++ } ++ ::arrow::io::ReadRange range{base_offset_ + position_, to_read}; ++ auto result = cache_->Read(range); ++ if (result.ok()) { ++ peek_buffer_ = *result; ++ } else { ++ // Peek is used for Thrift page headers (~30 bytes) — must read real data ++ ARROW_ASSIGN_OR_RAISE(peek_buffer_, ++ source_->ReadAt(range.offset, range.length)); ++ } ++ return std::string_view( ++ reinterpret_cast(peek_buffer_->data()), ++ static_cast(peek_buffer_->size())); ++ } ++ ++ ::arrow::Result Read(int64_t nbytes, void* out) override { ++ int64_t to_read = std::min(nbytes, length_ - position_); ++ if (to_read <= 0) return 0; ++ ::arrow::io::ReadRange range{base_offset_ + position_, to_read}; ++ auto result = cache_->Read(range); ++ if (result.ok()) { ++ auto& buf = *result; ++ memcpy(out, buf->data(), static_cast(buf->size())); ++ position_ += buf->size(); ++ return buf->size(); ++ } ++ // Cache miss: fall back to real I/O from source ++ ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length)); ++ memcpy(out, buf->data(), static_cast(buf->size())); ++ position_ += buf->size(); ++ return buf->size(); ++ } ++ ++ ::arrow::Result> Read(int64_t nbytes) override { ++ int64_t to_read = std::min(nbytes, length_ - position_); ++ if (to_read <= 0) { ++ return std::make_shared<::arrow::Buffer>(nullptr, 0); ++ } ++ ::arrow::io::ReadRange range{base_offset_ + position_, to_read}; ++ auto result = cache_->Read(range); ++ if (result.ok()) { ++ position_ += (*result)->size(); ++ return *result; ++ } ++ // Cache miss: fall back to real I/O from source ++ ARROW_ASSIGN_OR_RAISE(auto buf, source_->ReadAt(range.offset, range.length)); ++ position_ += buf->size(); ++ return std::shared_ptr<::arrow::Buffer>(std::move(buf)); ++ } ++ ++ private: ++ std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache_; ++ std::shared_ptr source_; ++ int64_t base_offset_; ++ int64_t length_; ++ int64_t position_ = 0; ++ bool closed_ = false; ++ std::shared_ptr<::arrow::Buffer> peek_buffer_; ++}; ++ + // RowGroupReader::Contents implementation for the Parquet file specification + class SerializedRowGroup : public RowGroupReader::Contents { + public: +@@ -242,6 +336,11 @@ + // segments. + PARQUET_ASSIGN_OR_THROW(auto buffer, cached_source_->Read(col_range)); + stream = std::make_shared<::arrow::io::BufferReader>(buffer); ++ } else if (cached_source_) { ++ // Page-level caching: read through cache with fallback to source. ++ // Advance() is zero-cost for skipped pages via data_page_filter. ++ stream = std::make_shared( ++ cached_source_, source_, col_range.offset, col_range.length); + } else { + stream = properties_.GetStream(source_, col_range.offset, col_range.length); + } +@@ -417,6 +516,26 @@ + return cached_source_->WaitFor(ranges); + } + ++ void PreBufferRanges(const std::vector<::arrow::io::ReadRange>& ranges, ++ const ::arrow::io::IOContext& ctx, ++ const ::arrow::io::CacheOptions& options) { ++ cached_source_ = ++ std::make_shared<::arrow::io::internal::ReadRangeCache>(source_, ctx, options); ++ // Do NOT set prebuffered_column_chunks_ bitmap — GetColumnPageReader will ++ // use CachedInputStream path instead of full-chunk BufferReader path. ++ prebuffered_column_chunks_.clear(); ++ PARQUET_THROW_NOT_OK(cached_source_->Cache(ranges)); ++ } ++ ++ ::arrow::Future<> WhenBufferedRanges( ++ const std::vector<::arrow::io::ReadRange>& ranges) const { ++ if (!cached_source_) { ++ return ::arrow::Status::Invalid( ++ "Must call PreBufferRanges before WhenBufferedRanges"); ++ } ++ return cached_source_->WaitFor(ranges); ++ } ++ + // Metadata/footer parsing. Divided up to separate sync/async paths, and to use + // exceptions for error handling (with the async path converting to Future/Status). + +@@ -911,6 +1030,22 @@ + return file->WhenBuffered(row_groups, column_indices); + } + ++void ParquetFileReader::PreBufferRanges( ++ const std::vector<::arrow::io::ReadRange>& ranges, ++ const ::arrow::io::IOContext& ctx, ++ const ::arrow::io::CacheOptions& options) { ++ SerializedFile* file = ++ ::arrow::internal::checked_cast(contents_.get()); ++ file->PreBufferRanges(ranges, ctx, options); ++} ++ ++::arrow::Future<> ParquetFileReader::WhenBufferedRanges( ++ const std::vector<::arrow::io::ReadRange>& ranges) const { ++ SerializedFile* file = ++ ::arrow::internal::checked_cast(contents_.get()); ++ return file->WhenBufferedRanges(ranges); ++} ++ + // ---------------------------------------------------------------------- + // File metadata helpers + diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake diff --git a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp index 941c18dd..adf3cd8f 100644 --- a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp +++ b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp @@ -20,6 +20,7 @@ #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include +#include #include #include "arrow/api.h" diff --git a/src/paimon/core/operation/key_value_file_store_scan.cpp b/src/paimon/core/operation/key_value_file_store_scan.cpp index 80f6b807..03550e42 100644 --- a/src/paimon/core/operation/key_value_file_store_scan.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan.cpp @@ -70,6 +70,7 @@ Result> KeyValueFileStoreScan::Create( scan->SplitAndSetFilter(table_schema->PartitionKeys(), arrow_schema, scan_filters)); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_pk, table_schema->TrimmedPrimaryKeys()); PAIMON_RETURN_NOT_OK(scan->SplitAndSetKeyValueFilter(trimmed_pk)); + return scan; } diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 2b993688..8f78fd6c 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -18,13 +18,16 @@ set(PAIMON_PARQUET_FILE_FORMAT parquet_field_id_converter.cpp predicate_converter.cpp file_reader_wrapper.cpp + page_filtered_row_group_reader.cpp parquet_timestamp_converter.cpp parquet_file_batch_reader.cpp parquet_file_format_factory.cpp parquet_format_writer.cpp parquet_schema_util.cpp parquet_stats_extractor.cpp - parquet_writer_builder.cpp) + parquet_writer_builder.cpp + row_ranges.cpp + column_index_filter.cpp) add_paimon_lib(paimon_parquet_file_format SOURCES @@ -44,10 +47,14 @@ add_paimon_lib(paimon_parquet_file_format SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) +target_include_directories(paimon_parquet_file_format_objlib SYSTEM + PRIVATE "${ARROW_SOURCE_DIR}/cpp/src") + if(PAIMON_BUILD_TESTS) add_paimon_test(parquet_format_test SOURCES file_reader_wrapper_test.cpp + page_filtered_row_group_reader_test.cpp parquet_timestamp_converter_test.cpp parquet_field_id_converter_test.cpp parquet_file_batch_reader_test.cpp @@ -56,6 +63,7 @@ if(PAIMON_BUILD_TESTS) parquet_writer_builder_test.cpp predicate_converter_test.cpp predicate_pushdown_test.cpp + column_index_filter_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static diff --git a/src/paimon/format/parquet/column_index_filter.cpp b/src/paimon/format/parquet/column_index_filter.cpp new file mode 100644 index 00000000..43a0e9df --- /dev/null +++ b/src/paimon/format/parquet/column_index_filter.cpp @@ -0,0 +1,718 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/column_index_filter.h" + +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/data/decimal.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/function.h" +#include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/literal.h" + +namespace paimon::parquet { + +Result ColumnIndexFilter::CalculateRowRanges( + const std::shared_ptr& predicate, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, + const std::map& column_name_to_index, int32_t row_group_index, + int64_t row_group_row_count) { + if (!predicate || !page_index_reader) { + return RowRanges::CreateSingle(row_group_row_count); + } + + auto rg_page_index_reader = page_index_reader->RowGroup(row_group_index); + if (!rg_page_index_reader) { + return RowRanges::CreateSingle(row_group_row_count); + } + + return VisitPredicate(predicate, column_name_to_index, row_group_row_count, + rg_page_index_reader.get()); +} + +Result ColumnIndexFilter::VisitPredicate( + const std::shared_ptr& predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader) { + if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { + return VisitLeafPredicate(leaf_predicate, column_name_to_index, row_group_row_count, + rg_page_index_reader); + } + + if (auto compound_predicate = std::dynamic_pointer_cast(predicate)) { + return VisitCompoundPredicate(compound_predicate, column_name_to_index, row_group_row_count, + rg_page_index_reader); + } + + return Status::Invalid("Unknown predicate type"); +} + +Result ColumnIndexFilter::VisitLeafPredicate( + const std::shared_ptr& leaf_predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader) { + const std::string& field_name = leaf_predicate->FieldName(); + auto it = column_name_to_index.find(field_name); + if (it == column_name_to_index.end()) { + // Predicates referencing fields absent from the data file are stripped + // upstream by FieldMappingBuilder, so reaching here indicates a contract + // violation by the caller. + return Status::Invalid( + fmt::format("column '{}' not found in column_name_to_index", field_name)); + } + const auto& function = leaf_predicate->GetFunction(); + auto function_type = function.GetType(); + + int32_t column_index = it->second; + auto column_index_ptr = rg_page_index_reader->GetColumnIndex(column_index); + auto offset_index_ptr = rg_page_index_reader->GetOffsetIndex(column_index); + + if (!column_index_ptr || !offset_index_ptr) { + // Column index or offset index not available, return all rows + return RowRanges::CreateSingle(row_group_row_count); + } + + const auto& literals = leaf_predicate->Literals(); + FieldType field_type = leaf_predicate->GetFieldType(); + + std::vector matching_pages; + + switch (function_type) { + case Function::Type::IS_NULL: + matching_pages = FilterPagesByIsNull(column_index_ptr); + break; + case Function::Type::IS_NOT_NULL: + matching_pages = FilterPagesByIsNotNull(column_index_ptr); + break; + case Function::Type::EQUAL: + if (!literals.empty()) { + matching_pages = FilterPagesByEqual(column_index_ptr, literals[0], field_type); + } + break; + case Function::Type::NOT_EQUAL: + if (!literals.empty()) { + matching_pages = FilterPagesByNotEqual(column_index_ptr, literals[0], field_type); + } + break; + case Function::Type::LESS_THAN: + if (!literals.empty()) { + matching_pages = FilterPagesByLessThan(column_index_ptr, literals[0], field_type); + } + break; + case Function::Type::LESS_OR_EQUAL: + if (!literals.empty()) { + matching_pages = + FilterPagesByLessOrEqual(column_index_ptr, literals[0], field_type); + } + break; + case Function::Type::GREATER_THAN: + if (!literals.empty()) { + matching_pages = + FilterPagesByGreaterThan(column_index_ptr, literals[0], field_type); + } + break; + case Function::Type::GREATER_OR_EQUAL: + if (!literals.empty()) { + matching_pages = + FilterPagesByGreaterOrEqual(column_index_ptr, literals[0], field_type); + } + break; + case Function::Type::IN: + matching_pages = FilterPagesByIn(column_index_ptr, literals, field_type); + break; + case Function::Type::NOT_IN: + matching_pages = FilterPagesByNotIn(column_index_ptr, literals); + break; + default: + // Unsupported function type for column index filtering + return RowRanges::CreateSingle(row_group_row_count); + } + + return BuildRowRangesFromPageIndices(matching_pages, offset_index_ptr, row_group_row_count); +} + +Result ColumnIndexFilter::VisitCompoundPredicate( + const std::shared_ptr& compound_predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader) { + const auto& children = compound_predicate->Children(); + const auto& function = compound_predicate->GetFunction(); + auto function_type = function.GetType(); + + if (children.empty()) { + return RowRanges::CreateSingle(row_group_row_count); + } + + // Calculate row ranges for first child + PAIMON_ASSIGN_OR_RAISE(RowRanges result, + VisitPredicate(children[0], column_name_to_index, row_group_row_count, + rg_page_index_reader)); + + if (function_type == Function::Type::AND) { + // Short-circuit: if result is empty, no need to continue + if (result.IsEmpty()) { + return result; + } + + for (size_t i = 1; i < children.size(); ++i) { + PAIMON_ASSIGN_OR_RAISE(RowRanges child_ranges, + VisitPredicate(children[i], column_name_to_index, + row_group_row_count, rg_page_index_reader)); + + result = RowRanges::Intersection(result, child_ranges); + + // Short-circuit: if result is empty, no need to continue + if (result.IsEmpty()) { + return result; + } + } + } else if (function_type == Function::Type::OR) { + // Short-circuit: if result already covers all rows, no need to continue + if (result.RowCount() == row_group_row_count) { + return result; + } + + for (size_t i = 1; i < children.size(); ++i) { + PAIMON_ASSIGN_OR_RAISE(RowRanges child_ranges, + VisitPredicate(children[i], column_name_to_index, + row_group_row_count, rg_page_index_reader)); + + result = RowRanges::Union(result, child_ranges); + + // Short-circuit: if result already covers all rows, no need to continue + if (result.RowCount() == row_group_row_count) { + return result; + } + } + } else { + return Status::Invalid("Unknown compound predicate type"); + } + + return result; +} + +std::vector ColumnIndexFilter::FilterPagesByEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + + if (literal.IsNull()) { + // value = NULL is UNKNOWN for any value. No rows can match. + return matching_pages; + } + + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + const auto& max_values = column_index->encoded_max_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainEqual(min_values[i], max_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByNotEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + + if (literal.IsNull()) { + // value != NULL is UNKNOWN for any value. No rows can match. + return matching_pages; + } + + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + const auto& max_values = column_index->encoded_max_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + // Null-only pages: NULL != x is NULL (UNKNOWN) in SQL semantics, + // which evaluates to false. Skip null-only pages for NOT_EQUAL. + continue; + } + + // Try to exclude pages where min == max == literal (all non-null values equal literal). + // NULL != literal is NULL (UNKNOWN) in SQL, so nulls don't produce true either. + auto cmp_min = CompareEncodedWithLiteral(min_values[i], literal, field_type); + auto cmp_max = CompareEncodedWithLiteral(max_values[i], literal, field_type); + if (cmp_min.has_value() && cmp_max.has_value() && *cmp_min == 0 && *cmp_max == 0) { + // min == max == literal: all non-null values equal literal, and nulls + // don't satisfy != either. Skip this page entirely. + continue; + } + + matching_pages.push_back(i); + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByLessThan( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainLessThan(min_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByLessOrEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainLessOrEqual(min_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByGreaterThan( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& max_values = column_index->encoded_max_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainGreaterThan(max_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByGreaterOrEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& max_values = column_index->encoded_max_values(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + continue; + } + + if (PageMightContainGreaterOrEqual(max_values[i], literal, field_type)) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByIsNull( + const std::shared_ptr<::parquet::ColumnIndex>& column_index) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& null_counts = column_index->null_counts(); + bool has_null_counts = column_index->has_null_counts(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + matching_pages.push_back(i); + continue; + } + + if (has_null_counts && null_counts[i] > 0) { + matching_pages.push_back(i); + } else if (!has_null_counts) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByIsNotNull( + const std::shared_ptr<::parquet::ColumnIndex>& column_index) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + auto num_pages = static_cast(null_pages.size()); + + for (int32_t i = 0; i < num_pages; ++i) { + if (!null_pages[i]) { + matching_pages.push_back(i); + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByIn( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const std::vector& literals, FieldType field_type) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + const auto& min_values = column_index->encoded_min_values(); + const auto& max_values = column_index->encoded_max_values(); + const auto& null_counts = column_index->null_counts(); + bool has_null_counts = column_index->has_null_counts(); + auto num_pages = static_cast(null_pages.size()); + + bool has_null = + std::any_of(literals.begin(), literals.end(), [](const Literal& l) { return l.IsNull(); }); + + // Pages outer loop, literals inner loop with early break when page is matched. + // Naturally produces sorted output, avoids unordered_set overhead. + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + // All-null page: include only if IN list contains null + if (has_null) { + matching_pages.push_back(i); + } + continue; + } + + // Check null-in-list match for non-all-null pages + if (has_null) { + if ((has_null_counts && null_counts[i] > 0) || !has_null_counts) { + matching_pages.push_back(i); + continue; // Already matched, skip literal checks + } + } + + // Check non-null literals against page min/max with early break + for (const auto& literal : literals) { + if (literal.IsNull()) { + continue; + } + if (PageMightContainEqual(min_values[i], max_values[i], literal, field_type)) { + matching_pages.push_back(i); + break; // Page matched, no need to check more literals + } + } + } + + return matching_pages; +} + +std::vector ColumnIndexFilter::FilterPagesByNotIn( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const std::vector& literals) { + std::vector matching_pages; + const auto& null_pages = column_index->null_pages(); + auto num_pages = static_cast(null_pages.size()); + + bool has_null = false; + for (const auto& literal : literals) { + if (literal.IsNull()) { + has_null = true; + break; + } + } + + if (has_null) { + // NOT_IN list contains null → value NOT IN (..., NULL, ...) evaluates to + // UNKNOWN for every value (because it expands to AND(..., value != NULL, ...) + // and value != NULL is always UNKNOWN). No rows can match. + return matching_pages; + } + + for (int32_t i = 0; i < num_pages; ++i) { + if (null_pages[i]) { + // Null-only pages: NULL NOT IN (non-null values) is UNKNOWN, skip. + continue; + } + + // Non-null pages could contain values not in the list + matching_pages.push_back(i); + } + + return matching_pages; +} + +RowRanges ColumnIndexFilter::BuildRowRangesFromPageIndices( + const std::vector& page_indices, + const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count) { + if (page_indices.empty()) { + return RowRanges::CreateEmpty(); + } + + const auto& page_locations = offset_index->page_locations(); + RowRanges ranges; + + for (int32_t page_idx : page_indices) { + if (page_idx < 0 || page_idx >= static_cast(page_locations.size())) { + continue; + } + + int64_t first_row_index = page_locations[page_idx].first_row_index; + + int64_t last_row_index; + if (page_idx + 1 < static_cast(page_locations.size())) { + last_row_index = page_locations[page_idx + 1].first_row_index - 1; + } else { + last_row_index = row_group_row_count - 1; + } + + ranges.Add(RowRanges::Range(first_row_index, last_row_index)); + } + + return ranges; +} + +std::optional ColumnIndexFilter::CompareEncodedWithLiteral(const std::string& encoded, + const Literal& literal, + FieldType field_type) { + if (literal.IsNull()) { + return std::nullopt; + } + + switch (field_type) { + case FieldType::BOOLEAN: { + if (encoded.size() < 1) { + return std::nullopt; + } + int32_t enc_val = (encoded[0] != 0) ? 1 : 0; + int32_t lit_val = literal.GetValue() ? 1 : 0; + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::DATE: { + if (encoded.size() < sizeof(int32_t)) { + return std::nullopt; + } + int32_t enc_val; + std::memcpy(&enc_val, encoded.data(), sizeof(int32_t)); + int32_t lit_val; + if (field_type == FieldType::TINYINT) { + lit_val = static_cast(literal.GetValue()); + } else if (field_type == FieldType::SMALLINT) { + lit_val = static_cast(literal.GetValue()); + } else { + lit_val = literal.GetValue(); + } + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + case FieldType::BIGINT: { + if (encoded.size() < sizeof(int64_t)) { + return std::nullopt; + } + int64_t enc_val; + std::memcpy(&enc_val, encoded.data(), sizeof(int64_t)); + auto lit_val = literal.GetValue(); + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + case FieldType::FLOAT: { + if (encoded.size() < sizeof(float)) { + return std::nullopt; + } + float enc_val; + std::memcpy(&enc_val, encoded.data(), sizeof(float)); + auto lit_val = literal.GetValue(); + if (std::isnan(enc_val) || std::isnan(lit_val)) { + return std::nullopt; + } + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + case FieldType::DOUBLE: { + if (encoded.size() < sizeof(double)) { + return std::nullopt; + } + double enc_val; + std::memcpy(&enc_val, encoded.data(), sizeof(double)); + auto lit_val = literal.GetValue(); + if (std::isnan(enc_val) || std::isnan(lit_val)) { + return std::nullopt; + } + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + case FieldType::STRING: + case FieldType::BINARY: { + auto lit_val = literal.GetValue(); + int cmp = encoded.compare(lit_val); + return (cmp < 0) ? -1 : (cmp > 0) ? 1 : 0; + } + case FieldType::DECIMAL: { + // Parquet stores DECIMAL as INT32, INT64, or FIXED_LEN_BYTE_ARRAY depending + // on precision. All are stored as unscaled integer values. + auto lit_decimal = literal.GetValue(); + Decimal::int128_t lit_val = lit_decimal.Value(); + Decimal::int128_t enc_val; + + if (encoded.size() == sizeof(int32_t)) { + // INT32 physical type (precision <= 9) + int32_t raw; + std::memcpy(&raw, encoded.data(), sizeof(int32_t)); + enc_val = static_cast(raw); + } else if (encoded.size() == sizeof(int64_t)) { + // INT64 physical type (precision <= 18) + int64_t raw; + std::memcpy(&raw, encoded.data(), sizeof(int64_t)); + enc_val = static_cast(raw); + } else { + // FIXED_LEN_BYTE_ARRAY / BYTE_ARRAY: big-endian two's complement. + // Defer to Decimal::FromUnscaledBytes so endianness, padding, and + // sign extension stay consistent with parquet_stats_extractor. + if (encoded.empty()) { + return std::nullopt; + } + Bytes bytes(encoded, GetDefaultPool().get()); + enc_val = + Decimal::FromUnscaledBytes(lit_decimal.Precision(), lit_decimal.Scale(), &bytes) + .Value(); + } + + return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0; + } + default: + // TIMESTAMP, etc. - not yet supported for page-level filtering. + // TIMESTAMP is blocked at predicate_converter level (returns NotImplemented). + // Return nullopt to fall back to safe behavior (include page). + return std::nullopt; + } +} + +bool ColumnIndexFilter::PageMightContainEqual(const std::string& encoded_min, + const std::string& encoded_max, + const Literal& literal, FieldType field_type) { + if (literal.IsNull()) { + return false; // Null is handled separately via null_pages + } + + // Page might contain equal if min <= literal <= max + auto cmp_min = CompareEncodedWithLiteral(encoded_min, literal, field_type); + if (!cmp_min.has_value()) { + return true; // Can't compare, assume match + } + if (*cmp_min > 0) { + return false; // min > literal + } + + auto cmp_max = CompareEncodedWithLiteral(encoded_max, literal, field_type); + if (!cmp_max.has_value()) { + return true; + } + if (*cmp_max < 0) { + return false; // max < literal + } + + return true; // min <= literal <= max +} + +bool ColumnIndexFilter::PageMightContainLessThan(const std::string& encoded_min, + const Literal& literal, FieldType field_type) { + if (literal.IsNull()) { + return false; + } + + // Page might contain values < literal if min < literal + auto cmp_min = CompareEncodedWithLiteral(encoded_min, literal, field_type); + if (!cmp_min.has_value()) { + return true; + } + return *cmp_min < 0; +} + +bool ColumnIndexFilter::PageMightContainLessOrEqual(const std::string& encoded_min, + const Literal& literal, FieldType field_type) { + if (literal.IsNull()) { + return false; + } + + // Page might contain values <= literal if min <= literal + auto cmp_min = CompareEncodedWithLiteral(encoded_min, literal, field_type); + if (!cmp_min.has_value()) { + return true; + } + return *cmp_min <= 0; +} + +bool ColumnIndexFilter::PageMightContainGreaterThan(const std::string& encoded_max, + const Literal& literal, FieldType field_type) { + if (literal.IsNull()) { + return false; + } + + // Page might contain values > literal if max > literal + auto cmp_max = CompareEncodedWithLiteral(encoded_max, literal, field_type); + if (!cmp_max.has_value()) { + return true; + } + return *cmp_max > 0; +} + +bool ColumnIndexFilter::PageMightContainGreaterOrEqual(const std::string& encoded_max, + const Literal& literal, + FieldType field_type) { + if (literal.IsNull()) { + return false; + } + + // Page might contain values >= literal if max >= literal + auto cmp_max = CompareEncodedWithLiteral(encoded_max, literal, field_type); + if (!cmp_max.has_value()) { + return true; + } + return *cmp_max >= 0; +} + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/column_index_filter.h b/src/paimon/format/parquet/column_index_filter.h new file mode 100644 index 00000000..56bb816e --- /dev/null +++ b/src/paimon/format/parquet/column_index_filter.h @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/defs.h" +#include "paimon/format/parquet/row_ranges.h" +#include "paimon/predicate/predicate.h" +#include "paimon/result.h" +#include "parquet/page_index.h" + +namespace paimon { +class CompoundPredicate; +class LeafPredicate; +class Literal; +} // namespace paimon + +namespace paimon::parquet { + +/// ColumnIndexFilter calculates row ranges based on ColumnIndex statistics. +/// It uses the min/max values in the column index to determine which pages +/// might contain rows matching the predicate. +/// +/// The computed RowRanges serve two purposes: +/// 1. Row-group elimination: if no pages match, the entire row group is skipped. +/// 2. Page-level skipping: for partially matched row groups, RowRanges are passed +/// to PageFilteredRowGroupReader which uses data_page_filter to skip +/// non-matching pages at the I/O level, and SkipRecords/ReadRecords to skip +/// non-matching rows at the decode level within kept pages. +class ColumnIndexFilter { + public: + ColumnIndexFilter() = delete; + + /// Calculate row ranges based on predicate and column indices. + /// @param predicate The predicate to evaluate. + /// @param page_index_reader The page index reader for the file. + /// @param column_name_to_index Map from column name to column index. + /// @param row_group_index The row group index to filter. + /// @param row_group_row_count The number of rows in the row group. + /// @return RowRanges that may contain matching rows. + static Result CalculateRowRanges( + const std::shared_ptr& predicate, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, + const std::map& column_name_to_index, int32_t row_group_index, + int64_t row_group_row_count); + + private: + /// Visit a predicate and calculate row ranges. + static Result VisitPredicate( + const std::shared_ptr& predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader); + + /// Visit a leaf predicate and calculate row ranges. + static Result VisitLeafPredicate( + const std::shared_ptr& leaf_predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader); + + /// Visit a compound predicate (AND/OR) and calculate row ranges. + static Result VisitCompoundPredicate( + const std::shared_ptr& compound_predicate, + const std::map& column_name_to_index, int64_t row_group_row_count, + ::parquet::RowGroupPageIndexReader* rg_page_index_reader); + + /// Filter pages based on column index statistics for EQUAL predicate. + static std::vector FilterPagesByEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for NOT_EQUAL predicate. + static std::vector FilterPagesByNotEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for LESS_THAN predicate. + static std::vector FilterPagesByLessThan( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for LESS_OR_EQUAL predicate. + static std::vector FilterPagesByLessOrEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for GREATER_THAN predicate. + static std::vector FilterPagesByGreaterThan( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for GREATER_OR_EQUAL predicate. + static std::vector FilterPagesByGreaterOrEqual( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, const Literal& literal, + FieldType field_type); + + /// Filter pages based on column index statistics for IS_NULL predicate. + static std::vector FilterPagesByIsNull( + const std::shared_ptr<::parquet::ColumnIndex>& column_index); + + /// Filter pages based on column index statistics for IS_NOT_NULL predicate. + static std::vector FilterPagesByIsNotNull( + const std::shared_ptr<::parquet::ColumnIndex>& column_index); + + /// Filter pages based on column index statistics for IN predicate. + static std::vector FilterPagesByIn( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const std::vector& literals, FieldType field_type); + + /// Filter pages based on column index statistics for NOT_IN predicate. + static std::vector FilterPagesByNotIn( + const std::shared_ptr<::parquet::ColumnIndex>& column_index, + const std::vector& literals); + + /// Build row ranges from page indices (must be sorted in ascending order). + static RowRanges BuildRowRangesFromPageIndices( + const std::vector& page_indices, + const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); + + /// Compare a parquet encoded value with a Literal. + /// @return -1 if encoded < literal, 0 if equal, 1 if encoded > literal. + /// nullopt if comparison cannot be performed (unsupported type, etc.). + static std::optional CompareEncodedWithLiteral(const std::string& encoded, + const Literal& literal, + FieldType field_type); + + /// Check if a page might contain a value equal to the literal. + /// Condition: min <= literal <= max + static bool PageMightContainEqual(const std::string& encoded_min, + const std::string& encoded_max, const Literal& literal, + FieldType field_type); + + /// Check if a page might contain values less than the literal. + /// Condition: min < literal + static bool PageMightContainLessThan(const std::string& encoded_min, const Literal& literal, + FieldType field_type); + + /// Check if a page might contain values less than or equal to the literal. + /// Condition: min <= literal + static bool PageMightContainLessOrEqual(const std::string& encoded_min, const Literal& literal, + FieldType field_type); + + /// Check if a page might contain values greater than the literal. + /// Condition: max > literal + static bool PageMightContainGreaterThan(const std::string& encoded_max, const Literal& literal, + FieldType field_type); + + /// Check if a page might contain values greater than or equal to the literal. + /// Condition: max >= literal + static bool PageMightContainGreaterOrEqual(const std::string& encoded_max, + const Literal& literal, FieldType field_type); +}; + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp new file mode 100644 index 00000000..f5d84389 --- /dev/null +++ b/src/paimon/format/parquet/column_index_filter_test.cpp @@ -0,0 +1,486 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/column_index_filter.h" + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/defs.h" +#include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_format_writer.h" +#include "paimon/format/parquet/row_ranges.h" +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" +#include "parquet/file_reader.h" + +namespace paimon::parquet::test { + +// ===================================================================== +// RowRanges unit tests +// ===================================================================== + +class RowRangesTest : public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(RowRangesTest, TestCreateSingle) { + RowRanges ranges = RowRanges::CreateSingle(100); + EXPECT_FALSE(ranges.IsEmpty()); + EXPECT_EQ(100, ranges.RowCount()); + EXPECT_EQ(1, ranges.GetRanges().size()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(99, ranges.GetRanges()[0].to); +} + +TEST_F(RowRangesTest, TestCreateEmpty) { + RowRanges ranges = RowRanges::CreateEmpty(); + EXPECT_TRUE(ranges.IsEmpty()); + EXPECT_EQ(0, ranges.RowCount()); + EXPECT_EQ(0, ranges.GetRanges().size()); +} + +TEST_F(RowRangesTest, TestAddRange) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + EXPECT_FALSE(ranges.IsEmpty()); + EXPECT_EQ(11, ranges.RowCount()); + EXPECT_EQ(1, ranges.GetRanges().size()); +} + +TEST_F(RowRangesTest, TestAddOverlappingRanges) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(15, 25)); // overlaps with [10, 20] + EXPECT_EQ(1, ranges.GetRanges().size()); + EXPECT_EQ(10, ranges.GetRanges()[0].from); + EXPECT_EQ(25, ranges.GetRanges()[0].to); + EXPECT_EQ(16, ranges.RowCount()); +} + +TEST_F(RowRangesTest, TestAddAdjacentRanges) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(21, 30)); // adjacent to [10, 20] + EXPECT_EQ(1, ranges.GetRanges().size()); + EXPECT_EQ(10, ranges.GetRanges()[0].from); + EXPECT_EQ(30, ranges.GetRanges()[0].to); + EXPECT_EQ(21, ranges.RowCount()); +} + +TEST_F(RowRangesTest, TestAddNonOverlappingRanges) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(30, 40)); + EXPECT_EQ(2, ranges.GetRanges().size()); + EXPECT_EQ(10, ranges.GetRanges()[0].from); + EXPECT_EQ(20, ranges.GetRanges()[0].to); + EXPECT_EQ(30, ranges.GetRanges()[1].from); + EXPECT_EQ(40, ranges.GetRanges()[1].to); + EXPECT_EQ(22, ranges.RowCount()); +} + +TEST_F(RowRangesTest, TestUnion) { + RowRanges left; + left.Add(RowRanges::Range(10, 20)); + left.Add(RowRanges::Range(40, 50)); + + RowRanges right; + right.Add(RowRanges::Range(15, 25)); + right.Add(RowRanges::Range(60, 70)); + + RowRanges result = RowRanges::Union(left, right); + EXPECT_EQ(3, result.GetRanges().size()); + EXPECT_EQ(10, result.GetRanges()[0].from); + EXPECT_EQ(25, result.GetRanges()[0].to); + EXPECT_EQ(40, result.GetRanges()[1].from); + EXPECT_EQ(50, result.GetRanges()[1].to); + EXPECT_EQ(60, result.GetRanges()[2].from); + EXPECT_EQ(70, result.GetRanges()[2].to); +} + +TEST_F(RowRangesTest, TestUnionWithOverlap) { + RowRanges left; + left.Add(RowRanges::Range(10, 30)); + + RowRanges right; + right.Add(RowRanges::Range(20, 40)); + + RowRanges result = RowRanges::Union(left, right); + EXPECT_EQ(1, result.GetRanges().size()); + EXPECT_EQ(10, result.GetRanges()[0].from); + EXPECT_EQ(40, result.GetRanges()[0].to); +} + +TEST_F(RowRangesTest, TestIntersection) { + RowRanges left; + left.Add(RowRanges::Range(10, 30)); + left.Add(RowRanges::Range(50, 70)); + + RowRanges right; + right.Add(RowRanges::Range(20, 40)); + right.Add(RowRanges::Range(60, 80)); + + RowRanges result = RowRanges::Intersection(left, right); + EXPECT_EQ(2, result.GetRanges().size()); + EXPECT_EQ(20, result.GetRanges()[0].from); + EXPECT_EQ(30, result.GetRanges()[0].to); + EXPECT_EQ(60, result.GetRanges()[1].from); + EXPECT_EQ(70, result.GetRanges()[1].to); +} + +TEST_F(RowRangesTest, TestIntersectionNoOverlap) { + RowRanges left; + left.Add(RowRanges::Range(10, 20)); + + RowRanges right; + right.Add(RowRanges::Range(30, 40)); + + RowRanges result = RowRanges::Intersection(left, right); + EXPECT_TRUE(result.IsEmpty()); +} + +TEST_F(RowRangesTest, TestIntersectionEmptyLeft) { + RowRanges left = RowRanges::CreateEmpty(); + + RowRanges right; + right.Add(RowRanges::Range(10, 20)); + + RowRanges result = RowRanges::Intersection(left, right); + EXPECT_TRUE(result.IsEmpty()); +} + +TEST_F(RowRangesTest, TestIsOverlapping) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(30, 40)); + + EXPECT_TRUE(ranges.IsOverlapping(10, 20)); + EXPECT_TRUE(ranges.IsOverlapping(15, 25)); + EXPECT_TRUE(ranges.IsOverlapping(30, 40)); + EXPECT_FALSE(ranges.IsOverlapping(21, 29)); + EXPECT_FALSE(ranges.IsOverlapping(5, 9)); + EXPECT_FALSE(ranges.IsOverlapping(41, 50)); +} + +TEST_F(RowRangesTest, TestRowCount) { + RowRanges ranges; + ranges.Add(RowRanges::Range(0, 9)); + ranges.Add(RowRanges::Range(20, 29)); + EXPECT_EQ(20, ranges.RowCount()); + + ranges.Add(RowRanges::Range(10, 19)); // Fill the gap + EXPECT_EQ(30, ranges.RowCount()); +} + +TEST_F(RowRangesTest, TestToString) { + RowRanges ranges; + ranges.Add(RowRanges::Range(10, 20)); + ranges.Add(RowRanges::Range(30, 40)); + EXPECT_EQ("[[10, 20], [30, 40]]", ranges.ToString()); +} + +TEST_F(RowRangesTest, TestRangeOperations) { + RowRanges::Range r1(10, 20); + RowRanges::Range r2(30, 40); + RowRanges::Range r3(15, 25); + + // r1 lies entirely before r2; r3 overlaps r1. + EXPECT_TRUE(r1.to < r2.from); + EXPECT_FALSE(r1.from > r2.to); + EXPECT_FALSE(r1.to < r3.from); + EXPECT_FALSE(r1.from > r3.to); + EXPECT_EQ(11, r1.Count()); +} + +// ===================================================================== +// ColumnIndexFilter integration tests +// ===================================================================== + +/// Test fixture that creates real Parquet files with page index for testing +/// ColumnIndexFilter::CalculateRowRanges end-to-end. +/// +/// Data layout: 100 rows, 10 pages of 10 rows each. +/// Page 0: val [0, 9] +/// Page 1: val [10, 19] +/// ... +/// Page 9: val [90, 99] +class ColumnIndexFilterTest : public ::testing::Test { + protected: + void SetUp() override { + pool_ = GetDefaultPool(); + arrow_pool_ = GetArrowPool(pool_); + dir_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + fs_ = dir_->GetFileSystem(); + + // Write the test file once for all tests + file_name_ = dir_->Str() + "/col_index_filter.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name_, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + // Open as raw ParquetFileReader + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name_)); + ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + parquet_reader_ = ::parquet::ParquetFileReader::Open(in_stream); + ASSERT_TRUE(parquet_reader_); + + page_index_reader_ = parquet_reader_->GetPageIndexReader(); + ASSERT_TRUE(page_index_reader_); + + column_name_to_index_["val"] = 0; + row_group_row_count_ = parquet_reader_->metadata()->RowGroup(0)->num_rows(); + } + + static std::shared_ptr MakeSequentialIntData(int32_t num_rows) { + arrow::Int32Builder builder; + EXPECT_TRUE(builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(i); + } + auto array = builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::int32()); + return arrow::StructArray::Make({array}, {field}).ValueOrDie(); + } + + void WriteTestFile(const std::string& file_name, + const std::shared_ptr& struct_array, + int32_t write_batch_size, int64_t max_row_group_length) { + auto data_type = struct_array->struct_type(); + auto data_schema = arrow::schema(data_type->fields()); + auto data_arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*struct_array, data_arrow_array.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_name, /*overwrite=*/false)); + ::parquet::WriterProperties::Builder wp_builder; + wp_builder.write_batch_size(write_batch_size); + wp_builder.max_row_group_length(max_row_group_length); + wp_builder.disable_dictionary(); + wp_builder.enable_write_page_index(); + wp_builder.data_pagesize(1); + auto writer_properties = wp_builder.build(); + ASSERT_OK_AND_ASSIGN( + auto format_writer, + ParquetFormatWriter::Create(out, data_schema, writer_properties, + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_)); + ASSERT_OK(format_writer->AddBatch(data_arrow_array.get())); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Close()); + } + + Result Filter(const std::shared_ptr& predicate) { + return ColumnIndexFilter::CalculateRowRanges(predicate, page_index_reader_, + column_name_to_index_, /*row_group_index=*/0, + row_group_row_count_); + } + + std::shared_ptr arrow_pool_; + std::shared_ptr pool_; + std::shared_ptr fs_; + std::unique_ptr dir_; + std::string file_name_; + std::unique_ptr<::parquet::ParquetFileReader> parquet_reader_; + std::shared_ptr<::parquet::PageIndexReader> page_index_reader_; + std::map column_name_to_index_; + int64_t row_group_row_count_ = 0; +}; + +/// EQUAL: val = 55 → should match only page 5 (rows [50,59]) +TEST_F(ColumnIndexFilterTest, EqualMatchSinglePage) { + auto pred = + PredicateBuilder::Equal(0, "val", FieldType::INT, Literal(static_cast(55))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + // Page 5 covers rows [50, 59] + EXPECT_EQ(10, ranges.RowCount()); + EXPECT_EQ(50, ranges.GetRanges()[0].from); + EXPECT_EQ(59, ranges.GetRanges()[0].to); +} + +/// EQUAL: val = 0 → should match page 0 (rows [0,9]) +TEST_F(ColumnIndexFilterTest, EqualMatchFirstPage) { + auto pred = PredicateBuilder::Equal(0, "val", FieldType::INT, Literal(static_cast(0))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + EXPECT_EQ(10, ranges.RowCount()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(9, ranges.GetRanges()[0].to); +} + +/// EQUAL: val = 999 → should match no pages (value out of range) +TEST_F(ColumnIndexFilterTest, EqualNoMatch) { + auto pred = + PredicateBuilder::Equal(0, "val", FieldType::INT, Literal(static_cast(999))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_TRUE(ranges.IsEmpty()); +} + +/// LESS_THAN: val < 25 → should match pages 0,1,2 (rows [0,29]) +/// Page 0: [0,9], Page 1: [10,19], Page 2: [20,29] — page 2 has min=20 < 25 +TEST_F(ColumnIndexFilterTest, LessThanMatchMultiplePages) { + auto pred = + PredicateBuilder::LessThan(0, "val", FieldType::INT, Literal(static_cast(25))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + // Pages 0-2 match (min < 25) + EXPECT_EQ(30, ranges.RowCount()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(29, ranges.GetRanges()[0].to); +} + +/// LESS_THAN: val < 0 → no pages match (min of page 0 is 0, which is not < 0) +TEST_F(ColumnIndexFilterTest, LessThanNoMatch) { + auto pred = + PredicateBuilder::LessThan(0, "val", FieldType::INT, Literal(static_cast(0))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_TRUE(ranges.IsEmpty()); +} + +/// GREATER_THAN: val > 85 → should match pages 8,9 +/// Page 8: max=89 > 85, Page 9: max=99 > 85 +TEST_F(ColumnIndexFilterTest, GreaterThanMatchLastPages) { + auto pred = + PredicateBuilder::GreaterThan(0, "val", FieldType::INT, Literal(static_cast(85))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + EXPECT_EQ(20, ranges.RowCount()); + EXPECT_EQ(80, ranges.GetRanges()[0].from); + EXPECT_EQ(99, ranges.GetRanges()[0].to); +} + +/// GREATER_THAN: val > 99 → no pages match +TEST_F(ColumnIndexFilterTest, GreaterThanNoMatch) { + auto pred = + PredicateBuilder::GreaterThan(0, "val", FieldType::INT, Literal(static_cast(99))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_TRUE(ranges.IsEmpty()); +} + +/// LESS_OR_EQUAL: val <= 9 → page 0 only (max=9 <= 9, but page 1 min=10 > 9) +TEST_F(ColumnIndexFilterTest, LessOrEqualBoundary) { + auto pred = + PredicateBuilder::LessOrEqual(0, "val", FieldType::INT, Literal(static_cast(9))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(10, ranges.RowCount()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(9, ranges.GetRanges()[0].to); +} + +/// GREATER_OR_EQUAL: val >= 90 → page 9 only +TEST_F(ColumnIndexFilterTest, GreaterOrEqualBoundary) { + auto pred = PredicateBuilder::GreaterOrEqual(0, "val", FieldType::INT, + Literal(static_cast(90))); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(10, ranges.RowCount()); + EXPECT_EQ(90, ranges.GetRanges()[0].from); + EXPECT_EQ(99, ranges.GetRanges()[0].to); +} + +/// IN: val IN (5, 55, 95) → pages 0, 5, 9 +TEST_F(ColumnIndexFilterTest, InMatchMultiplePages) { + auto pred = + PredicateBuilder::In(0, "val", FieldType::INT, + {Literal(static_cast(5)), Literal(static_cast(55)), + Literal(static_cast(95))}); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_FALSE(ranges.IsEmpty()); + // Pages 0, 5, 9 + EXPECT_EQ(3, ranges.GetRanges().size()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(9, ranges.GetRanges()[0].to); + EXPECT_EQ(50, ranges.GetRanges()[1].from); + EXPECT_EQ(59, ranges.GetRanges()[1].to); + EXPECT_EQ(90, ranges.GetRanges()[2].from); + EXPECT_EQ(99, ranges.GetRanges()[2].to); +} + +/// IN: val IN (999) → no match +TEST_F(ColumnIndexFilterTest, InNoMatch) { + auto pred = + PredicateBuilder::In(0, "val", FieldType::INT, {Literal(static_cast(999))}); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_TRUE(ranges.IsEmpty()); +} + +/// IS_NOT_NULL on non-nullable column → all pages match +TEST_F(ColumnIndexFilterTest, IsNotNullAllPages) { + auto pred = PredicateBuilder::IsNotNull(0, "val", FieldType::INT); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(row_group_row_count_, ranges.RowCount()); +} + +/// AND: val >= 30 AND val < 50 → pages 3, 4 +TEST_F(ColumnIndexFilterTest, AndCompound) { + auto ge = PredicateBuilder::GreaterOrEqual(0, "val", FieldType::INT, + Literal(static_cast(30))); + auto lt = + PredicateBuilder::LessThan(0, "val", FieldType::INT, Literal(static_cast(50))); + ASSERT_OK_AND_ASSIGN(auto pred, PredicateBuilder::And({ge, lt})); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(20, ranges.RowCount()); + EXPECT_EQ(30, ranges.GetRanges()[0].from); + EXPECT_EQ(49, ranges.GetRanges()[0].to); +} + +/// OR: val < 10 OR val >= 90 → pages 0, 9 +TEST_F(ColumnIndexFilterTest, OrCompound) { + auto lt = + PredicateBuilder::LessThan(0, "val", FieldType::INT, Literal(static_cast(10))); + auto ge = PredicateBuilder::GreaterOrEqual(0, "val", FieldType::INT, + Literal(static_cast(90))); + ASSERT_OK_AND_ASSIGN(auto pred, PredicateBuilder::Or({lt, ge})); + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(pred)); + EXPECT_EQ(2, ranges.GetRanges().size()); + EXPECT_EQ(0, ranges.GetRanges()[0].from); + EXPECT_EQ(9, ranges.GetRanges()[0].to); + EXPECT_EQ(90, ranges.GetRanges()[1].from); + EXPECT_EQ(99, ranges.GetRanges()[1].to); +} + +/// Predicates referencing fields absent from the data file are stripped upstream +/// by FieldMappingBuilder, so reaching ColumnIndexFilter with such a predicate is +/// a contract violation and surfaces as an error. +TEST_F(ColumnIndexFilterTest, UnknownColumnReturnsError) { + auto pred = PredicateBuilder::Equal(0, "nonexistent", FieldType::INT, + Literal(static_cast(42))); + EXPECT_FALSE(Filter(pred).ok()); +} + +/// Null predicate → all rows +TEST_F(ColumnIndexFilterTest, NullPredicateReturnsAllRows) { + ASSERT_OK_AND_ASSIGN(auto ranges, Filter(nullptr)); + EXPECT_EQ(row_group_row_count_, ranges.RowCount()); +} + +} // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 674e3c62..3e019598 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -18,114 +18,326 @@ #include "paimon/format/parquet/file_reader_wrapper.h" +#include #include #include +#include "arrow/io/interfaces.h" #include "arrow/record_batch.h" #include "arrow/util/range.h" #include "fmt/format.h" +#include "paimon/format/parquet/column_index_filter.h" +#include "paimon/format/parquet/page_filtered_row_group_reader.h" #include "paimon/macros.h" #include "parquet/arrow/reader.h" #include "parquet/file_reader.h" #include "parquet/metadata.h" +#include "parquet/page_index.h" + +// Convert any std::exception thrown by underlying Parquet/Arrow APIs into a +// Status. Used as the trailing catch clauses of a try block in every public +// method that calls into the parquet C++ API, so the read layer never throws. +#define PAIMON_PARQUET_CATCH_AND_RETURN_STATUS(context) \ + catch (const std::exception& e) { \ + return Status::Invalid(fmt::format("{}: {}", (context), e.what())); \ + } \ + catch (...) { \ + return Status::UnknownError((context), ": unknown error"); \ + } namespace paimon::parquet { +namespace { + +// Merge overlapping or adjacent ReadRanges into a minimal set of non-overlapping ranges. +// PreBufferRanges requires non-overlapping ranges, so this is necessary when combining +// ranges from multiple sources (page-level ranges, column chunk ranges, etc.). +std::vector<::arrow::io::ReadRange> MergeOverlappingRanges( + std::vector<::arrow::io::ReadRange> ranges) { + if (ranges.empty()) { + return ranges; + } + + // Sort by offset + std::sort(ranges.begin(), ranges.end(), + [](const ::arrow::io::ReadRange& a, const ::arrow::io::ReadRange& b) { + return a.offset < b.offset; + }); + + std::vector<::arrow::io::ReadRange> merged; + merged.push_back(ranges[0]); + + for (size_t i = 1; i < ranges.size(); ++i) { + auto& last = merged.back(); + const auto& curr = ranges[i]; + // Check if current range overlaps or is adjacent to the last merged range + int64_t last_end = last.offset + last.length; + if (curr.offset <= last_end) { + // Merge: extend the last range if current extends beyond it + int64_t curr_end = curr.offset + curr.length; + if (curr_end > last_end) { + last.length = curr_end - last.offset; + } + } else { + // No overlap, add as new range + merged.push_back(curr); + } + } + + return merged; +} + +} // namespace + Result> FileReaderWrapper::Create( - std::unique_ptr<::parquet::arrow::FileReader>&& file_reader) { - if (file_reader == nullptr) { - return Status::Invalid("file reader wrapper create failed. file reader is nullptr"); - } - std::vector> all_row_group_ranges; - auto meta_data = file_reader->parquet_reader()->metadata(); - // prepare [start_row_idx, end_row_idx) for all row groups - uint64_t start_row_idx = 0; - for (int32_t i = 0; i < meta_data->num_row_groups(); i++) { - uint64_t end_row_idx = start_row_idx + meta_data->RowGroup(i)->num_rows(); - all_row_group_ranges.emplace_back(start_row_idx, end_row_idx); - start_row_idx = end_row_idx; - } - uint64_t num_rows = file_reader->parquet_reader()->metadata()->num_rows(); - if (start_row_idx != num_rows) { - assert(false); - return Status::Invalid( - fmt::format("unexpected error. row group ranges not match with num rows {}", num_rows)); - } - std::vector row_groups_indices = arrow::internal::Iota(file_reader->num_row_groups()); - std::vector columns_indices = - arrow::internal::Iota(file_reader->parquet_reader()->metadata()->num_columns()); - auto file_reader_wrapper = std::unique_ptr( - new FileReaderWrapper(std::move(file_reader), all_row_group_ranges, num_rows)); - PAIMON_RETURN_NOT_OK(file_reader_wrapper->PrepareForReadingLazy( - std::set(row_groups_indices.begin(), row_groups_indices.end()), columns_indices)); - return file_reader_wrapper; + std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, ::arrow::MemoryPool* pool, + int64_t batch_size) { + try { + if (file_reader == nullptr) { + return Status::Invalid("file reader wrapper create failed. file reader is nullptr"); + } + std::vector> all_row_group_ranges; + auto meta_data = file_reader->parquet_reader()->metadata(); + // prepare [start_row_idx, end_row_idx) for all row groups + uint64_t start_row_idx = 0; + for (int32_t i = 0; i < meta_data->num_row_groups(); i++) { + uint64_t end_row_idx = start_row_idx + meta_data->RowGroup(i)->num_rows(); + all_row_group_ranges.emplace_back(start_row_idx, end_row_idx); + start_row_idx = end_row_idx; + } + uint64_t num_rows = file_reader->parquet_reader()->metadata()->num_rows(); + if (start_row_idx != num_rows) { + assert(false); + return Status::Invalid(fmt::format( + "unexpected error. row group ranges not match with num rows {}", num_rows)); + } + std::vector row_groups_indices = + arrow::internal::Iota(file_reader->num_row_groups()); + std::vector columns_indices = + arrow::internal::Iota(file_reader->parquet_reader()->metadata()->num_columns()); + auto file_reader_wrapper = std::unique_ptr(new FileReaderWrapper( + std::move(file_reader), all_row_group_ranges, num_rows, pool, batch_size)); + PAIMON_RETURN_NOT_OK(file_reader_wrapper->PrepareForReadingLazy( + std::set(row_groups_indices.begin(), row_groups_indices.end()), + columns_indices)); + return file_reader_wrapper; + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::Create") +} + +FileReaderWrapper::~FileReaderWrapper() { + WaitForPendingPreBuffer(); +} + +Result> FileReaderWrapper::GetSchema() const { + try { + std::shared_ptr file_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetSchema(&file_schema)); + return file_schema; + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::GetSchema") +} + +Status FileReaderWrapper::Close() { + try { + if (batch_reader_) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(batch_reader_->Close()); + } + return Status::OK(); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::Close") } FileReaderWrapper::FileReaderWrapper( std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, - const std::vector>& all_row_group_ranges, uint64_t num_rows) + const std::vector>& all_row_group_ranges, uint64_t num_rows, + ::arrow::MemoryPool* pool, int64_t batch_size) : file_reader_(std::move(file_reader)), all_row_group_ranges_(all_row_group_ranges), + pool_(pool), + batch_size_(batch_size), num_rows_(num_rows) {} +void FileReaderWrapper::WaitForPendingPreBuffer() { + if (!prebuffered_ranges_.empty() && file_reader_) { + // Wait for all outstanding PreBuffer async reads to complete before destruction. + // Without this, JindoSDK async pread callbacks may fire after the underlying + // buffers and memory pool are freed, causing use-after-free crashes. + auto status = + file_reader_->parquet_reader()->WhenBufferedRanges(prebuffered_ranges_).status(); + (void)status; // Best-effort; ignore errors during cleanup + prebuffered_ranges_.clear(); + } +} + Status FileReaderWrapper::SeekToRow(uint64_t row_number) { - for (uint64_t i = 0; i < target_row_groups_.size(); i++) { - if (row_number > target_row_groups_[i].first && row_number < target_row_groups_[i].second) { - return Status::Invalid(fmt::format( - "seek to row failed. row number {} should not be in the middle of readable range", - row_number)); - } - if (target_row_groups_[i].first >= row_number) { - current_row_group_idx_ = i; - next_row_to_read_ = target_row_groups_[i].first; - std::vector target_row_group_indices; - for (uint64_t j = i; j < target_row_groups_.size(); j++) { - PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, GetRowGroupId(target_row_groups_[j])); - target_row_group_indices.push_back(row_group_id); + try { + // Reset any in-progress page-filtered streaming + current_page_filtered_reader_.reset(); + filtered_global_offset_ = 0; + + for (uint64_t i = 0; i < target_row_groups_.size(); i++) { + if (row_number > target_row_groups_[i].first && + row_number < target_row_groups_[i].second) { + return Status::Invalid( + fmt::format("seek to row failed. row number {} should not be in the middle of " + "readable range", + row_number)); + } + if (target_row_groups_[i].first >= row_number) { + current_row_group_idx_ = i; + next_row_to_read_ = target_row_groups_[i].first; + + // Rebuild batch_reader_ only for non-page-filtered row groups at/after seek + // position. Page-filtered RGs need no seek-side bookkeeping: their per-RG + // reader is constructed on demand in Next() from row_group_row_ranges_ each + // time, so backward seek "just works". + std::vector target_row_group_indices; + for (uint64_t j = i; j < target_row_groups_.size(); j++) { + if (page_filtered_indices_.count(j) == 0) { + PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, + GetRowGroupId(target_row_groups_[j])); + target_row_group_indices.push_back(row_group_id); + } + } + if (!target_row_group_indices.empty()) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( + target_row_group_indices, target_column_indices_, &batch_reader_)); + } else { + batch_reader_.reset(); + } + return Status::OK(); } - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( - target_row_group_indices, target_column_indices_, &batch_reader_)); - return Status::OK(); } + next_row_to_read_ = num_rows_; + current_row_group_idx_ = target_row_groups_.size(); + return Status::OK(); } - next_row_to_read_ = num_rows_; - current_row_group_idx_ = target_row_groups_.size(); - return Status::OK(); + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::SeekToRow") } Result> FileReaderWrapper::Next() { - if (PAIMON_UNLIKELY(!reader_initialized_)) { - PAIMON_RETURN_NOT_OK(PrepareForReading(target_row_group_indices_, target_column_indices_)); - } - std::shared_ptr record_batch; - if (current_row_group_idx_ < target_row_groups_.size()) { - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(record_batch, batch_reader_->Next()); - } - if (record_batch) { - int64_t num_rows = record_batch->num_rows(); - previous_first_row_ = next_row_to_read_; - if (next_row_to_read_ + num_rows < target_row_groups_[current_row_group_idx_].second) { - next_row_to_read_ += num_rows; - } else if (next_row_to_read_ + num_rows == - target_row_groups_[current_row_group_idx_].second) { - if (current_row_group_idx_ == target_row_groups_.size() - 1) { - // current row group is the last. - next_row_to_read_ = num_rows_; + try { + if (PAIMON_UNLIKELY(!reader_initialized_)) { + PAIMON_RETURN_NOT_OK( + PrepareForReading(target_row_group_indices_, target_column_indices_)); + } + + // Loop until we produce a batch or exhaust all row groups. A null from the active + // per-RG reader means that RG is done; we advance and try the next RG without + // surfacing a spurious null to the caller. + while (current_row_group_idx_ < target_row_groups_.size()) { + std::shared_ptr record_batch; + bool is_page_filtered = page_filtered_indices_.count(current_row_group_idx_) > 0; + + if (is_page_filtered) { + // Construct the per-RG streaming reader on demand. Inputs are recomputed each + // time from existing wrapper fields (no per-RG meta cached on the wrapper), + // mirroring how the fully-matched path delegates to Arrow's stateless + // GetRecordBatchReader. This makes both forward and backward seeks work + // uniformly: SeekToRow only resets current_page_filtered_reader_, and the + // next Next() rebuilds from authoritative state. + if (!current_page_filtered_reader_) { + PAIMON_ASSIGN_OR_RAISE( + int32_t rg_index, + GetRowGroupId(target_row_groups_[current_row_group_idx_])); + auto range_it = row_group_row_ranges_.find(rg_index); + if (range_it == row_group_row_ranges_.end()) { + return Status::Invalid( + fmt::format("page-filtered row group {} missing row ranges in " + "row_group_row_ranges_", + rg_index)); + } + const RowRanges& row_ranges = range_it->second; + auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( + file_reader_->parquet_reader(), rg_index, row_ranges, + target_column_indices_); + bool pre_buffered = !prebuffered_ranges_.empty(); + // batch_size_ == 0 means "no per-batch row cap" in the wrapper's contract, + // but TableBatchReader::set_chunksize(0) would loop forever emitting empty + // batches. Translate to int64_max so the reader produces one batch per + // underlying chunk boundary instead. + int64_t max_chunksize = + batch_size_ > 0 ? batch_size_ : std::numeric_limits::max(); + PAIMON_ASSIGN_OR_RAISE(current_page_filtered_reader_, + PageFilteredRowGroupReader::ReadFilteredRowGroup( + file_reader_->parquet_reader(), rg_index, row_ranges, + target_column_indices_, page_filtered_read_schema_, + pool_, file_reader_->properties().cache_options(), + pre_buffered, page_ranges, max_chunksize)); + current_filtered_row_ranges_ = row_ranges; + current_filtered_rg_start_ = target_row_groups_[current_row_group_idx_].first; + filtered_global_offset_ = 0; + } + PAIMON_RETURN_NOT_OK_FROM_ARROW( + current_page_filtered_reader_->ReadNext(&record_batch)); + } else if (batch_reader_) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(record_batch, batch_reader_->Next()); + } + + if (record_batch) { + int64_t num_rows = record_batch->num_rows(); + if (is_page_filtered) { + // Map the cumulative filtered-row offset back to the original row index + // within this row group. Must be evaluated BEFORE incrementing the offset. + auto original_row = current_filtered_row_ranges_.MapFilteredIndexToOriginalRow( + filtered_global_offset_); + previous_first_row_ = + original_row.has_value() + ? current_filtered_rg_start_ + static_cast(*original_row) + : current_filtered_rg_start_; + filtered_global_offset_ += num_rows; + // Stay on this RG; the next ReadNext will either return more data or null. + } else { + previous_first_row_ = next_row_to_read_; + if (next_row_to_read_ + num_rows < + target_row_groups_[current_row_group_idx_].second) { + next_row_to_read_ += num_rows; + } else if (next_row_to_read_ + num_rows == + target_row_groups_[current_row_group_idx_].second) { + if (current_row_group_idx_ == target_row_groups_.size() - 1) { + next_row_to_read_ = num_rows_; + } else { + current_row_group_idx_++; + next_row_to_read_ = target_row_groups_[current_row_group_idx_].first; + } + } else { + return Status::Invalid(fmt::format( + "Next failed. Unexpected error, next row to read {} + num rows just " + "read {} should always be within current row group range or exactly " + "equals to current row group end {}", + next_row_to_read_, num_rows, + target_row_groups_[current_row_group_idx_].second)); + } + } + return record_batch; + } + + // Null batch: current row group is exhausted (or fully-matched RGs hit a degenerate + // EOF). Advance to the next row group and continue the loop. + if (is_page_filtered) { + current_page_filtered_reader_.reset(); + filtered_global_offset_ = 0; + if (current_row_group_idx_ == target_row_groups_.size() - 1) { + next_row_to_read_ = num_rows_; + current_row_group_idx_ = target_row_groups_.size(); + } else { + current_row_group_idx_++; + next_row_to_read_ = target_row_groups_[current_row_group_idx_].first; + } } else { - current_row_group_idx_++; - next_row_to_read_ = target_row_groups_[current_row_group_idx_].first; + // Fully-matched path: batch_reader_ is exhausted with no more RBs to align on + // row counts. Stop here — remaining RGs (if any) should be page-filtered and + // will be handled by re-entering the loop, but if we got here without advancing + // first, treat as terminal to avoid an infinite loop. + break; } - } else { - return Status::Invalid(fmt::format( - "Next failed. Unexpected error, next row to read {} + num rows just read {} " - "should always be within current row group range or exactly equals to current " - "row group end {}", - next_row_to_read_, num_rows, target_row_groups_[current_row_group_idx_].second)); } - } else { + previous_first_row_ = next_row_to_read_; + return std::shared_ptr(); // EOF } - return record_batch; + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::Next") } Result>> FileReaderWrapper::GetRowGroupRanges( @@ -151,24 +363,146 @@ Status FileReaderWrapper::PrepareForReadingLazy(const std::set& target_ Status FileReaderWrapper::PrepareForReading(const std::set& target_row_group_indices, const std::vector& column_indices) { - std::vector> target_row_groups; - PAIMON_ASSIGN_OR_RAISE(target_row_groups, GetRowGroupRanges(target_row_group_indices)); - std::unique_ptr batch_reader; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( - std::vector(target_row_group_indices.begin(), target_row_group_indices.end()), - column_indices, &batch_reader)); - target_row_groups_ = target_row_groups; - target_column_indices_ = column_indices; - batch_reader_ = std::move(batch_reader); - if (target_row_groups_.empty()) { - next_row_to_read_ = num_rows_; - } else { - next_row_to_read_ = target_row_groups_[0].first; + try { + std::vector> target_row_groups; + PAIMON_ASSIGN_OR_RAISE(target_row_groups, GetRowGroupRanges(target_row_group_indices)); + + // Build position map: rg_index -> position in target_row_groups (O(1) lookup) + std::map rg_idx_to_position; + { + uint64_t pos = 0; + for (int32_t rg_idx : target_row_group_indices) { + rg_idx_to_position[rg_idx] = pos++; + } + } + + // Separate row groups into fully matched (Arrow's standard reader) and partially + // matched (page-filtered, per-RG reader constructed on demand in Next()). + // Per-RG metadata for the page-filtered path is NOT cached on the wrapper — it's + // recomputed on demand in Next() from row_group_row_ranges_ + target_column_indices_, + // mirroring how the fully-matched path lets Arrow's FileReader own all metadata. + std::vector fully_matched_row_groups; + page_filtered_indices_.clear(); + page_filtered_read_schema_.reset(); + + // Page-level byte ranges collected here only for the bulk PreBuffer call below; + // discarded once PreBuffer is dispatched. + std::vector<::arrow::io::ReadRange> page_filtered_byte_ranges; + + for (int32_t rg_idx : target_row_group_indices) { + auto range_it = row_group_row_ranges_.find(rg_idx); + if (range_it != row_group_row_ranges_.end()) { + uint64_t pos = rg_idx_to_position[rg_idx]; + page_filtered_indices_.insert(pos); + + // Build the page-filter read_schema once on first encounter — it's identical + // across all page-filtered RGs in this session. + if (!page_filtered_read_schema_) { + std::shared_ptr schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetSchema(&schema)); + std::vector> fields; + auto parquet_schema = file_reader_->parquet_reader()->metadata()->schema(); + for (int32_t col_idx : column_indices) { + const std::string& col_name = parquet_schema->Column(col_idx)->name(); + auto field = schema->GetFieldByName(col_name); + if (!field) { + return Status::Invalid(fmt::format( + "PrepareForReading: Parquet column {} ('{}') has no matching Arrow " + "field in file schema", + col_idx, col_name)); + } + fields.push_back(field); + } + page_filtered_read_schema_ = arrow::schema(fields); + } + + auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( + file_reader_->parquet_reader(), rg_idx, range_it->second, column_indices); + page_filtered_byte_ranges.insert(page_filtered_byte_ranges.end(), + std::make_move_iterator(page_ranges.begin()), + std::make_move_iterator(page_ranges.end())); + } else { + fully_matched_row_groups.push_back(rg_idx); + } + } + + // Wait for any previously pre-buffered data before starting new pre-buffer. + WaitForPendingPreBuffer(); + + // Create standard reader for fully matched row groups FIRST. + // GetRecordBatchReader internally calls PreBuffer, but we'll override it below + // with a single PreBuffer covering ALL row groups (page-filtered + fully-matched) + // so that async I/O for all files starts in parallel. + std::unique_ptr batch_reader; + if (!fully_matched_row_groups.empty()) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( + fully_matched_row_groups, column_indices, &batch_reader)); + } + + // Collect all byte ranges for a single PreBufferRanges call. + // Page-filtered RGs: only matching page ranges (from ComputePageRanges). + // Fully-matched RGs: entire column chunk ranges. + // + // When there are no page-filtered RGs, skip the manual PreBufferRanges entirely: + // GetRecordBatchReader has already issued PreBuffer internally (driven by + // ArrowReaderProperties::pre_buffer=true), and a second PreBufferRanges call here + // would tear down and rebuild cached_source_, redundantly re-issuing the same IO + // on remote filesystems. The manual path is only needed to merge page-level ranges + // with column-chunk ranges into a single PreBuffer covering both kinds of RGs. + if (!page_filtered_indices_.empty()) { + std::vector<::arrow::io::ReadRange> all_ranges = std::move(page_filtered_byte_ranges); + + // Fully-matched row groups: add entire column chunk ranges + // The correct calculation follows Arrow's ColumnChunkMetaData::file_range(): + // - col_start = data_page_offset (or dictionary_page_offset if present and lower) + // - col_length = total_compressed_size (includes all pages: dictionary + data) + auto file_metadata = file_reader_->parquet_reader()->metadata(); + for (int32_t rg_idx : fully_matched_row_groups) { + auto rg_metadata = file_metadata->RowGroup(rg_idx); + for (int32_t col_idx : column_indices) { + auto col_chunk = rg_metadata->ColumnChunk(col_idx); + int64_t offset = col_chunk->data_page_offset(); + if (col_chunk->has_dictionary_page() && + col_chunk->dictionary_page_offset() > 0 && + offset > col_chunk->dictionary_page_offset()) { + offset = col_chunk->dictionary_page_offset(); + } + int64_t size = col_chunk->total_compressed_size(); + all_ranges.push_back({offset, size}); + } + } + + const auto& cache_opts = file_reader_->properties().cache_options(); + ::arrow::io::IOContext io_ctx(pool_); + // Merge overlapping ranges before calling PreBufferRanges, which rejects overlapping + // ranges. + auto merged_ranges = MergeOverlappingRanges(std::move(all_ranges)); + // PreBuffer is an optimization - if it fails (e.g., IO error during testing), + // continue without pre-buffering. Subsequent reads will fetch data on-demand. + try { + file_reader_->parquet_reader()->PreBufferRanges(merged_ranges, io_ctx, cache_opts); + // Track for cleanup on destruction + prebuffered_ranges_ = std::move(merged_ranges); + } catch (const std::exception& e) { + // Pre-buffering failed, clear ranges to indicate no pre-buffered data available. + // Reading will fall back to on-demand I/O. + prebuffered_ranges_.clear(); + } + } + target_row_groups_ = target_row_groups; + target_column_indices_ = column_indices; + batch_reader_ = std::move(batch_reader); + if (target_row_groups_.empty()) { + next_row_to_read_ = num_rows_; + } else { + next_row_to_read_ = target_row_groups_[0].first; + } + previous_first_row_ = std::numeric_limits::max(); + current_row_group_idx_ = 0; + reader_initialized_ = true; + return Status::OK(); } - previous_first_row_ = std::numeric_limits::max(); - current_row_group_idx_ = 0; - reader_initialized_ = true; - return Status::OK(); + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::PrepareForReading") } Result> FileReaderWrapper::FilterRowGroupsByReadRanges( @@ -206,4 +540,35 @@ Result FileReaderWrapper::GetRowGroupId(std::pair t target_range.first, target_range.second)); } +std::shared_ptr<::parquet::PageIndexReader> FileReaderWrapper::GetPageIndexReader() { + try { + return file_reader_->parquet_reader()->GetPageIndexReader(); + } catch (...) { + // Page index is optional; degrade gracefully if the metadata read throws. + return nullptr; + } +} + +Result FileReaderWrapper::CalculateFilteredRowRanges( + int32_t row_group_index, const std::shared_ptr& predicate, + const std::map& column_name_to_index) { + try { + auto meta_data = file_reader_->parquet_reader()->metadata(); + int64_t row_count = meta_data->RowGroup(row_group_index)->num_rows(); + + if (!predicate) { + return RowRanges::CreateSingle(row_count); + } + + auto page_index_reader = GetPageIndexReader(); + if (!page_index_reader) { + return RowRanges::CreateSingle(row_count); + } + + return ColumnIndexFilter::CalculateRowRanges( + predicate, page_index_reader, column_name_to_index, row_group_index, row_count); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::CalculateFilteredRowRanges") +} + } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h index f20e94e9..3d02164e 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.h +++ b/src/paimon/format/parquet/file_reader_wrapper.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -28,84 +29,124 @@ #include "arrow/array.h" #include "arrow/compute/api.h" #include "arrow/dataset/file_parquet.h" +#include "arrow/io/caching.h" #include "arrow/record_batch.h" #include "arrow/type.h" #include "arrow/type_fwd.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/format/parquet/row_ranges.h" #include "paimon/result.h" #include "paimon/status.h" #include "parquet/arrow/reader.h" +#include "parquet/page_index.h" namespace arrow { class Schema; } // namespace arrow +namespace paimon { +class Predicate; +} // namespace paimon + namespace paimon::parquet { // The FileReaderWrapper is a decorator class designed to support seek functionality, as well as the // methods GetPreviousBatchFirstRowNumber and GetNextRowToRead. class FileReaderWrapper { public: + ~FileReaderWrapper(); + static Result> Create( - std::unique_ptr<::parquet::arrow::FileReader>&& reader); + std::unique_ptr<::parquet::arrow::FileReader>&& reader, ::arrow::MemoryPool* pool, + int64_t batch_size); + /// Seek to the specified row number. + /// @param row_number The row to seek to (must be at a row group boundary). Status SeekToRow(uint64_t row_number); + /// Read the next batch of rows. + /// @return The next RecordBatch, or nullptr if end of data. Result> Next(); + /// Get the first row number of the previously returned batch. Result GetPreviousBatchFirstRowNumber() const { return previous_first_row_; } + /// Get the row number that will be read next. uint64_t GetNextRowToRead() const { return next_row_to_read_; } + /// Get the total number of rows in the file. uint64_t GetNumberOfRows() const { return num_rows_; } + /// Get the number of row groups in the file. int32_t GetNumberOfRowGroups() const { return file_reader_->num_row_groups(); } + /// Get the underlying Parquet file reader. ::parquet::arrow::FileReader* GetFileReader() const { return file_reader_.get(); } + /// Get the [start, end) ranges for all row groups. const std::vector>& GetAllRowGroupRanges() const { return all_row_group_ranges_; } - Result> GetSchema() const { - std::shared_ptr file_schema; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetSchema(&file_schema)); - return file_schema; - } + /// Get the Arrow schema of the file. + Result> GetSchema() const; - Status Close() { - if (batch_reader_) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(batch_reader_->Close()); - } - return Status::OK(); - } + /// Close the batch reader and release resources. + Status Close(); + /// Get the [start, end) ranges for the specified row groups. + /// @param row_group_indices The row group indices to get ranges for. Result>> GetRowGroupRanges( const std::set& row_group_indices) const; + /// Prepare for lazy reading of the specified row groups and columns. + /// Actual reader initialization is deferred until the first Next() call. Status PrepareForReadingLazy(const std::set& row_group_indices, const std::vector& column_indices); + + /// Prepare for immediate reading of the specified row groups and columns. + /// Initializes the reader and starts pre-buffering I/O. Status PrepareForReading(const std::set& row_group_indices, const std::vector& column_indices); + /// Filter row groups by read ranges, returning only those that overlap. Result> FilterRowGroupsByReadRanges( const std::vector>& read_ranges, const std::vector& src_row_groups) const; + /// Set per-row-group RowRanges for page-level filtering. + /// Only partially matched row groups should have entries. + void SetRowGroupRowRanges(const std::map& ranges) { + row_group_row_ranges_ = ranges; + } + + /// Get the page index reader for the file. + /// Returns nullptr if page index is not available. + std::shared_ptr<::parquet::PageIndexReader> GetPageIndexReader(); + + /// Calculate filtered row ranges for a row group based on predicate. + /// @param row_group_index The row group index. + /// @param predicate The predicate to evaluate. + /// @param column_name_to_index Map from column name to column index. + /// @return RowRanges that may contain matching rows. + Result CalculateFilteredRowRanges( + int32_t row_group_index, const std::shared_ptr& predicate, + const std::map& column_name_to_index); + private: FileReaderWrapper(std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, const std::vector>& all_row_group_ranges, - uint64_t num_rows); + uint64_t num_rows, ::arrow::MemoryPool* pool, int64_t batch_size); Result> ReadRangesToRowGroupIds( const std::vector>& read_ranges) const; @@ -119,11 +160,41 @@ class FileReaderWrapper { std::vector> target_row_groups_; std::vector target_column_indices_; + ::arrow::MemoryPool* pool_; + int64_t batch_size_; // 0 means no limit + const uint64_t num_rows_; uint64_t next_row_to_read_ = std::numeric_limits::max(); uint64_t previous_first_row_ = std::numeric_limits::max(); uint64_t current_row_group_idx_ = 0; bool reader_initialized_ = false; + + // Streaming reader for the currently-active page-filtered row group. Created lazily + // on the first Next() call into a page-filtered RG, drained batch-by-batch, then reset + // when ReadNext returns nullptr (end of that RG). + std::unique_ptr current_page_filtered_reader_; + int64_t filtered_global_offset_ = 0; // Cumulative filtered-row offset within RG + RowRanges current_filtered_row_ranges_; // RowRanges for the active page-filtered RG + uint64_t current_filtered_rg_start_ = 0; // Absolute row-group start row number + + // Page-level filtering state. Externally injected via SetRowGroupRowRanges and + // looked up by row group index when entering a page-filtered RG. + std::map row_group_row_ranges_; + + // Set of target_row_groups_ positional indices that use page-filtered reading. + // Built in PrepareForReading from row_group_row_ranges_. + std::set page_filtered_indices_; + + // Arrow schema covering target_column_indices_, used when constructing the per-RG + // page-filtered reader. Cached in PrepareForReading because it's identical across + // all page-filtered RGs in a session. + std::shared_ptr page_filtered_read_schema_; + + // Track pre-buffered ranges so we can wait on destruction + std::vector<::arrow::io::ReadRange> prebuffered_ranges_; + + /// Wait for all pending PreBuffer operations to complete. + void WaitForPendingPreBuffer(); }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/file_reader_wrapper_test.cpp b/src/paimon/format/parquet/file_reader_wrapper_test.cpp index 12f90a1f..c8bf5add 100644 --- a/src/paimon/format/parquet/file_reader_wrapper_test.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper_test.cpp @@ -117,7 +117,8 @@ class FileReaderWrapperTest : public ::testing::Test { ASSERT_OK(format_writer->AddBatch(batch->GetData())); } - Result> PrepareReaderWrapper(const std::string& file_path) { + Result> PrepareReaderWrapper( + const std::string& file_path, int64_t wrapper_batch_size = 0) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr in, fs_->Open(file_path)); PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, in->Length()); auto input_stream = std::make_unique(in, arrow_pool_, file_length); @@ -136,10 +137,12 @@ class FileReaderWrapperTest : public ::testing::Test { PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.memory_pool(arrow_pool_.get()) ->properties(arrow_reader_props) ->Build(&file_reader)); - return FileReaderWrapper::Create(std::move(file_reader)); + return FileReaderWrapper::Create(std::move(file_reader), ::arrow::default_memory_pool(), + wrapper_batch_size); } - void PrepareParquetFile(const std::string& file_path, int32_t row_count) { + void PrepareParquetFile(const std::string& file_path, int32_t row_count, + bool enable_page_index = false, int32_t write_batch_size = 10) { auto schema_pair = PrepareArrowSchema(); const auto& arrow_schema = schema_pair.first; const auto& struct_type = schema_pair.second; @@ -147,9 +150,14 @@ class FileReaderWrapperTest : public ::testing::Test { ASSERT_OK_AND_ASSIGN(std::shared_ptr out, fs_->Create(file_path, /*overwrite=*/false)); ::parquet::WriterProperties::Builder builder; - builder.write_batch_size(10); + builder.write_batch_size(write_batch_size); builder.max_row_group_length(1000); builder.enable_store_decimal_as_integer(); + if (enable_page_index) { + builder.enable_write_page_index(); + builder.disable_dictionary(); + builder.data_pagesize(1); + } auto writer_properties = builder.build(); ASSERT_OK_AND_ASSIGN( std::shared_ptr format_writer, @@ -190,7 +198,8 @@ TEST_F(FileReaderWrapperTest, EmptyFile) { } TEST_F(FileReaderWrapperTest, NullFileReader) { - ASSERT_NOK_WITH_MSG(FileReaderWrapper::Create(nullptr), + ASSERT_NOK_WITH_MSG(FileReaderWrapper::Create(nullptr, ::arrow::default_memory_pool(), + /*batch_size=*/0), "file reader wrapper create failed. file reader is nullptr"); } @@ -240,6 +249,126 @@ TEST_F(FileReaderWrapperTest, Simple) { ASSERT_EQ(5500, reader_wrapper->GetPreviousBatchFirstRowNumber().value()); } +/// Regression: when batch_size_ is 0 (the default) and a row group is consumed via +/// the page-filtered streaming path, we must not pass 0 to TableBatchReader::set_chunksize +/// — that would make ReadNext spin forever on zero-row batches. The wrapper now +/// translates 0 to int64_max so the reader produces one batch covering all matched rows. +TEST_F(FileReaderWrapperTest, PageFilteredZeroBatchSizeDoesNotHang) { + std::string file_path = PathUtil::JoinPath(dir_->Str(), "page_zero_batch.parquet"); + PrepareParquetFile(file_path, /*row_count=*/200, /*enable_page_index=*/true); + ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); + ASSERT_EQ(1, reader_wrapper->GetNumberOfRowGroups()); + + // Inject a per-RG RowRanges to drive the page-filtered streaming path. Two non- + // contiguous ranges keep the test honest about RowRanges semantics; the actual + // numbers don't matter as long as their total falls inside the row group. + RowRanges rr({RowRanges::Range(0, 49), RowRanges::Range(100, 149)}); + reader_wrapper->SetRowGroupRowRanges({{0, rr}}); + + std::vector all_columns = {0, 1, 2}; + ASSERT_OK(reader_wrapper->PrepareForReading({0}, all_columns)); + + int64_t total = 0; + int64_t batch_count = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + if (!batch) break; + total += batch->num_rows(); + ++batch_count; + ASSERT_LT(batch_count, 1000) << "Next() did not converge — likely an infinite loop"; + } + ASSERT_EQ(100, total); + ASSERT_GE(batch_count, 1); +} + +/// SeekToRow back to a previously-consumed page-filtered row group must rebuild the +/// per-RG streaming reader from row_group_row_ranges_ and re-yield the same rows. +/// The page-filter path holds no per-RG cache that consumption could destroy; the +/// reader is constructed on demand each time, mirroring Arrow's stateless +/// GetRecordBatchReader for the fully-matched path. +TEST_F(FileReaderWrapperTest, SeekBackToConsumedPageFilteredRowGroup) { + std::string file_path = PathUtil::JoinPath(dir_->Str(), "seek_back.parquet"); + // 2000 rows produces 2 row groups (max_row_group_length=1000) with page index enabled. + PrepareParquetFile(file_path, /*row_count=*/2000, /*enable_page_index=*/true); + ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); + ASSERT_EQ(2, reader_wrapper->GetNumberOfRowGroups()); + + // Both RGs page-filtered. RowRanges are RG-local: RG0 keeps 40 rows, RG1 keeps 50. + std::map row_ranges_map; + row_ranges_map[0] = RowRanges(RowRanges::Range(10, 49)); + row_ranges_map[1] = RowRanges(RowRanges::Range(100, 149)); + reader_wrapper->SetRowGroupRowRanges(row_ranges_map); + + std::vector all_columns = {0, 1, 2}; + ASSERT_OK(reader_wrapper->PrepareForReading({0, 1}, all_columns)); + + auto count_all_rows = [&](int64_t* out_total) { + int64_t total = 0; + while (true) { + auto next = reader_wrapper->Next(); + if (!next.ok()) return next.status(); + auto batch = std::move(next).value(); + if (!batch) break; + total += batch->num_rows(); + } + *out_total = total; + return Status::OK(); + }; + + int64_t first_total = 0; + ASSERT_OK(count_all_rows(&first_total)); + ASSERT_EQ(90, first_total); // 40 + 50 + + // Seek back to row 0 (start of RG0). The on-demand reader construction means RG0 + // is read again from scratch, producing the same 90 rows total. + ASSERT_OK(reader_wrapper->SeekToRow(0)); + + int64_t second_total = 0; + ASSERT_OK(count_all_rows(&second_total)); + ASSERT_EQ(90, second_total); +} + +/// When the page-level predicate matches more rows than the wrapper's batch_size, +/// the page-filtered streaming path must split the filtered rows across multiple +/// Next() calls. Pages are written 3 rows wide (write_batch_size=3 with +/// data_pagesize=1) so that filtered rows span multiple page-sized chunks; the +/// emitted batches must (a) sum to the RowRanges row count and (b) never exceed +/// the configured batch_size — TableBatchReader additionally caps each batch at +/// the underlying chunk boundary, which is fine as long as the cap holds. +TEST_F(FileReaderWrapperTest, PageFilteredRespectsBatchSize) { + constexpr int32_t kRowCount = 60; + constexpr int32_t kPageRowCount = 3; + constexpr int64_t kExpectedTotal = 30; + + std::string file_path = PathUtil::JoinPath(dir_->Str(), "page_split.parquet"); + PrepareParquetFile(file_path, kRowCount, /*enable_page_index=*/true, + /*write_batch_size=*/kPageRowCount); + + // Keep rows [0, 29] — the first 10 pages of the row group. + RowRanges rr({RowRanges::Range(0, kExpectedTotal - 1)}); + + for (int64_t batch_size : {int64_t{1}, int64_t{2}, int64_t{3}, int64_t{5}, int64_t{10}}) { + SCOPED_TRACE("batch_size=" + std::to_string(batch_size)); + ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path, batch_size)); + reader_wrapper->SetRowGroupRowRanges({{0, rr}}); + ASSERT_OK(reader_wrapper->PrepareForReading({0}, {0, 1, 2})); + + int64_t total = 0; + int64_t batch_count = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + if (!batch) break; + ASSERT_GT(batch->num_rows(), 0); + ASSERT_LE(batch->num_rows(), batch_size); + total += batch->num_rows(); + ++batch_count; + } + ASSERT_EQ(kExpectedTotal, total); + const int64_t min_batches = (kExpectedTotal + batch_size - 1) / batch_size; + ASSERT_GE(batch_count, min_batches); + } +} + TEST_F(FileReaderWrapperTest, GetRowGroupRanges) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); PrepareParquetFile(file_path, /*row_count=*/5500); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp new file mode 100644 index 00000000..5f43c035 --- /dev/null +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/page_filtered_row_group_reader.h" + +#include + +#include "arrow/array.h" +#include "arrow/builder.h" +#include "arrow/chunked_array.h" +#include "arrow/io/caching.h" +#include "arrow/io/interfaces.h" +#include "arrow/table.h" +#include "arrow/util/future.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "parquet/arrow/reader_internal.h" +#include "parquet/metadata.h" +#include "parquet/schema.h" + +namespace paimon::parquet { + +namespace { + +/// Wraps an arrow::Table + TableBatchReader as a RecordBatchReader so the caller can +/// stream zero-copy-sliced batches without deep-copying multi-chunk columns. The Table +/// is held to keep its ChunkedArrays alive for the inner TableBatchReader. +class TableRecordBatchReader : public arrow::RecordBatchReader { + public: + TableRecordBatchReader(std::shared_ptr table, int64_t chunksize) + : table_(std::move(table)), inner_(*table_) { + inner_.set_chunksize(chunksize); + } + + std::shared_ptr schema() const override { + return table_->schema(); + } + + arrow::Status ReadNext(std::shared_ptr* out) override { + return inner_.ReadNext(out); + } + + private: + std::shared_ptr table_; + arrow::TableBatchReader inner_; +}; + +} // namespace + +std::function PageFilteredRowGroupReader::MakePageFilter( + const RowRanges& row_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, + int64_t row_group_row_count) { + // Shared counter tracks the current page index as the callback is invoked + // in order for each data page. + auto page_counter = std::make_shared(0); + + const auto& page_locations = offset_index->page_locations(); + auto num_pages = static_cast(page_locations.size()); + + return [row_ranges, page_locations, num_pages, row_group_row_count, + page_counter](const ::parquet::DataPageStats& /*stats*/) -> bool { + int32_t page_idx = (*page_counter)++; + + if (page_idx >= num_pages) { + // Safety: if more pages than expected, don't skip + return false; + } + + int64_t first_row = page_locations[page_idx].first_row_index; + int64_t last_row; + if (page_idx + 1 < num_pages) { + last_row = page_locations[page_idx + 1].first_row_index - 1; + } else { + last_row = row_group_row_count - 1; + } + + // Return true to skip this page if it has no overlap with RowRanges + return !row_ranges.IsOverlapping(first_row, last_row); + }; +} + +std::pair PageFilteredRowGroupReader::ComputeCompressedRowRanges( + const RowRanges& original_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, + int64_t row_group_row_count) { + const auto& page_locations = offset_index->page_locations(); + auto num_pages = static_cast(page_locations.size()); + const auto& ranges = original_ranges.GetRanges(); + + RowRanges compressed; + int64_t compressed_offset = 0; + + for (int32_t page_idx = 0; page_idx < num_pages; ++page_idx) { + int64_t page_from = page_locations[page_idx].first_row_index; + int64_t page_to = (page_idx + 1 < num_pages) + ? page_locations[page_idx + 1].first_row_index - 1 + : row_group_row_count - 1; + int64_t page_size = page_to - page_from + 1; + + if (!original_ranges.IsOverlapping(page_from, page_to)) { + // Page will be skipped by data_page_filter, not in compressed space + continue; + } + + // Page is kept. Map overlapping original ranges to compressed row space. + for (const auto& range : ranges) { + if (range.to < page_from) { + continue; + } + if (range.from > page_to) { + break; // Ranges are sorted + } + int64_t overlap_from = std::max(range.from, page_from); + int64_t overlap_to = std::min(range.to, page_to); + int64_t c_from = compressed_offset + (overlap_from - page_from); + int64_t c_to = compressed_offset + (overlap_to - page_from); + compressed.Add(RowRanges::Range(c_from, c_to)); + } + + compressed_offset += page_size; + } + + return {compressed, compressed_offset}; +} + +Result> PageFilteredRowGroupReader::ReadFilteredColumn( + const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader, + ::parquet::ParquetFileReader* parquet_reader, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, int32_t row_group_index, + int32_t column_index, const RowRanges& row_ranges, const std::shared_ptr& field, + int64_t row_group_row_count, ::arrow::MemoryPool* pool) { + auto file_metadata = parquet_reader->metadata(); + const auto* col_descriptor = file_metadata->schema()->Column(column_index); + + // Try to get OffsetIndex for I/O-level page skipping + RowRanges effective_ranges = row_ranges; + int64_t effective_row_count = row_group_row_count; + + std::shared_ptr<::parquet::OffsetIndex> offset_index; + if (page_index_reader) { + auto rg_page_index_reader = page_index_reader->RowGroup(row_group_index); + if (rg_page_index_reader) { + offset_index = rg_page_index_reader->GetOffsetIndex(column_index); + } + } + + auto page_reader = row_group_reader->GetColumnPageReader(column_index); + + if (offset_index) { + // Set data_page_filter for I/O-level page skipping + page_reader->set_data_page_filter( + MakePageFilter(row_ranges, offset_index, row_group_row_count)); + // Compute compressed RowRanges for the decode-level skip/read pattern + auto [compressed_ranges, compressed_total] = + ComputeCompressedRowRanges(row_ranges, offset_index, row_group_row_count); + effective_ranges = std::move(compressed_ranges); + effective_row_count = compressed_total; + } + + // Create RecordReader + ::parquet::internal::LevelInfo leaf_info = + ::parquet::internal::LevelInfo::ComputeLevelInfo(col_descriptor); + auto record_reader = ::parquet::internal::RecordReader::Make(col_descriptor, leaf_info, pool); + record_reader->SetPageReader(std::move(page_reader)); + + // Execute skip/read pattern based on effective RowRanges + const auto& ranges = effective_ranges.GetRanges(); + int64_t current_row = 0; + + for (const auto& range : ranges) { + // Skip rows before this range + if (range.from > current_row) { + int64_t to_skip = range.from - current_row; + int64_t skipped = record_reader->SkipRecords(to_skip); + if (skipped != to_skip) { + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: expected to skip {} records but skipped {} " + "(row_group={}, column={})", + to_skip, skipped, row_group_index, column_index)); + } + current_row = range.from; + } + + // Read rows in this range + int64_t to_read = range.Count(); + int64_t read = record_reader->ReadRecords(to_read); + if (read != to_read) { + return Status::Invalid( + fmt::format("PageFilteredRowGroupReader: expected to read {} records but read {} " + "(row_group={}, column={}, range=[{},{}])", + to_read, read, row_group_index, column_index, range.from, range.to)); + } + current_row += to_read; + } + + // Skip remaining rows after the last range to properly finalize the reader + if (current_row < effective_row_count) { + record_reader->SkipRecords(effective_row_count - current_row); + } + + // Transfer to Arrow ChunkedArray + std::shared_ptr chunked_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(::parquet::arrow::TransferColumnData( + record_reader.get(), field, col_descriptor, pool, &chunked_array)); + + return chunked_array; +} + +Result> PageFilteredRowGroupReader::ReadFilteredRowGroup( + ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, + const RowRanges& row_ranges, const std::vector& column_indices, + const std::shared_ptr& arrow_schema, ::arrow::MemoryPool* pool, + const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize) { + if (row_ranges.IsEmpty()) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr empty_table, + arrow::Table::MakeEmpty(arrow_schema, pool)); + return std::make_unique(std::move(empty_table), max_chunksize); + } + + int64_t expected_rows = row_ranges.RowCount(); + + // Wait for pre-buffered data to be ready. + // When pre_buffered=true, PreBuffer was already called in PrepareForReading() covering + // all row groups in parallel. We only need to wait. Calling PreBuffer again would create + // a new cached_source_, discarding the parallel I/O already in progress. + { + std::vector rg_vec = {row_group_index}; + std::vector col_vec(column_indices.begin(), column_indices.end()); + if (!pre_buffered) { + ::arrow::io::IOContext io_ctx(pool); + parquet_reader->PreBuffer(rg_vec, col_vec, io_ctx, cache_options); + } + if (!page_ranges.empty()) { + // Page-level PreBuffer: wait on specific page byte ranges + // If pre-buffering failed (e.g., IO error during testing), fall back to on-demand read + auto status = parquet_reader->WhenBufferedRanges(page_ranges).status(); + if (!status.ok()) { + // Pre-buffering failed, fall back to row-group level PreBuffer + ::arrow::io::IOContext io_ctx(pool); + parquet_reader->PreBuffer(rg_vec, col_vec, io_ctx, cache_options); + } + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(parquet_reader->WhenBuffered(rg_vec, col_vec).status()); + } + } + + // Open row group and page index once, share across all columns + auto row_group_reader = parquet_reader->RowGroup(row_group_index); + auto rg_metadata = parquet_reader->metadata()->RowGroup(row_group_index); + int64_t row_group_row_count = rg_metadata->num_rows(); + auto page_index_reader = parquet_reader->GetPageIndexReader(); + + // Read each column with page filtering + std::vector> columns; + columns.reserve(column_indices.size()); + + for (size_t i = 0; i < column_indices.size(); ++i) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr chunked_array, + ReadFilteredColumn(row_group_reader, parquet_reader, page_index_reader, row_group_index, + column_indices[i], row_ranges, + arrow_schema->field(static_cast(i)), row_group_row_count, + pool)); + + if (chunked_array->length() != expected_rows) { + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: column {} produced {} rows but expected {} " + "(row_group={})", + column_indices[i], chunked_array->length(), expected_rows, row_group_index)); + } + + columns.push_back(std::move(chunked_array)); + } + + // Wrap columns in a Table and stream zero-copy-sliced batches via TableBatchReader. + // For multi-chunk variable-length columns this avoids the deep copy of CombineChunks: + // each emitted batch contains at most max_chunksize rows (capped further by the + // smallest remaining chunk across columns), and every column's Array is a zero-copy + // Slice of its underlying chunk. + auto table = arrow::Table::Make(arrow_schema, std::move(columns), expected_rows); + return std::make_unique(std::move(table), max_chunksize); +} + +std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRanges( + ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, + const RowRanges& row_ranges, const std::vector& column_indices) { + std::vector<::arrow::io::ReadRange> ranges; + auto file_metadata = parquet_reader->metadata(); + auto rg_metadata = file_metadata->RowGroup(row_group_index); + int64_t row_group_row_count = rg_metadata->num_rows(); + + auto page_index_reader = parquet_reader->GetPageIndexReader(); + std::shared_ptr<::parquet::RowGroupPageIndexReader> rg_page_index_reader; + if (page_index_reader) { + rg_page_index_reader = page_index_reader->RowGroup(row_group_index); + } + + for (int32_t col_idx : column_indices) { + auto col_chunk = rg_metadata->ColumnChunk(col_idx); + int64_t data_page_offset = col_chunk->data_page_offset(); + int64_t total_compressed_size = col_chunk->total_compressed_size(); + int64_t chunk_end = data_page_offset + total_compressed_size; + + // Dictionary page: always include if present + if (col_chunk->has_dictionary_page()) { + int64_t dict_offset = col_chunk->dictionary_page_offset(); + int64_t dict_size = data_page_offset - dict_offset; + if (dict_size > 0) { + ranges.push_back({dict_offset, dict_size}); + } + } + + // Try to get OffsetIndex for page-level ranges + std::shared_ptr<::parquet::OffsetIndex> offset_index; + if (rg_page_index_reader) { + offset_index = rg_page_index_reader->GetOffsetIndex(col_idx); + } + + if (!offset_index) { + // No OffsetIndex: fall back to entire column chunk + ranges.push_back({data_page_offset, total_compressed_size}); + continue; + } + + const auto& page_locations = offset_index->page_locations(); + auto num_pages = static_cast(page_locations.size()); + + for (int32_t page_idx = 0; page_idx < num_pages; ++page_idx) { + int64_t first_row = page_locations[page_idx].first_row_index; + int64_t last_row = (page_idx + 1 < num_pages) + ? page_locations[page_idx + 1].first_row_index - 1 + : row_group_row_count - 1; + + if (!row_ranges.IsOverlapping(first_row, last_row)) { + continue; // Page doesn't overlap with target rows + } + + // Compute page byte range + int64_t page_offset = page_locations[page_idx].offset; + int64_t page_size; + if (page_idx + 1 < num_pages) { + page_size = page_locations[page_idx + 1].offset - page_offset; + } else { + page_size = chunk_end - page_offset; + } + ranges.push_back({page_offset, page_size}); + } + } + + return ranges; +} + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h new file mode 100644 index 00000000..b3323ce4 --- /dev/null +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/io/caching.h" +#include "arrow/memory_pool.h" +#include "arrow/record_batch.h" +#include "arrow/type.h" +#include "paimon/format/parquet/row_ranges.h" +#include "paimon/result.h" +#include "parquet/column_reader.h" +#include "parquet/file_reader.h" +#include "parquet/page_index.h" + +namespace paimon::parquet { + +/// Reads a single row group using page-level filtering. +/// Non-matching rows are skipped at the decoding level via RecordReader::SkipRecords, +/// using RowRanges computed from the page index (ColumnIndex + OffsetIndex). +/// MakePageFilter is available for future I/O-level page skipping optimization. +class PageFilteredRowGroupReader { + public: + PageFilteredRowGroupReader() = delete; + ~PageFilteredRowGroupReader() = delete; + + /// Read a row group with page-level filtering. + /// @param parquet_reader The underlying ParquetFileReader + /// @param row_group_index Row group to read + /// @param row_ranges Matching row ranges within this row group + /// @param column_indices Leaf column indices to read + /// @param arrow_schema The target Arrow schema for output columns + /// @param pool Memory pool + /// @param cache_options Cache options for PreBuffer + /// @param pre_buffered If true, assumes PreBuffer was already called externally + /// and only waits via WhenBuffered (no redundant PreBuffer). + /// @param page_ranges If non-empty, wait via WhenBufferedRanges instead of WhenBuffered + /// @param max_chunksize Per-batch row cap for the returned reader, mirroring Arrow's + /// TableBatchReader::set_chunksize. Each batch yields at most this many rows; + /// actual size may be smaller when an underlying ChunkedArray's chunk boundary + /// is reached first (zero-copy slice). + /// @return A RecordBatchReader streaming the filtered rows. Multi-chunk variable-length + /// columns are emitted as multiple zero-copy-sliced batches along chunk boundaries + /// instead of being concatenated, avoiding the deep copy of CombineChunks. + static Result> ReadFilteredRowGroup( + ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, + const RowRanges& row_ranges, const std::vector& column_indices, + const std::shared_ptr& arrow_schema, ::arrow::MemoryPool* pool, + const ::arrow::io::CacheOptions& cache_options = ::arrow::io::CacheOptions::Defaults(), + bool pre_buffered = false, const std::vector<::arrow::io::ReadRange>& page_ranges = {}, + int64_t max_chunksize = std::numeric_limits::max()); + + /// Compute the byte ranges of pages that overlap with the given RowRanges. + /// Uses OffsetIndex to determine per-page file offsets and sizes. + /// Includes dictionary pages unconditionally. + /// Falls back to entire column chunk range if OffsetIndex is unavailable. + static std::vector<::arrow::io::ReadRange> ComputePageRanges( + ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, + const RowRanges& row_ranges, const std::vector& column_indices); + + private: + /// Create a data_page_filter callback for a column based on RowRanges + OffsetIndex. + /// Returns true (skip) if the page's row range has no overlap with RowRanges. + static std::function MakePageFilter( + const RowRanges& row_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, + int64_t row_group_row_count); + + /// Read a single column using skip/read pattern driven by RowRanges. + /// When OffsetIndex is available, uses data_page_filter for I/O-level page skipping + /// and compressed RowRanges for decode-level row skipping. + static Result> ReadFilteredColumn( + const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader, + ::parquet::ParquetFileReader* parquet_reader, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, + int32_t row_group_index, int32_t column_index, const RowRanges& row_ranges, + const std::shared_ptr& field, int64_t row_group_row_count, + ::arrow::MemoryPool* pool); + + /// Compute compressed RowRanges after data_page_filter skips non-matching pages. + /// Maps original RowRanges to the compressed row space where skipped pages are removed. + /// @return pair of (compressed RowRanges, compressed total row count) + static std::pair ComputeCompressedRowRanges( + const RowRanges& original_ranges, + const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); +}; + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp new file mode 100644 index 00000000..87fe7349 --- /dev/null +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -0,0 +1,725 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/page_filtered_row_group_reader.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/array/array_nested.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/defs.h" +#include "paimon/format/parquet/parquet_file_batch_reader.h" +#include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_format_writer.h" +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" +#include "parquet/arrow/reader.h" +#include "parquet/file_reader.h" +#include "parquet/properties.h" + +namespace paimon { +class Predicate; +} // namespace paimon + +namespace paimon::parquet::test { + +/// Test fixture for page-level filtering. +/// Creates Parquet files with multiple row groups and small page sizes to ensure +/// multiple pages per row group, enabling page-level filtering tests. +class PageFilteredRowGroupReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + arrow_pool_ = GetArrowPool(pool_); + dir_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + fs_ = dir_->GetFileSystem(); + } + + /// Write a Parquet file with controlled page boundaries. + /// @param file_name Output file name + /// @param struct_array Data to write + /// @param write_batch_size Controls page size (number of rows per page) + /// @param max_row_group_length Controls row group size + void WriteTestFile(const std::string& file_name, + const std::shared_ptr& struct_array, + int32_t write_batch_size, int64_t max_row_group_length) { + auto data_type = struct_array->struct_type(); + auto data_schema = arrow::schema(data_type->fields()); + auto data_arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*struct_array, data_arrow_array.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_name, /*overwrite=*/false)); + ::parquet::WriterProperties::Builder builder; + builder.write_batch_size(write_batch_size); + builder.max_row_group_length(max_row_group_length); + builder.disable_dictionary(); // Ensure page index min/max are meaningful + builder.enable_write_page_index(); // Enable page index for page-level filtering + // Set data page size to 1 byte to force a new page after every write_batch_size rows. + // The writer flushes a page when accumulated data exceeds data_pagesize, so setting + // it to 1 ensures each batch of write_batch_size rows becomes exactly one page. + builder.data_pagesize(1); + auto writer_properties = builder.build(); + ASSERT_OK_AND_ASSIGN( + auto format_writer, + ParquetFormatWriter::Create(out, data_schema, writer_properties, + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_)); + ASSERT_OK(format_writer->AddBatch(data_arrow_array.get())); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Close()); + } + + /// Read back a Parquet file with an optional predicate and page index filter enabled. + /// Returns the collected result as a ChunkedArray. + void ReadWithPredicateImpl(const std::string& file_name, + const std::shared_ptr& read_schema, + const std::shared_ptr& predicate, + std::shared_ptr* out, + int32_t batch_size = 1024) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + + std::map options; + options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "true"; + ASSERT_OK_AND_ASSIGN( + auto batch_reader, + ParquetFileBatchReader::Create(std::move(in_stream), arrow_pool_, options, batch_size)); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(*out, + paimon::test::ReadResultCollector::CollectResult(batch_reader.get())); + } + + protected: + std::shared_ptr arrow_pool_; + std::shared_ptr pool_; + std::shared_ptr fs_; + std::unique_ptr dir_; +}; + +// Helper: build a StructArray with N rows of int32 "val" column with sequential values. +// val[i] = i for i in [0, N). +static std::shared_ptr MakeSequentialIntData(int32_t num_rows) { + arrow::Int32Builder val_builder; + EXPECT_TRUE(val_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + val_builder.UnsafeAppend(i); + } + auto val_array = val_builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::int32()); + return arrow::StructArray::Make({val_array}, {field}).ValueOrDie(); +} + +// Helper: build a StructArray with two int32 columns: "a" and "b". +// a[i] = i, b[i] = i * 10, for i in [0, N). +static std::shared_ptr MakeTwoColumnData(int32_t num_rows) { + arrow::Int32Builder a_builder, b_builder; + EXPECT_TRUE(a_builder.Reserve(num_rows).ok()); + EXPECT_TRUE(b_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + a_builder.UnsafeAppend(i); + b_builder.UnsafeAppend(i * 10); + } + auto a_array = a_builder.Finish().ValueOrDie(); + auto b_array = b_builder.Finish().ValueOrDie(); + auto field_a = arrow::field("a", arrow::int32()); + auto field_b = arrow::field("b", arrow::int32()); + return arrow::StructArray::Make({a_array, b_array}, {field_a, field_b}).ValueOrDie(); +} + +/// Test: page-level filtering correctly skips non-matching pages. +/// +/// Scenario: 100 rows, 10 rows per page, 1 row group. +/// val[i] = i. Predicate: val >= 50. Pages 0-4 (rows 0-49) should be skipped, +/// pages 5-9 (rows 50-99) should be read. +TEST_F(PageFilteredRowGroupReaderTest, SingleRowGroupPartialPageMatch) { + std::string file_name = dir_->Str() + "/single_rg_partial.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(50)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + // Should get rows 50-99 = 50 rows + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + // Verify actual values + auto flat = result->chunk(0); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + ASSERT_TRUE(val_arr); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(50 + i, val_arr->Value(i)) << "Mismatch at index " << i; + } +} + +/// Test: predicate matches all pages → same as unfiltered read. +TEST_F(PageFilteredRowGroupReaderTest, AllPagesMatch) { + std::string file_name = dir_->Str() + "/all_match.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(0)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(100, result->length()); +} + +/// Test: predicate matches no pages → empty result. +TEST_F(PageFilteredRowGroupReaderTest, NoPagesMatch) { + std::string file_name = dir_->Str() + "/no_match.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterThan( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(999)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + // No matching rows; result should be null (empty) + ASSERT_FALSE(result); +} + +/// Test: multiple row groups, page filtering active on some. +/// +/// 200 rows, 10 rows per page, 50 rows per row group → 4 row groups. +/// Predicate: val >= 150. Row groups 0-2 (rows 0-149) should be eliminated entirely. +/// Row group 3 (rows 150-199): all pages match → full read, no page filtering. +TEST_F(PageFilteredRowGroupReaderTest, MultipleRowGroupsFullElimination) { + std::string file_name = dir_->Str() + "/multi_rg_elim.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(150)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + // Verify values are 150-199 + auto flat = result->chunk(0); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(150 + i, val_arr->Value(i)); + } +} + +/// Test: multiple row groups, partial page match within a row group. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Predicate: val >= 50 AND val < 150. +/// Row group 0 (rows 0-99): pages 0-4 skipped, pages 5-9 read → 50 rows +/// Row group 1 (rows 100-199): pages 0-4 read, pages 5-9 skipped → 50 rows +/// Total: 100 rows +TEST_F(PageFilteredRowGroupReaderTest, MultipleRowGroupsPartialPageMatch) { + std::string file_name = dir_->Str() + "/multi_rg_partial.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::And( + {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val", + FieldType::INT, Literal(50)), + PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT, + Literal(150))})); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(100, result->length()); + + // Collect all values and verify they are 50-149 + int64_t offset = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_arr = std::dynamic_pointer_cast(result->chunk(i)); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int64_t j = 0; j < val_arr->length(); ++j) { + ASSERT_EQ(50 + offset, val_arr->Value(j)) << "Mismatch at offset " << offset; + ++offset; + } + } + ASSERT_EQ(100, offset); +} + +/// Test: two columns remain aligned after page-level filtering. +/// +/// 100 rows, a[i] = i, b[i] = i*10. 10 rows per page. +/// Predicate on "a": a >= 50. After filtering, b should be b[50..99] = {500, 510, ..., 990}. +TEST_F(PageFilteredRowGroupReaderTest, MultiColumnAlignment) { + std::string file_name = dir_->Str() + "/multi_col.parquet"; + auto data = MakeTwoColumnData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = + arrow::schema({arrow::field("a", arrow::int32()), arrow::field("b", arrow::int32())}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"a", FieldType::INT, Literal(50)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + auto struct_arr = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(struct_arr); + auto a_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + auto b_arr = std::dynamic_pointer_cast(struct_arr->field(1)); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(50 + i, a_arr->Value(i)); + ASSERT_EQ((50 + i) * 10, b_arr->Value(i)); + } +} + +/// Test: predicate matches pages in the middle of a row group. +/// +/// 100 rows, 10 rows per page. Predicate: val >= 30 AND val < 70. +/// Pages 0-2 (rows 0-29) skipped, pages 3-6 (rows 30-69) read, pages 7-9 (rows 70-99) skipped. +TEST_F(PageFilteredRowGroupReaderTest, MiddlePagesMatch) { + std::string file_name = dir_->Str() + "/middle_pages.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::And( + {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val", + FieldType::INT, Literal(30)), + PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT, + Literal(70))})); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(40, result->length()); + + int64_t offset = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_arr = std::dynamic_pointer_cast(result->chunk(i)); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int64_t j = 0; j < val_arr->length(); ++j) { + ASSERT_EQ(30 + offset, val_arr->Value(j)); + ++offset; + } + } + ASSERT_EQ(40, offset); +} + +/// Test: no predicate → all data returned (no filtering). +TEST_F(PageFilteredRowGroupReaderTest, NoPredicate) { + std::string file_name = dir_->Str() + "/no_predicate.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, /*predicate=*/nullptr, &result); + ASSERT_NE(nullptr, result); + ASSERT_EQ(100, result->length()); +} + +/// Test: page filtering with EQUAL predicate that matches a single page. +/// +/// 100 rows, 10 rows per page. Predicate: val == 55. +/// Only page 5 (rows 50-59) should match, containing value 55. +TEST_F(PageFilteredRowGroupReaderTest, EqualPredicateSinglePageMatch) { + std::string file_name = dir_->Str() + "/equal_single_page.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(55)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + // Page 5 has rows 50-59, which includes 55. The entire page is returned. + ASSERT_EQ(10, result->length()); + + auto struct_arr = std::dynamic_pointer_cast(result->chunk(0)); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(50 + i, val_arr->Value(i)); + } +} + +/// Test: page filtering with LessThan predicate. +/// +/// 100 rows, 10 rows per page. Predicate: val < 25. +/// Pages 0-2 (rows 0-29) match (page 2 has min=20 < 25). +/// Pages 3-9 don't match. +TEST_F(PageFilteredRowGroupReaderTest, LessThanPredicatePageMatch) { + std::string file_name = dir_->Str() + "/less_than.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::LessThan( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(25)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + // Pages 0 (0-9), 1 (10-19), 2 (20-29) match because their min < 25. + // Page 2 has min=20, max=29, and 20 < 25, so it matches. + ASSERT_EQ(30, result->length()); + + auto struct_arr = std::dynamic_pointer_cast(result->chunk(0)); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 30; ++i) { + ASSERT_EQ(i, val_arr->Value(i)); + } +} + +/// Test: large data with multiple row groups and page filtering. +/// +/// 1000 rows, 10 rows per page, 200 rows per row group → 5 row groups. +/// Predicate: val >= 500 AND val < 700. +/// Row groups 0,1 (rows 0-399): all pages eliminated +/// Row group 2 (rows 400-599): pages 0-9 (400-499) eliminated, pages 10-19 (500-599) read +/// Row group 3 (rows 600-799): pages 0-9 (600-699) read, pages 10-19 (700-799) eliminated +/// Row group 4 (rows 800-999): all pages eliminated +/// Total: 200 rows (500-699) +TEST_F(PageFilteredRowGroupReaderTest, LargeDataMultiRowGroupPageFilter) { + std::string file_name = dir_->Str() + "/large_data.parquet"; + auto data = MakeSequentialIntData(1000); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/200); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::And( + {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val", + FieldType::INT, Literal(500)), + PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT, + Literal(700))})); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + ASSERT_EQ(200, result->length()); + + // Verify values are 500-699 + int64_t offset = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_arr = std::dynamic_pointer_cast(result->chunk(i)); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int64_t j = 0; j < val_arr->length(); ++j) { + ASSERT_EQ(500 + offset, val_arr->Value(j)) << "Mismatch at offset " << offset; + ++offset; + } + } + ASSERT_EQ(200, offset); +} + +/// Test: string column page filtering. +/// +/// Write 40 rows with string values: "aaa_00", "aaa_01", ..., "aaa_09", +/// "bbb_10", ..., "bbb_19", "ccc_20", ..., "ccc_29", "ddd_30", ..., "ddd_39". +/// 10 rows per page → 4 pages. Predicate: val >= "ccc" should match pages 2-3. +TEST_F(PageFilteredRowGroupReaderTest, StringColumnPageFilter) { + std::string file_name = dir_->Str() + "/string_filter.parquet"; + + arrow::StringBuilder str_builder; + ASSERT_TRUE(str_builder.Reserve(40).ok()); + std::vector prefixes = {"aaa", "bbb", "ccc", "ddd"}; + for (int32_t i = 0; i < 40; ++i) { + std::string val = prefixes[i / 10] + "_" + (i < 10 ? "0" : "") + std::to_string(i); + ASSERT_TRUE(str_builder.Append(val).ok()); + } + auto str_array = str_builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::utf8()); + auto struct_arr = arrow::StructArray::Make({str_array}, {field}).ValueOrDie(); + + WriteTestFile(file_name, struct_arr, /*write_batch_size=*/10, /*max_row_group_length=*/40); + + auto read_schema = arrow::schema({field}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::STRING, + Literal(FieldType::STRING, "ccc", 3)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + ASSERT_TRUE(result); + // Pages 2 (ccc_20..ccc_29) and 3 (ddd_30..ddd_39) should match. + ASSERT_EQ(20, result->length()); +} + +/// Test: ComputePageRanges returns only matching page byte ranges. +/// +/// 100 rows, 10 rows per page, 1 row group with page index enabled. +/// RowRanges = [50, 59] (page 5 only). Should return exactly 1 page range per column. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesPartialMatch) { + std::string file_name = dir_->Str() + "/compute_ranges_partial.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + // Open as raw ParquetFileReader + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + ASSERT_TRUE(parquet_reader); + + // Single page match: rows [50, 59] = page 5 + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(50, 59)); + + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + parquet_reader.get(), /*row_group_index=*/0, row_ranges, /*column_indices=*/{0}); + + // Should have exactly 1 range (page 5 of column 0, no dictionary since disabled) + ASSERT_EQ(1, ranges.size()); + ASSERT_GT(ranges[0].offset, 0); + ASSERT_GT(ranges[0].length, 0); +} + +/// Test: ComputePageRanges returns all page ranges when RowRanges covers entire row group. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesAllMatch) { + std::string file_name = dir_->Str() + "/compute_ranges_all.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + + // All rows match + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(0, 99)); + + auto ranges = + PageFilteredRowGroupReader::ComputePageRanges(parquet_reader.get(), 0, row_ranges, {0}); + + // 10 pages, all matching + ASSERT_EQ(10, ranges.size()); + for (const auto& r : ranges) { + ASSERT_GT(r.offset, 0); + ASSERT_GT(r.length, 0); + } +} + +/// Test: ComputePageRanges returns no page ranges for empty RowRanges. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesNoMatch) { + std::string file_name = dir_->Str() + "/compute_ranges_none.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + + RowRanges row_ranges; // empty + + auto ranges = + PageFilteredRowGroupReader::ComputePageRanges(parquet_reader.get(), 0, row_ranges, {0}); + + ASSERT_EQ(0, ranges.size()); +} + +/// Test: ComputePageRanges with multiple columns returns ranges for each column. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiColumn) { + std::string file_name = dir_->Str() + "/compute_ranges_multi_col.parquet"; + auto data = MakeTwoColumnData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + + // Match page 5 only (rows 50-59) + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(50, 59)); + + auto ranges = + PageFilteredRowGroupReader::ComputePageRanges(parquet_reader.get(), 0, row_ranges, {0, 1}); + + // 1 matching page per column = 2 ranges total + ASSERT_EQ(2, ranges.size()); + // Ranges should be at different offsets (different columns) + ASSERT_NE(ranges[0].offset, ranges[1].offset); +} + +/// Test: ComputePageRanges with multiple matching pages. +/// +/// 100 rows, 10 per page. RowRanges = [20,29] + [70,79] = pages 2 and 7. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) { + std::string file_name = dir_->Str() + "/compute_ranges_multi_page.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(20, 29)); + row_ranges.Add(RowRanges::Range(70, 79)); + + auto ranges = + PageFilteredRowGroupReader::ComputePageRanges(parquet_reader.get(), 0, row_ranges, {0}); + + // 2 matching pages for 1 column + ASSERT_EQ(2, ranges.size()); + // Pages should be at increasing offsets + ASSERT_LT(ranges[0].offset, ranges[1].offset); +} + +/// Test: variable-length columns are streamed across multiple zero-copy-sliced +/// RecordBatches when batch_size is smaller than the matched row count, instead of +/// being concatenated into a single RecordBatch via CombineChunks. +/// +/// This verifies the alignment with Arrow's standard TableBatchReader path: +/// multi-chunk binary/string columns split along chunk + batch_size boundaries, +/// with no deep copy. Asserts both correctness (total rows + full content order) and +/// the multi-batch shape (more than one chunk in the collected ChunkedArray). +TEST_F(PageFilteredRowGroupReaderTest, StringColumnMultiBatchStreaming) { + std::string file_name = dir_->Str() + "/string_multi_batch.parquet"; + + arrow::StringBuilder str_builder; + ASSERT_TRUE(str_builder.Reserve(60).ok()); + // 6 pages of 10 rows each: prefix "p0_".."p5_" so each page has a distinct min/max. + for (int32_t i = 0; i < 60; ++i) { + std::string val = + "p" + std::to_string(i / 10) + "_" + (i < 10 ? "0" : "") + std::to_string(i); + ASSERT_TRUE(str_builder.Append(val).ok()); + } + auto str_array = str_builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::utf8()); + auto struct_arr = arrow::StructArray::Make({str_array}, {field}).ValueOrDie(); + + WriteTestFile(file_name, struct_arr, /*write_batch_size=*/10, /*max_row_group_length=*/60); + + // Predicate matches pages 2..5 (40 rows: "p2_20".."p5_59"). batch_size=7 forces + // the wrapper to surface multiple batches per page-filtered RG. + auto read_schema = arrow::schema({field}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::STRING, + Literal(FieldType::STRING, "p2", 2)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/7); + ASSERT_TRUE(result); + ASSERT_EQ(40, result->length()); + + // Multi-batch shape: with 40 matched rows and batch_size=7 we expect at least + // ceil(40/7)=6 chunks. Anything > 1 already proves we did not collapse to a single + // post-CombineChunks RecordBatch. + ASSERT_GT(result->num_chunks(), 1); + + // Content correctness: rows arrive in the original page order, "p2_20" through "p5_59". + int64_t seen = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_chunk = std::dynamic_pointer_cast(result->chunk(i)); + ASSERT_TRUE(struct_chunk); + auto str_chunk = std::dynamic_pointer_cast(struct_chunk->field(0)); + ASSERT_TRUE(str_chunk); + for (int64_t j = 0; j < str_chunk->length(); ++j) { + int32_t row = 20 + static_cast(seen); + std::string expected = + "p" + std::to_string(row / 10) + "_" + (row < 10 ? "0" : "") + std::to_string(row); + ASSERT_EQ(expected, str_chunk->GetString(j)); + ++seen; + } + } + ASSERT_EQ(40, seen); +} + +/// Test: end-to-end page-filtered read produces correct results when using page-level PreBuffer. +/// +/// This exercises the full path: ComputePageRanges → PreBufferRanges → CachedInputStream → +/// ReadFilteredRowGroup with page_ranges. +TEST_F(PageFilteredRowGroupReaderTest, EndToEndPageLevelPreBuffer) { + std::string file_name = dir_->Str() + "/e2e_page_prebuffer.parquet"; + auto data = MakeSequentialIntData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + // Read via the standard ParquetFileBatchReader path (page index enabled) + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto predicate = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(55)); + + // Use small batch_size to verify batched consumption of page-filtered results + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/3); + ASSERT_TRUE(result); + // Page 5 (rows 50-59) matches, should return 10 rows + ASSERT_EQ(10, result->length()); + + // Verify actual values across chunks + int64_t offset = 0; + for (int i = 0; i < result->num_chunks(); ++i) { + auto struct_arr = std::dynamic_pointer_cast(result->chunk(i)); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int64_t j = 0; j < val_arr->length(); ++j) { + ASSERT_EQ(50 + offset, val_arr->Value(j)); + ++offset; + } + } + ASSERT_EQ(10, offset); +} + +} // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index d7a1cde1..7eb3066e 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -18,6 +18,7 @@ #include "paimon/format/parquet/parquet_file_batch_reader.h" +#include #include #include @@ -49,6 +50,17 @@ #include "parquet/arrow/reader.h" #include "parquet/properties.h" +// Convert any std::exception thrown by underlying Parquet/Arrow APIs into a +// Status. Used as the trailing catch clauses of a try block in every public +// method that calls into the parquet C++ API, so the read layer never throws. +#define PAIMON_PARQUET_CATCH_AND_RETURN_STATUS(context) \ + catch (const std::exception& e) { \ + return Status::Invalid(fmt::format("{}: {}", (context), e.what())); \ + } \ + catch (...) { \ + return Status::UnknownError((context), ": unknown error"); \ + } + namespace arrow { class MemoryPool; } // namespace arrow @@ -67,99 +79,149 @@ ParquetFileBatchReader::ParquetFileBatchReader( input_stream_(std::move(input_stream)), reader_(std::move(reader)), read_ranges_(reader_->GetAllRowGroupRanges()), - metrics_(std::make_shared()) {} + metrics_(std::make_shared()), + logger_(Logger::GetLogger("ParquetFileBatchReader")) {} Result> ParquetFileBatchReader::Create( std::shared_ptr&& input_stream, const std::shared_ptr& pool, const std::map& options, int32_t batch_size) { - assert(input_stream); - PAIMON_ASSIGN_OR_RAISE(::parquet::ReaderProperties reader_properties, - CreateReaderProperties(pool, options)); - PAIMON_ASSIGN_OR_RAISE(::parquet::ArrowReaderProperties arrow_reader_properties, - CreateArrowReaderProperties(pool, options, batch_size)); - - ::parquet::arrow::FileReaderBuilder file_reader_builder; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.Open(input_stream, reader_properties)); - - std::unique_ptr<::parquet::arrow::FileReader> file_reader; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.memory_pool(pool.get()) - ->properties(arrow_reader_properties) - ->Build(&file_reader)); - - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, - FileReaderWrapper::Create(std::move(file_reader))); - auto parquet_file_batch_reader = std::unique_ptr( - new ParquetFileBatchReader(std::move(input_stream), std::move(reader), options, pool)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, - parquet_file_batch_reader->GetFileSchema()); - PAIMON_RETURN_NOT_OK(parquet_file_batch_reader->SetReadSchema( - file_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); - return parquet_file_batch_reader; + try { + assert(input_stream); + PAIMON_ASSIGN_OR_RAISE(::parquet::ReaderProperties reader_properties, + CreateReaderProperties(pool, options)); + + PAIMON_ASSIGN_OR_RAISE(::parquet::ArrowReaderProperties arrow_reader_properties, + CreateArrowReaderProperties(pool, options, batch_size)); + + ::parquet::arrow::FileReaderBuilder file_reader_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.Open(input_stream, reader_properties)); + + std::unique_ptr<::parquet::arrow::FileReader> file_reader; + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.memory_pool(pool.get()) + ->properties(arrow_reader_properties) + ->Build(&file_reader)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + FileReaderWrapper::Create(std::move(file_reader), pool.get(), + static_cast(batch_size))); + auto parquet_file_batch_reader = std::unique_ptr( + new ParquetFileBatchReader(std::move(input_stream), std::move(reader), options, pool)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, + parquet_file_batch_reader->GetFileSchema()); + PAIMON_RETURN_NOT_OK(parquet_file_batch_reader->SetReadSchema( + file_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); + return parquet_file_batch_reader; + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::Create") } Result> ParquetFileBatchReader::GetFileSchema() const { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_schema, - ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(file_schema)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr new_type, - ParquetTimestampConverter::AdjustTimezone(arrow::struct_(new_schema->fields()))); + try { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_schema, + ParquetFieldIdConverter::GetPaimonIdsFromParquetIds(file_schema)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr new_type, + ParquetTimestampConverter::AdjustTimezone(arrow::struct_(new_schema->fields()))); - auto c_schema = std::make_unique<::ArrowSchema>(); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*new_type, c_schema.get())); - return c_schema; + auto c_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportType(*new_type, c_schema.get())); + return c_schema; + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::GetFileSchema") } Status ParquetFileBatchReader::SetReadSchema( ::ArrowSchema* schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { - if (!schema) { - return Status::Invalid("SetReadSchema failed: read schema cannot be nullptr"); - } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema, - arrow::ImportSchema(schema)); - - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); - std::unordered_map> field_index_map; - int32_t i = 0; - for (const auto& field : file_schema->fields()) { - std::vector v; - FlattenSchema(field->type(), &i, &v); - field_index_map[field->name()] = v; - } + try { + if (!schema) { + return Status::Invalid("SetReadSchema failed: read schema cannot be nullptr"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema, + arrow::ImportSchema(schema)); - std::vector column_indices; - for (const auto& field : read_schema->field_names()) { - if (field_index_map.find(field) != field_index_map.end()) { - for (int32_t index : field_index_map[field]) { - column_indices.push_back(index); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); + std::unordered_map> field_index_map; + int32_t i = 0; + for (const auto& field : file_schema->fields()) { + std::vector v; + FlattenSchema(field->type(), &i, &v); + field_index_map[field->name()] = v; + } + + std::vector column_indices; + for (const auto& field : read_schema->field_names()) { + if (field_index_map.find(field) != field_index_map.end()) { + for (int32_t index : field_index_map[field]) { + column_indices.push_back(index); + } + } else { + return Status::Invalid(fmt::format("Field {} is not found in schema.", field)); } - } else { - return Status::Invalid(fmt::format("Field {} is not found in schema.", field)); } - } - std::vector row_groups = arrow::internal::Iota(reader_->GetNumberOfRowGroups()); - if (predicate) { - PAIMON_ASSIGN_OR_RAISE(row_groups, - FilterRowGroupsByPredicate(predicate, file_schema, row_groups)); - } - if (selection_bitmap) { - PAIMON_ASSIGN_OR_RAISE(row_groups, - FilterRowGroupsByBitmap(selection_bitmap.value(), row_groups)); - } + // Build column name to index map for page-level filtering. + // For leaf columns, indices[0] is the correct leaf column index in Parquet. + // For nested types (struct/list/map), FlattenSchema produces multiple leaf indices, + // but predicate pushdown only targets leaf columns with simple types, so indices[0] + // is always the correct single leaf index for predicate evaluation. + std::map column_name_to_index; + for (const auto& [name, indices] : field_index_map) { + if (!indices.empty()) { + column_name_to_index[name] = indices[0]; + } + } - read_data_type_ = arrow::struct_(read_schema->fields()); - read_row_groups_ = row_groups; - read_column_indices_ = column_indices; + std::vector row_groups = arrow::internal::Iota(reader_->GetNumberOfRowGroups()); + if (predicate) { + PAIMON_ASSIGN_OR_RAISE(row_groups, + FilterRowGroupsByPredicate(predicate, file_schema, row_groups)); + } + if (selection_bitmap) { + PAIMON_ASSIGN_OR_RAISE(row_groups, + FilterRowGroupsByBitmap(selection_bitmap.value(), row_groups)); + } + // Apply page-level filtering after bitmap pruning so we don't read page index + // pages for row groups that the bitmap already excluded. + if (predicate && !row_groups.empty()) { + PAIMON_ASSIGN_OR_RAISE( + bool enable_page_index_filter, + OptionsUtils::GetValueFromMap(options_, PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, + DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER)); + if (enable_page_index_filter) { + PAIMON_ASSIGN_OR_RAISE( + auto page_filter_result, + FilterRowGroupsByPageIndex(predicate, column_name_to_index, row_groups)); + row_groups = std::move(page_filter_result.first); + reader_->SetRowGroupRowRanges(page_filter_result.second); + } + } + + read_data_type_ = arrow::struct_(read_schema->fields()); + read_row_groups_ = row_groups; + read_column_indices_ = column_indices; - metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_TOTAL, reader_->GetNumberOfRowGroups()); - metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER, row_groups.size()); + metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_TOTAL, + reader_->GetNumberOfRowGroups()); + metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER, row_groups.size()); - PAIMON_ASSIGN_OR_RAISE(std::set ordered_row_groups, - reader_->FilterRowGroupsByReadRanges(read_ranges_, read_row_groups_)); - return reader_->PrepareForReadingLazy(ordered_row_groups, read_column_indices_); + PAIMON_ASSIGN_OR_RAISE( + std::set ordered_row_groups, + reader_->FilterRowGroupsByReadRanges(read_ranges_, read_row_groups_)); + + // When predicate or selection is applied, prepare eagerly so PreBuffer I/O + // starts immediately. All file readers are created before consumption begins, + // so eager preparation allows I/O for multiple files to overlap. + Status ret; + if (predicate || selection_bitmap) { + ret = reader_->PrepareForReading(ordered_row_groups, read_column_indices_); + } else { + ret = reader_->PrepareForReadingLazy(ordered_row_groups, read_column_indices_); + } + return ret; + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::SetReadSchema") } Result> ParquetFileBatchReader::FilterRowGroupsByPredicate( @@ -226,42 +288,100 @@ Result> ParquetFileBatchReader::FilterRowGroupsByBitmap( return target_row_groups; } -Result ParquetFileBatchReader::NextBatch() { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, reader_->Next()); - if (batch == nullptr) { - return BatchReader::MakeEofBatch(); +// Uses page-level column index statistics to filter row groups and store per-row-group +// RowRanges for true page-level skipping. A row group is excluded if ALL its pages are +// determined to not match the predicate. For partially matched row groups, RowRanges +// are stored for page-level filtering during reading. +Result, std::map>> +ParquetFileBatchReader::FilterRowGroupsByPageIndex( + const std::shared_ptr& predicate, + const std::map& column_name_to_index, + const std::vector& src_row_groups) { + std::map rg_row_ranges; + + if (!predicate) { + return std::make_pair(src_row_groups, rg_row_ranges); } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, batch->ToStructArray()); - PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - array->type(), read_data_type_)); - if (need_cast) { - PAIMON_ASSIGN_OR_RAISE(array, ParquetTimestampConverter::CastArrayForTimestamp( - array, read_data_type_, arrow_pool_)); + + auto page_index_reader = reader_->GetPageIndexReader(); + if (!page_index_reader) { + PAIMON_LOG_DEBUG(logger_, + "Page index not available in file, skipping page-level filtering (%s)", + PARQUET_WRITE_ENABLE_PAGE_INDEX); + return std::make_pair(src_row_groups, rg_row_ranges); } - PAIMON_ASSIGN_OR_RAISE(need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( - array->type(), read_data_type_)); - if (need_cast) { - return Status::Invalid( - fmt::format("unexpected: in parquet, after CastArrayForTimestamp, output type {} not " - "equal with read schema {}", - array->type()->ToString(), read_data_type_->ToString())); + + auto file_metadata = reader_->GetFileReader()->parquet_reader()->metadata(); + + std::vector target_row_groups; + target_row_groups.reserve(src_row_groups.size()); + + for (int32_t row_group_idx : src_row_groups) { + auto result = + reader_->CalculateFilteredRowRanges(row_group_idx, predicate, column_name_to_index); + + if (!result.ok()) { + target_row_groups.push_back(row_group_idx); + continue; + } + + const auto& row_ranges = result.value(); + if (!row_ranges.IsEmpty()) { + target_row_groups.push_back(row_group_idx); + + int64_t rg_row_count = file_metadata->RowGroup(row_group_idx)->num_rows(); + if (row_ranges.RowCount() < rg_row_count) { + rg_row_ranges[row_group_idx] = row_ranges; + } + } } - std::unique_ptr c_array = std::make_unique(); - std::unique_ptr c_schema = std::make_unique(); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); - read_rows_ += array->length(); - read_batch_count_++; - metrics_->SetCounter(ParquetMetrics::READ_ROWS, read_rows_); - metrics_->SetCounter(ParquetMetrics::READ_BATCH_COUNT, read_batch_count_); + return std::make_pair(std::move(target_row_groups), std::move(rg_row_ranges)); +} - return make_pair(std::move(c_array), std::move(c_schema)); +Result ParquetFileBatchReader::NextBatch() { + try { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, reader_->Next()); + if (batch == nullptr) { + return BatchReader::MakeEofBatch(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + batch->ToStructArray()); + PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( + array->type(), read_data_type_)); + if (need_cast) { + PAIMON_ASSIGN_OR_RAISE(array, ParquetTimestampConverter::CastArrayForTimestamp( + array, read_data_type_, arrow_pool_)); + } + PAIMON_ASSIGN_OR_RAISE(need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( + array->type(), read_data_type_)); + if (need_cast) { + return Status::Invalid(fmt::format( + "unexpected: in parquet, after CastArrayForTimestamp, output type {} not " + "equal with read schema {}", + array->type()->ToString(), read_data_type_->ToString())); + } + std::unique_ptr c_array = std::make_unique(); + std::unique_ptr c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); + + read_rows_ += array->length(); + read_batch_count_++; + metrics_->SetCounter(ParquetMetrics::READ_ROWS, read_rows_); + metrics_->SetCounter(ParquetMetrics::READ_BATCH_COUNT, read_batch_count_); + + return make_pair(std::move(c_array), std::move(c_schema)); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::NextBatch") } Result>> ParquetFileBatchReader::GenReadRanges( bool* need_prefetch) const { - *need_prefetch = true; - return reader_->GetAllRowGroupRanges(); + try { + *need_prefetch = true; + return reader_->GetAllRowGroupRanges(); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::GenReadRanges") } Result<::parquet::ReaderProperties> ParquetFileBatchReader::CreateReaderProperties( diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 806afbe7..d9dfe91a 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -38,6 +38,8 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/parquet/file_reader_wrapper.h" +#include "paimon/format/parquet/row_ranges.h" +#include "paimon/logging.h" #include "paimon/reader/prefetch_file_batch_reader.h" #include "paimon/result.h" #include "paimon/status.h" @@ -163,6 +165,13 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { Result> FilterRowGroupsByBitmap( const RoaringBitmap32& bitmap, const std::vector& src_row_groups) const; + // Apply page-level filtering using column index. + // Returns (filtered row groups, per-row-group RowRanges for partial matches). + Result, std::map>> + FilterRowGroupsByPageIndex(const std::shared_ptr& predicate, + const std::map& column_name_to_index, + const std::vector& src_row_groups); + private: std::map options_; // hold the lifecycle of arrow memory pool. @@ -175,6 +184,7 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { std::vector> read_ranges_; std::shared_ptr metrics_; + std::unique_ptr logger_; uint64_t read_rows_ = 0; uint64_t read_batch_count_ = 0; diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h index a1ae3473..90cd716a 100644 --- a/src/paimon/format/parquet/parquet_format_defs.h +++ b/src/paimon/format/parquet/parquet_format_defs.h @@ -20,6 +20,7 @@ #include #include + namespace paimon::parquet { // write @@ -39,6 +40,10 @@ static inline const char PARQUET_COMPRESSION_CODEC_BROTLI_LEVEL[] = "compression static inline const char PARQUET_WRITER_MAX_MEMORY_USE[] = "parquet.writer.max.memory.use"; static constexpr uint64_t DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE = 512 * 1024 * 1024; // 512MB +// Enable writing page index (ColumnIndex + OffsetIndex) for page-level filtering on read +static inline const char PARQUET_WRITE_ENABLE_PAGE_INDEX[] = "parquet.write.enable-page-index"; +static constexpr bool DEFAULT_PARQUET_WRITE_ENABLE_PAGE_INDEX = true; + // read static inline const char PARQUET_READ_EXECUTOR_THREAD_COUNT[] = "parquet.read.executor.thread-count"; @@ -54,12 +59,17 @@ static inline const char PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT[] = static inline const char PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT[] = "parquet.read.predicate-node-count-limit"; +// Enable page-level filtering using column index +static inline const char PARQUET_READ_ENABLE_PAGE_INDEX_FILTER[] = + "parquet.read.enable-page-index-filter"; + // Default is true. Compaction will set to false to reduce memory consumption. static inline const char PARQUET_READ_ENABLE_PRE_BUFFER[] = "parquet.read.enable-pre-buffer"; static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT = 0; static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT = 32 * 1024 * 1024; static constexpr uint32_t DEFAULT_PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT = 512; +static constexpr bool DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER = true; class ParquetMetrics { public: diff --git a/src/paimon/format/parquet/parquet_writer_builder.cpp b/src/paimon/format/parquet/parquet_writer_builder.cpp index 7fa18995..5946dd70 100644 --- a/src/paimon/format/parquet/parquet_writer_builder.cpp +++ b/src/paimon/format/parquet/parquet_writer_builder.cpp @@ -101,6 +101,15 @@ Result> ParquetWriterBuilder::Prepa PAIMON_ASSIGN_OR_RAISE(::parquet::ParquetVersion::type version, ConvertWriterVersion(writer_version)); builder.version(version); + + // Enable writing page index (ColumnIndex + OffsetIndex) for page-level filtering + PAIMON_ASSIGN_OR_RAISE(bool enable_page_index, OptionsUtils::GetValueFromMap( + options_, PARQUET_WRITE_ENABLE_PAGE_INDEX, + DEFAULT_PARQUET_WRITE_ENABLE_PAGE_INDEX)); + if (enable_page_index) { + builder.enable_write_page_index(); + } + return builder.build(); } diff --git a/src/paimon/format/parquet/row_ranges.cpp b/src/paimon/format/parquet/row_ranges.cpp new file mode 100644 index 00000000..2bf469e1 --- /dev/null +++ b/src/paimon/format/parquet/row_ranges.cpp @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/format/parquet/row_ranges.h" + +#include +#include + +namespace paimon::parquet { + +namespace { + +// Returns the union of the two ranges or nullopt if there are elements between them. +// Used by Add to splice an inserted range into the existing sorted-disjoint sequence. +std::optional UnionRanges(const RowRanges::Range& left, + const RowRanges::Range& right) { + if (left.from <= right.from) { + if (left.to + 1 >= right.from) { + return RowRanges::Range(left.from, std::max(left.to, right.to)); + } + } else if (right.to + 1 >= left.from) { + return RowRanges::Range(right.from, std::max(left.to, right.to)); + } + return std::nullopt; +} + +} // namespace + +RowRanges RowRanges::Union(const RowRanges& left, const RowRanges& right) { + std::vector combined; + combined.reserve(left.ranges_.size() + right.ranges_.size()); + combined.insert(combined.end(), left.ranges_.begin(), left.ranges_.end()); + combined.insert(combined.end(), right.ranges_.begin(), right.ranges_.end()); + return RowRanges(Range::SortAndMergeOverlap(combined, /*adjacent=*/true)); +} + +RowRanges RowRanges::Intersection(const RowRanges& left, const RowRanges& right) { + return RowRanges(Range::And(left.ranges_, right.ranges_)); +} + +int64_t RowRanges::RowCount() const { + int64_t count = 0; + for (const auto& range : ranges_) { + count += range.Count(); + } + return count; +} + +bool RowRanges::IsOverlapping(int64_t from, int64_t to) const { + Range target(from, to); + auto it = std::lower_bound(ranges_.begin(), ranges_.end(), target, + [](const Range& r, const Range& t) { return r.to < t.from; }); + return it != ranges_.end() && it->from <= target.to; +} + +void RowRanges::Add(const Range& range) { + if (ranges_.empty()) { + ranges_.push_back(range); + return; + } + + // Find insertion point using binary search (sorted by 'from') + auto pos = + std::lower_bound(ranges_.begin(), ranges_.end(), range, + [](const Range& r, const Range& target) { return r.from < target.from; }); + + // Scan backward and forward to find all ranges that overlap or are adjacent + Range merged = range; + auto merge_begin = pos; + auto merge_end = pos; + + // Merge with preceding ranges + while (merge_begin != ranges_.begin()) { + auto prev = merge_begin - 1; + auto u = UnionRanges(*prev, merged); + if (!u.has_value()) break; + merged = u.value(); + merge_begin = prev; + } + + // Merge with following ranges + while (merge_end != ranges_.end()) { + auto u = UnionRanges(*merge_end, merged); + if (!u.has_value()) break; + merged = u.value(); + ++merge_end; + } + + // Replace [merge_begin, merge_end) with the single merged range + auto it = ranges_.erase(merge_begin, merge_end); + ranges_.insert(it, merged); +} + +std::optional RowRanges::MapFilteredIndexToOriginalRow(int64_t filtered_index) const { + int64_t accumulated = 0; + for (const auto& range : ranges_) { + int64_t count = range.Count(); + if (filtered_index < accumulated + count) { + return range.from + (filtered_index - accumulated); + } + accumulated += count; + } + return std::nullopt; +} + +std::string RowRanges::ToString() const { + if (ranges_.empty()) { + return "[]"; + } + std::string result = "["; + for (size_t i = 0; i < ranges_.size(); ++i) { + if (i > 0) { + result += ", "; + } + result += ranges_[i].ToString(); + } + result += "]"; + return result; +} + +} // namespace paimon::parquet diff --git a/src/paimon/format/parquet/row_ranges.h b/src/paimon/format/parquet/row_ranges.h new file mode 100644 index 00000000..956622d3 --- /dev/null +++ b/src/paimon/format/parquet/row_ranges.h @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/utils/range.h" + +namespace paimon::parquet { + +/// RowRanges represents a set of row ranges in a row group. +/// Each range is defined by [from, to] where both are inclusive. +/// This is used for page-level filtering to skip rows that don't match predicates. +class RowRanges { + public: + /// A single inclusive range. Aliased to paimon::Range so the parquet code shares the + /// common range type and helpers (Intersection, And, SortAndMergeOverlap, ...). + using Range = paimon::Range; + + /// Creates an empty RowRanges. + RowRanges() = default; + + /// Creates a RowRanges with a single range [from, to]. + explicit RowRanges(const Range& range) : ranges_({range}) {} + + /// Creates a RowRanges from a list of ranges. + explicit RowRanges(const std::vector& ranges) : ranges_(ranges) {} + + /// Creates a RowRanges with a single range [0, row_count - 1]. + static RowRanges CreateSingle(int64_t row_count) { + if (row_count <= 0) { + return RowRanges(); + } + return RowRanges(Range(0, row_count - 1)); + } + + /// Creates an empty RowRanges. + static RowRanges CreateEmpty() { + return RowRanges(); + } + + /// Calculates the union of two RowRanges. + /// The union contains all row indexes that were contained in either of the inputs. + static RowRanges Union(const RowRanges& left, const RowRanges& right); + + /// Calculates the intersection of two RowRanges. + /// The intersection contains all row indexes that were contained in both inputs. + static RowRanges Intersection(const RowRanges& left, const RowRanges& right); + + /// Returns the number of rows in the ranges. + int64_t RowCount() const; + + /// Returns the ranges. + const std::vector& GetRanges() const { + return ranges_; + } + + /// Returns true if there are no ranges. + bool IsEmpty() const { + return ranges_.empty(); + } + + /// Returns true if the specified range overlaps with any of the ranges. + bool IsOverlapping(int64_t from, int64_t to) const; + + /// Returns true if the specified row is contained in any of the ranges. + bool Contains(int64_t row) const { + return IsOverlapping(row, row); + } + + /// Adds a range to the end of the list, maintaining sorted disjoint ranges. + void Add(const Range& range); + + /// Maps a filtered-result index to the original row index within the row group. + /// For example, if RowRanges = {[10,19], [50,59]}, then: + /// MapFilteredIndexToOriginalRow(0) = 10 (first row of first range) + /// MapFilteredIndexToOriginalRow(9) = 19 (last row of first range) + /// MapFilteredIndexToOriginalRow(10) = 50 (first row of second range) + /// Returns nullopt if filtered_index is out of bounds. + std::optional MapFilteredIndexToOriginalRow(int64_t filtered_index) const; + + std::string ToString() const; + + private: + std::vector ranges_; +}; + +} // namespace paimon::parquet diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 88d81151..a80ebe90 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -520,7 +520,7 @@ TEST_F(AppendCompactionInteTest, TestAppendTableCompactionWithIOException) { bool compaction_run_complete = false; auto io_hook = IOHook::GetInstance(); - for (size_t i = 0; i < 600; ++i) { + for (size_t i = 0; i < 2000; ++i) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index ad4c66fe..2c2c9f31 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -53,6 +53,7 @@ #include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/plan.h" +#include "paimon/table/source/startup_mode.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/io_exception_helper.h" diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 3f9fc624..8aa07faf 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -25,6 +25,8 @@ #include #include +#include "arrow/api.h" +#include "arrow/ipc/json_simple.h" #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/common/utils/date_time_utils.h" @@ -32,9 +34,17 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" #include "paimon/result.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/startup_mode.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -864,6 +874,229 @@ std::vector> GetTestValuesForWriteAndReadInt return values; } +/// End-to-end test for parquet page-level filtering with a PK table. +/// Writes data with page index enabled and small page size so multiple pages are created, +/// then reads with a PK equality predicate and verifies only matching rows are returned. +TEST_P(WriteAndReadInteTest, TestPKWithParquetPageIndexFilter) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" || file_system != "local") { + return; + } + + auto test_dir = UniqueTestDirectory::Create("local"); + arrow::FieldVector fields = { + arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::utf8()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, "local"}, + // Force exactly one row per parquet page. Parquet's writer checks the page + // byte threshold only after every `write_batch_size` values, so the default + // batch=1024 packs all rows into a single page regardless of page.size. + // write.batch-size=1 + page.size=1 + no dictionary together guarantee that + // every value triggers a page flush, giving ColumnIndexFilter pages whose + // min == max == that row's value. With predicate f0="Alice", exactly one + // page survives page pruning, so the reader emits exactly one row -- and + // that result is attributable purely to page filtering (no row-level + // filter is enabled below). + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir->Str(), schema, /*partition_keys=*/{"f1"}, + /*primary_keys=*/{"f0", "f1"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = test_dir->Str() + "/foo.db/bar"; + int64_t commit_identifier = 0; + + // Write data: 12 rows across 2 partitions + std::string data_p1 = R"([ + ["Alice", "p1", 10, 1.1], + ["Bob", "p1", 20, 2.2], + ["Cathy", "p1", 30, 3.3], + ["David", "p1", 40, 4.4], + ["Emily", "p1", 50, 5.5], + ["Frank", "p1", 60, 6.6] + ])"; + std::string data_p2 = R"([ + ["Grace", "p2", 70, 7.7], + ["Helen", "p2", 80, 8.8], + ["Ivan", "p2", 90, 9.9], + ["Jack", "p2", 100, 10.1], + ["Kate", "p2", 110, 11.2], + ["Lucy", "p2", 120, 12.3] + ])"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_p1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_p1, + /*partition_map=*/{{"f1", "p1"}}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_p2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_p2, + /*partition_map=*/{{"f1", "p2"}}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, + helper->WriteAndCommit(std::move(batch_p1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, + helper->WriteAndCommit(std::move(batch_p2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Scan with PK predicate: f0 = "Alice" + std::string literal_str = "Alice"; + auto predicate = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 2); + ASSERT_FALSE(result_plan->Splits().empty()); + + // Read with predicate but WITHOUT EnablePredicateFilter -- so any narrowing + // of the result is attributable to split/file/RG/page pruning, not to a + // post-read row-level filter. This is what makes the exact assertion below + // meaningful as a check that page-index filtering is wired and working. + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + // Expected: p2 file is pruned by file-level min/max key stats (f0 range + // [Grace, Lucy] doesn't overlap "Alice"). Inside p1's file, write.batch-size=1 + // + page.size=1 produces one row per page, so page-index filter keeps only + // the page whose min == max == "Alice" -- one row. + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_data_type = arrow::struct_(fields_with_row_kind); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_data_type, R"([ +[0, "Alice", "p1", 10, 1.1] +])") + .ValueOrDie()); + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); +} + +/// End-to-end test for parquet page-level filtering on an append-only table. +/// Append-only tables read parquet files directly without PK merge, so the result +/// reflects exactly what survives row-group and page-index pruning. +TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" || file_system != "local") { + return; + } + + auto test_dir = UniqueTestDirectory::Create("local"); + arrow::FieldVector fields = { + arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::utf8()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, "local"}, + // Force exactly one row per parquet page (see the PK variant for why these + // three options together are required). With one row per page, + // ColumnIndexFilter keeps only the page whose min == max == "Alice", and + // without row-level filter the reader output is precisely that one row. + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir->Str(), schema, /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + std::string table_path = test_dir->Str() + "/foo.db/bar"; + int64_t commit_identifier = 0; + + // Write data: 12 rows across 2 partitions. + std::string data_p1 = R"([ + ["Alice", "p1", 10, 1.1], + ["Bob", "p1", 20, 2.2], + ["Cathy", "p1", 30, 3.3], + ["David", "p1", 40, 4.4], + ["Emily", "p1", 50, 5.5], + ["Frank", "p1", 60, 6.6] + ])"; + std::string data_p2 = R"([ + ["Grace", "p2", 70, 7.7], + ["Helen", "p2", 80, 8.8], + ["Ivan", "p2", 90, 9.9], + ["Jack", "p2", 100, 10.1], + ["Kate", "p2", 110, 11.2], + ["Lucy", "p2", 120, 12.3] + ])"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_p1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_p1, + /*partition_map=*/{{"f1", "p1"}}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_p2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data_p2, + /*partition_map=*/{{"f1", "p2"}}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, + helper->WriteAndCommit(std::move(batch_p1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, + helper->WriteAndCommit(std::move(batch_p2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Predicate: f0 = "Alice" + std::string literal_str = "Alice"; + auto predicate = PredicateBuilder::Equal( + /*field_index=*/0, /*field_name=*/"f0", FieldType::STRING, + Literal(FieldType::STRING, literal_str.data(), literal_str.size())); + + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 2); + ASSERT_FALSE(result_plan->Splits().empty()); + + // Read with predicate but WITHOUT EnablePredicateFilter, so the narrowing + // observed below is attributable to page-index filtering rather than a + // post-read row-level filter. + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + // Partition p2's row groups don't overlap "Alice" (min/max f0 in [Grace, Lucy]), + // so the whole file is skipped. Within p1, page-index pruning narrows down to the + // page containing "Alice". With no PK merge, the result is exactly that one row. + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_data_type = arrow::struct_(fields_with_row_kind); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_data_type, R"([ +[0, "Alice", "p1", 10, 1.1] +])") + .ValueOrDie()); + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); +} + INSTANTIATE_TEST_SUITE_P(FileFormatAndFileSystem, WriteAndReadInteTest, ::testing::ValuesIn(GetTestValuesForWriteAndReadInteTest())); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 62c61b7b..eef6d6e4 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -1876,6 +1876,7 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) { } TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { + auto file_format = GetParam(); ::testing::GTEST_FLAG(throw_on_failure) = true; // create table arrow::FieldVector fields = { @@ -1884,7 +1885,6 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { auto schema = arrow::schema(fields); std::vector primary_keys = {"f0", "f1"}; std::vector partition_keys = {"f1"}; - auto file_format = GetParam(); std::map options = { {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "2"}, @@ -1893,7 +1893,11 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { bool run_complete = false; auto io_hook = IOHook::GetInstance(); - for (size_t i = 0; i < 500; i++) { + // Loop bound must exceed the workflow's total IO operations so the loop can + // naturally terminate at the iteration where injection position falls past + // the last IO. Measured IO counts: orc=310, parquet=506, avro=195, lance=69. + // 1000 leaves headroom for future format/workflow changes. + for (size_t i = 0; i < 1000; i++) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); From 3f02435f2540fc9f21e072d47c949c8a91a43326 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Thu, 28 May 2026 18:12:08 +0800 Subject: [PATCH 010/138] fix: add ScopeGuard to wait async tasks before early return in OrphanFilesCleanerImpl::Clean() From ed1787739d1e4c20e8039cf6753ff851c8f33ccf Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Fri, 29 May 2026 10:36:31 +0800 Subject: [PATCH 011/138] feat(lumina): support null values in vector column during index building From 8372606c4345fed1b05a3fa20db977876d0c2041 Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Fri, 29 May 2026 11:35:30 +0800 Subject: [PATCH 012/138] chore: modify comment in RecordBatch & add check in GenericRow From 98c87aadc43e2877a629e6b153d014cc0d96dcde Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Fri, 29 May 2026 13:40:37 +0800 Subject: [PATCH 013/138] fix: fix clang-tidy issues --- src/paimon/common/memory/memory_segment_utils.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/paimon/common/memory/memory_segment_utils.cpp b/src/paimon/common/memory/memory_segment_utils.cpp index 15c05455..7f56792d 100644 --- a/src/paimon/common/memory/memory_segment_utils.cpp +++ b/src/paimon/common/memory/memory_segment_utils.cpp @@ -240,8 +240,7 @@ bool MemorySegmentUtils::EqualsMultiSegments(const std::vector& s int32_t seg_offset2 = offset2 - seg_size2 * seg_index2; // equal to % while (len > 0) { - int32_t equal_len = - std::min(std::min(len, seg_size1 - seg_offset1), seg_size2 - seg_offset2); + int32_t equal_len = std::min({len, seg_size1 - seg_offset1, seg_size2 - seg_offset2}); if (!segments1[seg_index1].EqualTo(segments2[seg_index2], seg_offset1, seg_offset2, equal_len)) { return false; From 48d55dd921d8f659a12871034f312be8a6c7a89d Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Fri, 29 May 2026 14:33:25 +0800 Subject: [PATCH 014/138] chore: add check scale in BinaryRowWriter From 2a4aa2b0dec3e5ae9df9040030601eb58c33a0a5 Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Fri, 29 May 2026 16:00:41 +0800 Subject: [PATCH 015/138] fix: fix ub in Blob From 6f90168b62028584cfc39cb1fc04297249946f04 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Fri, 29 May 2026 16:45:50 +0800 Subject: [PATCH 016/138] refact: Refactor Literal operator== to delegate to CompareTo for consistent equality semantics From 4cb047c2c18f077cd66fcfe667151f015f31cf4b Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:50:38 +0800 Subject: [PATCH 017/138] refactor: Reuse RowGroupPageIndexReader across columns to improve page-level predicate pushdown performance --- .../page_filtered_row_group_reader.cpp | 25 +++++++++++-------- .../parquet/page_filtered_row_group_reader.h | 2 +- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 5f43c035..d44c11b4 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -141,9 +141,10 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa Result> PageFilteredRowGroupReader::ReadFilteredColumn( const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader, ::parquet::ParquetFileReader* parquet_reader, - const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, int32_t row_group_index, - int32_t column_index, const RowRanges& row_ranges, const std::shared_ptr& field, - int64_t row_group_row_count, ::arrow::MemoryPool* pool) { + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, + int32_t row_group_index, int32_t column_index, const RowRanges& row_ranges, + const std::shared_ptr& field, int64_t row_group_row_count, + ::arrow::MemoryPool* pool) { auto file_metadata = parquet_reader->metadata(); const auto* col_descriptor = file_metadata->schema()->Column(column_index); @@ -152,11 +153,8 @@ Result> PageFilteredRowGroupReader::ReadFil int64_t effective_row_count = row_group_row_count; std::shared_ptr<::parquet::OffsetIndex> offset_index; - if (page_index_reader) { - auto rg_page_index_reader = page_index_reader->RowGroup(row_group_index); - if (rg_page_index_reader) { - offset_index = rg_page_index_reader->GetOffsetIndex(column_index); - } + if (rg_page_index_reader) { + offset_index = rg_page_index_reader->GetOffsetIndex(column_index); } auto page_reader = row_group_reader->GetColumnPageReader(column_index); @@ -266,6 +264,13 @@ Result> PageFilteredRowGroupReader::Re int64_t row_group_row_count = rg_metadata->num_rows(); auto page_index_reader = parquet_reader->GetPageIndexReader(); + // reuse RowGroupPageIndexReader for multiple columns in the same row group to avoid redundant + // metadata reads + std::shared_ptr<::parquet::RowGroupPageIndexReader> rg_page_index_reader; + if (page_index_reader) { + rg_page_index_reader = page_index_reader->RowGroup(row_group_index); + } + // Read each column with page filtering std::vector> columns; columns.reserve(column_indices.size()); @@ -273,8 +278,8 @@ Result> PageFilteredRowGroupReader::Re for (size_t i = 0; i < column_indices.size(); ++i) { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr chunked_array, - ReadFilteredColumn(row_group_reader, parquet_reader, page_index_reader, row_group_index, - column_indices[i], row_ranges, + ReadFilteredColumn(row_group_reader, parquet_reader, rg_page_index_reader, + row_group_index, column_indices[i], row_ranges, arrow_schema->field(static_cast(i)), row_group_row_count, pool)); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index b3323ce4..f2a06c50 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -93,7 +93,7 @@ class PageFilteredRowGroupReader { static Result> ReadFilteredColumn( const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader, ::parquet::ParquetFileReader* parquet_reader, - const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader, + const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, int32_t row_group_index, int32_t column_index, const RowRanges& row_ranges, const std::shared_ptr& field, int64_t row_group_row_count, ::arrow::MemoryPool* pool); From 3fc019aa42dcf49ef0377741c3d98cf0fb8692c7 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:04:30 +0800 Subject: [PATCH 018/138] feat(blob): Support blob-descriptor-field for inline blob descriptor storage --- include/paimon/data/blob.h | 3 +- include/paimon/defs.h | 2 +- src/paimon/CMakeLists.txt | 3 + src/paimon/common/data/blob.cpp | 5 +- src/paimon/common/data/blob_descriptor.h | 2 +- src/paimon/common/data/blob_test.cpp | 14 +- src/paimon/common/data/blob_utils.cpp | 110 +- src/paimon/common/data/blob_utils.h | 36 +- src/paimon/common/data/blob_utils_test.cpp | 234 +++- src/paimon/core/append/append_only_writer.cpp | 83 +- src/paimon/core/append/append_only_writer.h | 9 +- .../casting/binary_to_blob_cast_executor.cpp | 84 ++ .../casting/binary_to_blob_cast_executor.h | 45 + .../core/casting/cast_executor_factory.cpp | 3 + .../casting/cast_executor_factory_test.cpp | 8 + .../core/casting/cast_executor_test.cpp | 38 + src/paimon/core/io/data_file_path_factory.h | 6 + .../core/io/data_file_path_factory_test.cpp | 15 + .../core/io/external_storage_blob_writer.cpp | 231 ++++ .../core/io/external_storage_blob_writer.h | 114 ++ .../io/external_storage_blob_writer_test.cpp | 153 ++ .../core/io/field_mapping_reader_test.cpp | 31 + .../core/io/rolling_blob_file_writer.cpp | 5 +- src/paimon/core/io/rolling_blob_file_writer.h | 5 +- .../core/operation/abstract_split_read.cpp | 9 +- .../append_only_file_store_write.cpp | 8 +- .../operation/append_only_file_store_write.h | 1 - src/paimon/core/operation/file_store_scan.cpp | 7 +- src/paimon/core/schema/schema_validation.cpp | 8 +- src/paimon/core/utils/field_mapping.cpp | 4 +- src/paimon/core/utils/field_mapping.h | 2 - .../format/avro/avro_direct_encoder.cpp | 9 +- .../avro/avro_file_batch_reader_test.cpp | 50 + .../format/avro/avro_schema_converter.cpp | 1 + .../format/avro/avro_stats_extractor.cpp | 1 + .../blob/blob_file_batch_reader_test.cpp | 5 +- src/paimon/format/blob/blob_format_writer.cpp | 51 +- src/paimon/format/blob/blob_format_writer.h | 18 +- .../format/blob/blob_format_writer_test.cpp | 107 +- src/paimon/format/blob/blob_writer_builder.h | 14 +- .../format/blob/blob_writer_builder_test.cpp | 37 + src/paimon/format/orc/orc_adapter.cpp | 4 + src/paimon/format/orc/orc_adapter_test.cpp | 44 +- .../format/orc/orc_file_batch_reader_test.cpp | 44 + .../parquet_file_batch_reader_test.cpp | 42 + src/paimon/testing/utils/test_helper.h | 68 - test/inte/blob_table_inte_test.cpp | 1232 ++++++++++++++--- 47 files changed, 2586 insertions(+), 419 deletions(-) create mode 100644 src/paimon/core/casting/binary_to_blob_cast_executor.cpp create mode 100644 src/paimon/core/casting/binary_to_blob_cast_executor.h create mode 100644 src/paimon/core/io/external_storage_blob_writer.cpp create mode 100644 src/paimon/core/io/external_storage_blob_writer.h create mode 100644 src/paimon/core/io/external_storage_blob_writer_test.cpp diff --git a/include/paimon/data/blob.h b/include/paimon/data/blob.h index ff89714f..0b66ce7b 100644 --- a/include/paimon/data/blob.h +++ b/include/paimon/data/blob.h @@ -100,7 +100,8 @@ class PAIMON_EXPORT Blob { /// @param metadata A map of key-value metadata to be attached to the field. /// @return A result containing a unique pointer to the generated `ArrowSchema` or an error. static Result> ArrowField( - const std::string& field_name, std::unordered_map metadata = {}); + const std::string& field_name, bool nullable = false, + std::unordered_map metadata = {}); private: class Impl; diff --git a/include/paimon/defs.h b/include/paimon/defs.h index c38cb492..17ec196d 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -369,7 +369,7 @@ struct PAIMON_EXPORT Options { /// "partition.legacy-name" - The legacy partition name is using `ToString` for all types. If /// false, using casting to string for all types. Default value is "true". static const char PARTITION_GENERATE_LEGACY_NAME[]; - /// "blob-as-descriptor" - Read and write blob field using blob descriptor rather than blob + /// "blob-as-descriptor" - Read blob field using blob descriptor rather than blob /// bytes. Default value is "false". static const char BLOB_AS_DESCRIPTOR[]; /// "blob-field" - Specifies column names that should be stored as blob type. This is used diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index c550226c..b1702a9b 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -161,6 +161,7 @@ set(PAIMON_CORE_SRCS core/bucket/hive_bucket_function.cpp core/bucket/mod_bucket_function.cpp core/bucket/bucket_id_calculator.cpp + core/casting/binary_to_blob_cast_executor.cpp core/casting/binary_to_string_cast_executor.cpp core/casting/boolean_to_decimal_cast_executor.cpp core/casting/boolean_to_numeric_cast_executor.cpp @@ -224,6 +225,7 @@ set(PAIMON_CORE_SRCS core/io/key_value_meta_projection_consumer.cpp core/io/key_value_projection_consumer.cpp core/io/key_value_projection_reader.cpp + core/io/external_storage_blob_writer.cpp core/io/multiple_blob_file_writer.cpp core/io/rolling_blob_file_writer.cpp core/manifest/file_kind.cpp @@ -603,6 +605,7 @@ if(PAIMON_BUILD_TESTS) core/io/file_index_evaluator_test.cpp core/io/single_file_writer_test.cpp core/io/rolling_blob_file_writer_test.cpp + core/io/external_storage_blob_writer_test.cpp core/global_index/indexed_split_test.cpp core/manifest/file_source_test.cpp core/manifest/file_kind_test.cpp diff --git a/src/paimon/common/data/blob.cpp b/src/paimon/common/data/blob.cpp index 66953273..da47a920 100644 --- a/src/paimon/common/data/blob.cpp +++ b/src/paimon/common/data/blob.cpp @@ -108,8 +108,9 @@ Result> Blob::ToData(const std::shared_ptr& } Result> Blob::ArrowField( - const std::string& field_name, std::unordered_map metadata) { - auto blob_field = BlobUtils::ToArrowField(field_name, /*nullable=*/false, metadata); + const std::string& field_name, bool nullable, + std::unordered_map metadata) { + auto blob_field = BlobUtils::ToArrowField(field_name, nullable, metadata); auto field = std::make_unique<::ArrowSchema>(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportField(*blob_field, field.get())); return field; diff --git a/src/paimon/common/data/blob_descriptor.h b/src/paimon/common/data/blob_descriptor.h index 6664b85e..ea41c97d 100644 --- a/src/paimon/common/data/blob_descriptor.h +++ b/src/paimon/common/data/blob_descriptor.h @@ -41,7 +41,7 @@ namespace paimon { /// | 13 + N | offset | long | 8 | /// | 21 + N | length | long | 8 | -class BlobDescriptor { +class PAIMON_EXPORT BlobDescriptor { public: static Result> Create(const std::string& uri, int64_t offset, int64_t length); diff --git a/src/paimon/common/data/blob_test.cpp b/src/paimon/common/data/blob_test.cpp index 4c95711c..dbb2401e 100644 --- a/src/paimon/common/data/blob_test.cpp +++ b/src/paimon/common/data/blob_test.cpp @@ -147,38 +147,34 @@ TEST_F(BlobTest, TestNewInputStreamWithDynamicLength) { } TEST_F(BlobTest, TestArrowField) { - { - // basic: field name, non-nullable by default - ASSERT_OK_AND_ASSIGN(auto schema, Blob::ArrowField("my_blob")); + for (bool nullable : {false, true}) { + ASSERT_OK_AND_ASSIGN(auto schema, Blob::ArrowField("my_blob", nullable)); ASSERT_NE(schema, nullptr); - // import back to arrow::Field to verify auto field_result = arrow::ImportField(schema.get()); ASSERT_TRUE(field_result.ok()); auto field = field_result.ValueUnsafe(); ASSERT_EQ(field->name(), "my_blob"); ASSERT_EQ(field->type()->id(), arrow::Type::LARGE_BINARY); - ASSERT_FALSE(field->nullable()); + ASSERT_EQ(field->nullable(), nullable); ASSERT_TRUE(field->HasMetadata()); auto extension_type = field->metadata()->Get("paimon.extension.type"); ASSERT_TRUE(extension_type.ok()); ASSERT_EQ(extension_type.ValueUnsafe(), "paimon.type.blob"); } { - // with custom metadata std::unordered_map custom_metadata = { {"custom_key", "custom_value"}}; - ASSERT_OK_AND_ASSIGN(auto schema, Blob::ArrowField("meta_blob", custom_metadata)); + ASSERT_OK_AND_ASSIGN(auto schema, + Blob::ArrowField("meta_blob", /*nullable=*/false, custom_metadata)); auto field = arrow::ImportField(schema.get()).ValueUnsafe(); ASSERT_EQ(field->name(), "meta_blob"); ASSERT_FALSE(field->nullable()); ASSERT_TRUE(field->HasMetadata()); - // blob extension metadata should be present auto extension_type = field->metadata()->Get("paimon.extension.type"); ASSERT_TRUE(extension_type.ok()); ASSERT_EQ(extension_type.ValueUnsafe(), "paimon.type.blob"); - // custom metadata should also be present auto custom_val = field->metadata()->Get("custom_key"); ASSERT_TRUE(custom_val.ok()); ASSERT_EQ(custom_val.ValueUnsafe(), "custom_value"); diff --git a/src/paimon/common/data/blob_utils.cpp b/src/paimon/common/data/blob_utils.cpp index 9eb716a1..f33c220d 100644 --- a/src/paimon/common/data/blob_utils.cpp +++ b/src/paimon/common/data/blob_utils.cpp @@ -20,65 +20,77 @@ #include "paimon/common/data/blob_utils.h" #include -#include +#include #include #include "arrow/api.h" #include "arrow/array/array_nested.h" #include "arrow/type.h" +#include "fmt/format.h" #include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/string_utils.h" - namespace arrow { class Array; } namespace paimon { - BlobUtils::SeparatedSchemas BlobUtils::SeparateBlobSchema( - const std::shared_ptr& schema) { - std::vector> remaining_fields; + const std::shared_ptr& schema, const std::set& inline_fields) { + std::vector> main_fields; std::vector> blob_fields; - for (auto i = 0; i < schema->num_fields(); i++) { + for (int32_t i = 0; i < schema->num_fields(); i++) { auto field = schema->field(i); - if (IsBlobField(field)) { + if (IsBlobField(field) && inline_fields.count(field->name()) == 0) { + // Non-inline BLOB -> goes to blob file blob_fields.emplace_back(field); } else { - remaining_fields.emplace_back(field); + // Non-blob fields OR inline BLOB fields -> stay in main + main_fields.emplace_back(field); } } SeparatedSchemas result; - result.main_schema = arrow::schema(remaining_fields); + result.main_schema = arrow::schema(main_fields); result.blob_schema = arrow::schema(blob_fields); return result; } Result BlobUtils::SeparateBlobArray( - const std::shared_ptr& struct_array) { + const std::shared_ptr& struct_array, + const std::set& inline_fields) { std::shared_ptr old_type = std::static_pointer_cast(struct_array->type()); const auto& old_fields = old_type->fields(); const auto& old_arrays = struct_array->fields(); - std::vector> remaining_fields; - std::vector> remaining_arrays; - std::vector> blob_fields; - std::vector> blob_arrays; + arrow::ArrayVector main_arrays; + arrow::ArrayVector blob_arrays; + arrow::FieldVector main_fields; + arrow::FieldVector blob_fields; for (size_t i = 0; i < old_fields.size(); i++) { - if (IsBlobField(old_fields[i])) { + if (IsBlobField(old_fields[i]) && inline_fields.count(old_fields[i]->name()) == 0) { blob_fields.push_back(old_fields[i]); blob_arrays.push_back(old_arrays[i]); } else { - remaining_fields.push_back(old_fields[i]); - remaining_arrays.push_back(old_arrays[i]); + main_fields.push_back(old_fields[i]); + main_arrays.push_back(old_arrays[i]); } } + if (blob_fields.empty()) { + return Status::Invalid( + "SeparateBlobArray expects at least one non-inline blob field, but got none."); + } + if (main_fields.empty()) { + return Status::Invalid("SeparateBlobArray expects at least one main field, but got none."); + } + SeparatedStructArrays result; PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(result.main_array, - arrow::StructArray::Make(remaining_arrays, remaining_fields)); + arrow::StructArray::Make(main_arrays, main_fields)); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(result.blob_array, arrow::StructArray::Make(blob_arrays, blob_fields)); return result; @@ -117,4 +129,66 @@ std::shared_ptr BlobUtils::ToArrowField( return arrow::field(field_name, arrow::large_binary(), nullable, std::make_shared(metadata)); } + +Status BlobUtils::ValidateInlineBlobDescriptors( + const std::shared_ptr& struct_array, + const std::set& inline_descriptor_fields) { + if (inline_descriptor_fields.empty()) { + return Status::OK(); + } + if (!struct_array) { + return Status::Invalid("array in ValidateInlineBlobDescriptors must be a struct_array"); + } + for (const auto& field_name : inline_descriptor_fields) { + auto field_array = struct_array->GetFieldByName(field_name); + if (!field_array) { + continue; + } + const auto* binary_array = + arrow::internal::checked_cast(field_array.get()); + if (!binary_array) { + return Status::Invalid( + fmt::format("cannot cast array for field {} to LargeBinaryArray", field_name)); + } + for (int64_t row = 0; row < binary_array->length(); ++row) { + if (binary_array->IsNull(row)) { + continue; + } + auto value = binary_array->GetView(row); + PAIMON_ASSIGN_OR_RAISE(bool is_descriptor, + BlobDescriptor::IsBlobDescriptor(value.data(), value.size())); + if (!is_descriptor) { + return Status::Invalid(fmt::format( + "BLOB inline field {} configured by blob-descriptor-field or blob-view-field " + "require values to be a BlobDescriptor or BlobViewStruct.", + field_name)); + } + } + } + return Status::OK(); +} + +std::vector BlobUtils::ConvertBlobInlineDataFields( + const std::vector& data_fields, const std::vector& blob_inline_fields) { + if (blob_inline_fields.empty()) { + return data_fields; + } + + std::set blob_inline_field_set(blob_inline_fields.begin(), + blob_inline_fields.end()); + std::vector converted_fields; + converted_fields.reserve(data_fields.size()); + for (const auto& data_field : data_fields) { + if (blob_inline_field_set.find(data_field.Name()) == blob_inline_field_set.end()) { + converted_fields.push_back(data_field); + continue; + } + + auto binary_field = arrow::field(data_field.Name(), arrow::binary(), data_field.Nullable(), + data_field.ArrowField()->metadata()); + converted_fields.emplace_back(data_field.Id(), binary_field, data_field.Description()); + } + return converted_fields; +} + } // namespace paimon diff --git a/src/paimon/common/data/blob_utils.h b/src/paimon/common/data/blob_utils.h index 505cad57..f9ac0b18 100644 --- a/src/paimon/common/data/blob_utils.h +++ b/src/paimon/common/data/blob_utils.h @@ -20,8 +20,10 @@ #pragma once #include +#include #include #include +#include #include "paimon/result.h" #include "paimon/visibility.h" @@ -33,6 +35,10 @@ class Schema; class StructArray; } // namespace arrow +namespace paimon { +class DataField; +} // namespace paimon + namespace paimon { /// Utils for blob type. class PAIMON_EXPORT BlobUtils { @@ -41,23 +47,29 @@ class PAIMON_EXPORT BlobUtils { ~BlobUtils() = delete; struct SeparatedSchemas { - /// Non-blob fields + /// Non-blob fields (includes inline blob fields when inline_fields is provided) std::shared_ptr main_schema; - /// Blob fields only + /// Blob fields that go to separate .blob files std::shared_ptr blob_schema; }; struct SeparatedStructArrays { - /// Non-blob fields + /// Non-blob fields (includes inline blob fields when inline_fields is provided) std::shared_ptr main_array; - /// Blob fields only + /// Blob fields that go to separate .blob files std::shared_ptr blob_array; }; - static SeparatedSchemas SeparateBlobSchema(const std::shared_ptr& schema); + /// Separates schema with inline field awareness. + /// BLOB fields in inline_fields stay in main_schema; others go to blob_schema. + static SeparatedSchemas SeparateBlobSchema(const std::shared_ptr& schema, + const std::set& inline_fields); + /// Separates array with inline field awareness. + /// BLOB fields in inline_fields stay in main_array; others go to blob_array. static Result SeparateBlobArray( - const std::shared_ptr& struct_array); + const std::shared_ptr& struct_array, + const std::set& inline_fields); static bool IsBlobField(const std::shared_ptr& field); static bool IsBlobMetadata(const std::shared_ptr& metadata); @@ -66,6 +78,18 @@ class PAIMON_EXPORT BlobUtils { static std::shared_ptr ToArrowField( const std::string& field_name, bool nullable = false, std::unordered_map metadata = {}); + + static Status ValidateInlineBlobDescriptors( + const std::shared_ptr& struct_array, + const std::set& inline_descriptor_fields); + + /// Converts inline blob DataFields from large_binary to binary type. + /// Inline blob fields use large_binary in the table schema (because they are BLOB type), + /// but are stored as binary in data files. This conversion aligns the field type with + /// the actual on-disk storage format for correct reading. + static std::vector ConvertBlobInlineDataFields( + const std::vector& data_fields, + const std::vector& blob_inline_fields); }; } // namespace paimon diff --git a/src/paimon/common/data/blob_utils_test.cpp b/src/paimon/common/data/blob_utils_test.cpp index d042ecda..9b4284d6 100644 --- a/src/paimon/common/data/blob_utils_test.cpp +++ b/src/paimon/common/data/blob_utils_test.cpp @@ -23,7 +23,10 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/types/data_field.h" #include "paimon/data/blob.h" +#include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -77,7 +80,8 @@ TEST_F(BlobUtilsTest, SeparateBlobSchema) { std::shared_ptr original_schema = arrow::schema({int_field, string_field, blob_field_1}); - BlobUtils::SeparatedSchemas schemas = BlobUtils::SeparateBlobSchema(original_schema); + BlobUtils::SeparatedSchemas schemas = + BlobUtils::SeparateBlobSchema(original_schema, /*inline_fields=*/{}); std::shared_ptr expected_main_schema = arrow::schema({int_field, string_field}); @@ -88,17 +92,46 @@ TEST_F(BlobUtilsTest, SeparateBlobSchema) { } { std::shared_ptr no_blob_schema = arrow::schema({int_field, string_field}); - BlobUtils::SeparatedSchemas no_blob_schemas = BlobUtils::SeparateBlobSchema(no_blob_schema); + BlobUtils::SeparatedSchemas no_blob_schemas = + BlobUtils::SeparateBlobSchema(no_blob_schema, /*inline_fields=*/{}); ASSERT_TRUE(no_blob_schemas.main_schema->Equals(*no_blob_schema)); ASSERT_EQ(no_blob_schemas.blob_schema->num_fields(), 0); } { std::shared_ptr only_blob_schema = arrow::schema({blob_field_1}); BlobUtils::SeparatedSchemas only_blob_schemas = - BlobUtils::SeparateBlobSchema(only_blob_schema); + BlobUtils::SeparateBlobSchema(only_blob_schema, /*inline_fields=*/{}); ASSERT_TRUE(only_blob_schemas.blob_schema->Equals(*only_blob_schema)); ASSERT_EQ(only_blob_schemas.main_schema->num_fields(), 0); } + { + // Inline blob field stays in main_schema instead of going to blob_schema + auto blob_field_2 = BlobUtils::ToArrowField("f4_blob_2", false); + std::shared_ptr schema = + arrow::schema({int_field, blob_field_1, blob_field_2, string_field}); + + BlobUtils::SeparatedSchemas schemas = + BlobUtils::SeparateBlobSchema(schema, /*inline_fields=*/{"f3_blob_1"}); + + // f3_blob_1 is inline -> stays in main; f4_blob_2 goes to blob + std::shared_ptr expected_main = + arrow::schema({int_field, blob_field_1, string_field}); + ASSERT_TRUE(schemas.main_schema->Equals(*expected_main)); + + std::shared_ptr expected_blob = arrow::schema({blob_field_2}); + ASSERT_TRUE(schemas.blob_schema->Equals(*expected_blob)); + } + { + // All blob fields are inline -> blob_schema is empty + std::shared_ptr schema = + arrow::schema({int_field, blob_field_1, string_field}); + + BlobUtils::SeparatedSchemas schemas = + BlobUtils::SeparateBlobSchema(schema, /*inline_fields=*/{"f3_blob_1"}); + + ASSERT_TRUE(schemas.main_schema->Equals(*schema)); + ASSERT_EQ(schemas.blob_schema->num_fields(), 0); + } } TEST_F(BlobUtilsTest, SeparateBlobArray) { @@ -128,7 +161,8 @@ TEST_F(BlobUtilsTest, SeparateBlobArray) { std::shared_ptr struct_array = std::static_pointer_cast(raw_struct_array); - ASSERT_OK_AND_ASSIGN(auto separated, BlobUtils::SeparateBlobArray(struct_array)); + ASSERT_OK_AND_ASSIGN(auto separated, + BlobUtils::SeparateBlobArray(struct_array, /*inline_fields=*/{})); std::shared_ptr expected_main_type = arrow::struct_({int_field, string_field}); ASSERT_TRUE(separated.main_array->type()->Equals(*expected_main_type)); @@ -140,6 +174,198 @@ TEST_F(BlobUtilsTest, SeparateBlobArray) { ASSERT_TRUE(separated.blob_array->type()->Equals(*expected_blob_type)); ASSERT_EQ(separated.blob_array->num_fields(), 1); ASSERT_TRUE(separated.blob_array->field(0)->Equals(*blob_array_data)); + + // All blob fields are inline -> should return error (no blob field to separate) + ASSERT_NOK_WITH_MSG( + BlobUtils::SeparateBlobArray(struct_array, /*inline_fields=*/{"f2_blob"}), + "SeparateBlobArray expects at least one non-inline blob field, but got none."); + + // All fields are blob with no inline -> no main field -> should return error + auto all_blob_struct = arrow::StructArray::Make({blob_array_data}, {blob_field}).ValueOrDie(); + auto all_blob_sa = std::dynamic_pointer_cast(all_blob_struct); + ASSERT_NOK_WITH_MSG(BlobUtils::SeparateBlobArray(all_blob_sa, /*inline_fields=*/{}), + "SeparateBlobArray expects at least one main field, but got none."); +} + +TEST_F(BlobUtilsTest, SeparateBlobArrayWithPartialInline) { + auto int_field = arrow::field("f1_int", arrow::int32()); + std::shared_ptr blob_field_1 = BlobUtils::ToArrowField("f2_blob_1", false); + std::shared_ptr blob_field_2 = BlobUtils::ToArrowField("f3_blob_2", true); + auto schema = arrow::schema({int_field, blob_field_1, blob_field_2}); + + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.AppendValues({1, 2}).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + + arrow::LargeBinaryBuilder blob_builder_1; + ASSERT_TRUE(blob_builder_1.Append("a", 1).ok()); + ASSERT_TRUE(blob_builder_1.Append("b", 1).ok()); + auto blob_array_1 = blob_builder_1.Finish().ValueOrDie(); + + arrow::LargeBinaryBuilder blob_builder_2; + ASSERT_TRUE(blob_builder_2.Append("x", 1).ok()); + ASSERT_TRUE(blob_builder_2.AppendNull().ok()); + auto blob_array_2 = blob_builder_2.Finish().ValueOrDie(); + + auto raw_struct_array = + arrow::StructArray::Make({int_array, blob_array_1, blob_array_2}, schema->fields()) + .ValueOrDie(); + auto struct_array = std::static_pointer_cast(raw_struct_array); + + // f2_blob_1 is inline, f3_blob_2 goes to blob + ASSERT_OK_AND_ASSIGN(auto separated, BlobUtils::SeparateBlobArray( + struct_array, /*inline_fields=*/{"f2_blob_1"})); + + std::shared_ptr expected_main_type = arrow::struct_({int_field, blob_field_1}); + ASSERT_TRUE(separated.main_array->type()->Equals(*expected_main_type)); + ASSERT_EQ(separated.main_array->num_fields(), 2); + ASSERT_TRUE(separated.main_array->field(0)->Equals(*int_array)); + ASSERT_TRUE(separated.main_array->field(1)->Equals(*blob_array_1)); + + std::shared_ptr expected_blob_type = arrow::struct_({blob_field_2}); + ASSERT_TRUE(separated.blob_array->type()->Equals(*expected_blob_type)); + ASSERT_EQ(separated.blob_array->num_fields(), 1); + ASSERT_TRUE(separated.blob_array->field(0)->Equals(*blob_array_2)); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsEmptyFields) { + // Empty inline_descriptor_fields -> always OK + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append("random_data").ok()); + auto array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateInlineBlobDescriptors(sa, {})); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsFieldNotPresent) { + // Field not in struct_array -> skip, OK + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.Append(42).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({int_array}, {arrow::field("f0", arrow::int32())}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + // "b0" does not exist in the struct -> should pass + ASSERT_OK(BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"})); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithValidDescriptor) { + // Valid BlobDescriptor bytes -> OK + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); + auto serialized = descriptor->Serialize(pool); + + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"})); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithNullValue) { + // Null values in blob column -> skip, OK + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.AppendNull().ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"})); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithRawBytes) { + // Raw bytes (not a descriptor) -> error + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append("not_a_descriptor_just_raw_data").ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG( + BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"}), + "BLOB inline field b0 configured by blob-descriptor-field or blob-view-field " + "require values to be a BlobDescriptor or BlobViewStruct."); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsMixedValidAndInvalid) { + // First row is valid descriptor, second row is raw bytes -> error on row 1 + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); + auto serialized = descriptor->Serialize(pool); + + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); + ASSERT_TRUE(builder.Append("raw_bytes_not_descriptor").ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG( + BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"}), + "BLOB inline field b0 configured by blob-descriptor-field or blob-view-field " + "require values to be a BlobDescriptor or BlobViewStruct."); +} + +TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsMultipleFields) { + // Two inline fields: b0 is valid, b1 has raw bytes -> error on b1 + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); + auto serialized = descriptor->Serialize(pool); + + arrow::LargeBinaryBuilder b0_builder; + ASSERT_TRUE(b0_builder.Append(serialized->data(), serialized->size()).ok()); + auto b0_array = b0_builder.Finish().ValueOrDie(); + + arrow::LargeBinaryBuilder b1_builder; + ASSERT_TRUE(b1_builder.Append("invalid_raw_data").ok()); + auto b1_array = b1_builder.Finish().ValueOrDie(); + + auto struct_array = + arrow::StructArray::Make({b0_array, b1_array}, + {BlobUtils::ToArrowField("b0"), BlobUtils::ToArrowField("b1")}) + .ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG( + BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0", "b1"}), + "BLOB inline field b1 configured by blob-descriptor-field or blob-view-field " + "require values to be a BlobDescriptor or BlobViewStruct."); +} + +TEST_F(BlobUtilsTest, TestConvertBlobInlineDataFields) { + // Schema with a blob field (large_binary with blob metadata) and normal fields. + auto blob_field = BlobUtils::ToArrowField("blob_col", /*nullable=*/true); + std::vector data_fields = {DataField(0, arrow::field("int_col", arrow::int32())), + DataField(1, blob_field), + DataField(2, arrow::field("str_col", arrow::utf8()))}; + + // Without inline fields — blob_col stays as large_binary + { + auto result = BlobUtils::ConvertBlobInlineDataFields(data_fields, {}); + ASSERT_EQ(result.size(), 3); + ASSERT_EQ(result[1].ArrowField()->type()->id(), arrow::Type::LARGE_BINARY); + } + + // With inline fields — blob_col should be converted from large_binary to binary + { + auto result = BlobUtils::ConvertBlobInlineDataFields(data_fields, {"blob_col"}); + ASSERT_EQ(result.size(), 3); + ASSERT_EQ(result[1].ArrowField()->type()->id(), arrow::Type::BINARY); + ASSERT_EQ(result[1].Name(), "blob_col"); + ASSERT_EQ(result[1].Nullable(), true); + // Other fields unchanged + ASSERT_EQ(result[0].ArrowField()->type()->id(), arrow::Type::INT32); + ASSERT_EQ(result[2].ArrowField()->type()->id(), arrow::Type::STRING); + } + + // Non-matching inline field name — no conversion should happen + { + auto result = BlobUtils::ConvertBlobInlineDataFields(data_fields, {"non_existent_field"}); + ASSERT_EQ(result[1].ArrowField()->type()->id(), arrow::Type::LARGE_BINARY); + } } } // namespace paimon::test diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 3edc12a4..3d579e07 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -35,11 +35,13 @@ #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/data_file_writer.h" #include "paimon/core/io/data_increment.h" +#include "paimon/core/io/external_storage_blob_writer.h" #include "paimon/core/io/multiple_blob_file_writer.h" #include "paimon/core/io/rolling_blob_file_writer.h" #include "paimon/core/io/rolling_file_writer.h" #include "paimon/core/io/single_file_writer.h" #include "paimon/core/manifest/file_source.h" +#include "paimon/core/operation/blob_file_context.h" #include "paimon/core/utils/commit_increment.h" #include "paimon/format/file_format.h" #include "paimon/format/file_format_factory.h" @@ -84,6 +86,36 @@ Status AppendOnlyWriter::Write(std::unique_ptr&& batch) { if (writer_ == nullptr) { PAIMON_ASSIGN_OR_RAISE(writer_, CreateRollingRowWriter()); } + + // Transform batch for external storage descriptor fields before writing. + if (external_storage_writer_) { + auto data_type = arrow::struct_(write_schema_->fields()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(batch->GetData(), data_type)); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr transformed, + external_storage_writer_->TransformBatch(struct_array)); + auto transformed_struct = std::dynamic_pointer_cast(transformed); + // TODO(lc.lsz): validate blob view + PAIMON_RETURN_NOT_OK(BlobUtils::ValidateInlineBlobDescriptors(transformed_struct, + inline_descriptor_fields_)); + ::ArrowArray c_transformed; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*transformed, &c_transformed)); + return writer_->Write(&c_transformed); + } + + if (!inline_descriptor_fields_.empty()) { + auto data_type = arrow::struct_(write_schema_->fields()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(batch->GetData(), data_type)); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + // TODO(lc.lsz): validate blob view + PAIMON_RETURN_NOT_OK( + BlobUtils::ValidateInlineBlobDescriptors(struct_array, inline_descriptor_fields_)); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, &c_array)); + return writer_->Write(&c_array); + } return writer_->Write(batch->GetData()); } @@ -155,14 +187,45 @@ Status AppendOnlyWriter::Flush(bool wait_for_latest_compaction, bool forced_full return Status::OK(); } -AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWriter() const { - auto schemas = BlobUtils::SeparateBlobSchema(write_schema_); - if (schemas.blob_schema && schemas.blob_schema->num_fields() > 0) { - return CreateRollingBlobWriter(schemas); +AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWriter() { + auto blob_context = BlobFileContext::Create(write_schema_, options_); + std::optional> main_write_cols = write_cols_; + + // Save inline descriptor fields for validation in Write() + if (blob_context) { + inline_descriptor_fields_ = blob_context->GetDescriptorFields(); + } + + // Initialize ExternalStorageBlobWriter if needed + if (blob_context && blob_context->RequireExternalStorageWriter()) { + assert(blob_context->GetExternalStoragePath()); + external_storage_writer_ = std::make_unique( + write_schema_, blob_context->GetExternalStorageFields(), + blob_context->GetExternalStoragePath().value(), schema_id_, seq_num_counter_, + path_factory_, options_, memory_pool_); + if (!main_write_cols) { + // To align with java, when require external storage writer, main writer will set write + // cols in DataFileMeta + main_write_cols = write_schema_->field_names(); + } + } + + if (blob_context && blob_context->RequireBlobFileWriter()) { + // Use context-aware schema separation: inline BLOB fields stay in main + auto schemas = + BlobUtils::SeparateBlobSchema(write_schema_, blob_context->GetInlineFields()); + return CreateRollingBlobWriter(schemas, blob_context->GetInlineFields()); + } else if (!blob_context) { + // No BLOB fields at all -> plain rolling writer + return std::make_unique>>( + options_.GetTargetFileSize(/*has_primary_key=*/false), + GetDataFileWriterCreator(write_schema_, main_write_cols)); } else { + // All BLOB fields are inline, no .blob files needed -> plain rolling writer + // The main data file contains all fields including inline descriptors/views. return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterCreator(write_schema_, write_cols_)); + GetDataFileWriterCreator(write_schema_, main_write_cols)); } } @@ -214,7 +277,7 @@ AppendOnlyWriter::SingleFileWriterCreator AppendOnlyWriter::GetBlobFileWriterCre } AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWriter( - const BlobUtils::SeparatedSchemas& schemas) const { + const BlobUtils::SeparatedSchemas& schemas, const std::set& inline_fields) const { // Multiple blob fields are supported. Each blob field gets its own rolling file writer // via MultipleBlobFileWriter. auto blob_schema = schemas.blob_schema; @@ -251,7 +314,7 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri return std::make_unique( options_.GetTargetFileSize(/*has_primary_key=*/false), GetDataFileWriterCreator(schemas.main_schema, schemas.main_schema->field_names()), - blob_schema, blob_writer_creator, arrow::struct_(write_schema_->fields())); + blob_schema, blob_writer_creator, arrow::struct_(write_schema_->fields()), inline_fields); } Status AppendOnlyWriter::Sync() { @@ -277,10 +340,14 @@ Status AppendOnlyWriter::Close() { writer_.reset(); } + if (external_storage_writer_) { + PAIMON_RETURN_NOT_OK(external_storage_writer_->Close()); + external_storage_writer_.reset(); + } + if (compact_deletion_file_ != nullptr) { compact_deletion_file_->Clean(); } return Status::OK(); } - } // namespace paimon diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index 25febf7c..d598725a 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -44,6 +45,7 @@ class Schema; namespace paimon { class CommitIncrement; +class ExternalStorageBlobWriter; class RecordBatch; template class RollingFileWriter; @@ -96,9 +98,10 @@ class AppendOnlyWriter : public BatchWriter { using RollingFileWriterResult = Result>>>; - RollingFileWriterResult CreateRollingRowWriter() const; + RollingFileWriterResult CreateRollingRowWriter(); RollingFileWriterResult CreateRollingBlobWriter( - const BlobUtils::SeparatedSchemas& schemas) const; + const BlobUtils::SeparatedSchemas& schemas, + const std::set& inline_fields) const; Result DrainIncrement(); Status Flush(bool wait_for_latest_compaction, bool forced_full_compaction); @@ -132,6 +135,8 @@ class AppendOnlyWriter : public BatchWriter { std::shared_ptr compact_deletion_file_; std::unique_ptr>> writer_; + std::unique_ptr external_storage_writer_; + std::set inline_descriptor_fields_; }; } // namespace paimon diff --git a/src/paimon/core/casting/binary_to_blob_cast_executor.cpp b/src/paimon/core/casting/binary_to_blob_cast_executor.cpp new file mode 100644 index 00000000..5642e62b --- /dev/null +++ b/src/paimon/core/casting/binary_to_blob_cast_executor.cpp @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/casting/binary_to_blob_cast_executor.h" + +#include +#include + +#include "arrow/array/array_binary.h" +#include "arrow/buffer.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/status.h" + +namespace arrow { +class Array; +} // namespace arrow + +namespace paimon { +Result BinaryToBlobCastExecutor::Cast( + const Literal& literal, const std::shared_ptr& target_type) const { + return Status::Invalid( + fmt::format("BinaryToBlobCastExecutor does not support literal cast from {} to {}", + static_cast(literal.GetType()), target_type->ToString())); +} + +Result> BinaryToBlobCastExecutor::Cast( + const std::shared_ptr& array, const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const { + if (array->type_id() != arrow::Type::BINARY) { + return Status::Invalid( + fmt::format("BinaryToBlobCastExecutor only supports binary input, got {}", + array->type()->ToString())); + } + if (target_type->id() != arrow::Type::LARGE_BINARY) { + return Status::Invalid( + fmt::format("BinaryToBlobCastExecutor only supports large_binary target, got {}", + target_type->ToString())); + } + + auto binary_array = std::static_pointer_cast(array); + if (binary_array->offset() != 0) { + return Status::Invalid("BinaryToBlobCastExecutor only supports arrays with zero offset"); + } + + const int64_t length = binary_array->length(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr large_offsets_buffer, + arrow::AllocateBuffer((length + 1) * static_cast(sizeof(int64_t)), pool)); + auto* large_offsets = reinterpret_cast(large_offsets_buffer->mutable_data()); + for (int64_t row_index = 0; row_index <= length; row_index++) { + large_offsets[row_index] = binary_array->value_offset(row_index); + } + + std::shared_ptr null_bitmap = binary_array->null_bitmap(); + if (binary_array->null_count() == 0) { + null_bitmap.reset(); + } + + auto value_data = binary_array->value_data(); + auto array_data = + arrow::ArrayData::Make(target_type, length, {null_bitmap, large_offsets_buffer, value_data}, + binary_array->null_count()); + return arrow::MakeArray(array_data); +} + +} // namespace paimon diff --git a/src/paimon/core/casting/binary_to_blob_cast_executor.h b/src/paimon/core/casting/binary_to_blob_cast_executor.h new file mode 100644 index 00000000..8ea06a3a --- /dev/null +++ b/src/paimon/core/casting/binary_to_blob_cast_executor.h @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "arrow/array/array_base.h" +#include "paimon/core/casting/cast_executor.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class MemoryPool; +} // namespace arrow + +namespace paimon { +class BinaryToBlobCastExecutor : public CastExecutor { + public: + Result Cast(const Literal& literal, + const std::shared_ptr& target_type) const override; + + Result> Cast(const std::shared_ptr& array, + const std::shared_ptr& target_type, + arrow::MemoryPool* pool) const override; +}; + +} // namespace paimon diff --git a/src/paimon/core/casting/cast_executor_factory.cpp b/src/paimon/core/casting/cast_executor_factory.cpp index 2c0a4116..8e56ca04 100644 --- a/src/paimon/core/casting/cast_executor_factory.cpp +++ b/src/paimon/core/casting/cast_executor_factory.cpp @@ -21,6 +21,7 @@ #include +#include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/binary_to_string_cast_executor.h" #include "paimon/core/casting/boolean_to_decimal_cast_executor.h" #include "paimon/core/casting/boolean_to_numeric_cast_executor.h" @@ -149,6 +150,8 @@ CastExecutorFactory::CastExecutorFactory() { REGISTER_CAST_EXECUTOR(STRING, BINARY, BinaryToStringCastExecutor); + REGISTER_CAST_EXECUTOR(BLOB, BINARY, BinaryToBlobCastExecutor); + REGISTER_CAST_EXECUTOR(STRING, DATE, DateToStringCastExecutor); REGISTER_CAST_EXECUTOR(TIMESTAMP, DATE, DateToTimestampCastExecutor); diff --git a/src/paimon/core/casting/cast_executor_factory_test.cpp b/src/paimon/core/casting/cast_executor_factory_test.cpp index 56acbe48..9ab7d168 100644 --- a/src/paimon/core/casting/cast_executor_factory_test.cpp +++ b/src/paimon/core/casting/cast_executor_factory_test.cpp @@ -20,6 +20,7 @@ #include "paimon/core/casting/cast_executor_factory.h" #include "gtest/gtest.h" +#include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/binary_to_string_cast_executor.h" #include "paimon/core/casting/boolean_to_decimal_cast_executor.h" #include "paimon/core/casting/boolean_to_numeric_cast_executor.h" @@ -123,6 +124,13 @@ TEST(CastExecutorFactoryTest, TestRegister) { ASSERT_TRUE(cast_executor); ASSERT_TRUE(std::dynamic_pointer_cast(cast_executor)); } + { + auto* factory = CastExecutorFactory::GetCastExecutorFactory(); + ASSERT_FALSE(factory->executor_map_.empty()); + auto cast_executor = factory->GetCastExecutor(FieldType::BINARY, FieldType::BLOB); + ASSERT_TRUE(cast_executor); + ASSERT_TRUE(std::dynamic_pointer_cast(cast_executor)); + } { auto* factory = CastExecutorFactory::GetCastExecutorFactory(); ASSERT_FALSE(factory->executor_map_.empty()); diff --git a/src/paimon/core/casting/cast_executor_test.cpp b/src/paimon/core/casting/cast_executor_test.cpp index 7e122549..df9caff0 100644 --- a/src/paimon/core/casting/cast_executor_test.cpp +++ b/src/paimon/core/casting/cast_executor_test.cpp @@ -37,6 +37,7 @@ #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/decimal_utils.h" #include "paimon/common/utils/field_type_utils.h" +#include "paimon/core/casting/binary_to_blob_cast_executor.h" #include "paimon/core/casting/binary_to_string_cast_executor.h" #include "paimon/core/casting/boolean_to_decimal_cast_executor.h" #include "paimon/core/casting/boolean_to_numeric_cast_executor.h" @@ -1302,6 +1303,43 @@ TEST_F(CastExecutorTest, TestBinaryToStringCastExecutorCastArray) { } } +TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastLiteral) { + auto cast_executor = std::make_shared(); + std::string src_data = "blob-descriptor-bytes"; + ASSERT_NOK_WITH_MSG( + cast_executor->Cast(Literal(FieldType::BINARY, src_data.data(), src_data.size()), + arrow::large_binary()), + "BinaryToBlobCastExecutor does not support literal cast"); +} + +TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastArray) { + auto cast_executor = std::make_shared(); + auto src_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::binary(), R"(["foo", "bar", "", null, "blob"])") + .ValueOrDie(); + auto expected_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::large_binary(), R"(["foo", "bar", "", null, "blob"])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr target_array, + cast_executor->Cast(src_array, arrow::large_binary(), arrow::default_memory_pool())); + ASSERT_TRUE(target_array->Equals(expected_array)); + ASSERT_EQ(target_array->data()->buffers[2], src_array->data()->buffers[2]); +} + +TEST_F(CastExecutorTest, TestBinaryToBlobCastExecutorCastArrayWithOffset) { + auto cast_executor = std::make_shared(); + auto src_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::binary(), R"(["skip", "foo", "bar"])") + .ValueOrDie() + ->Slice(1, 2); + + ASSERT_NOK_WITH_MSG( + cast_executor->Cast(src_array, arrow::large_binary(), arrow::default_memory_pool()), + "BinaryToBlobCastExecutor only supports arrays with zero offset"); +} + TEST_F(CastExecutorTest, TestDateToStringCastExecutorCastLiteral) { auto cast_executor = std::make_shared(); // date values ranging from 0000-01-01 to 9999-12-31 diff --git a/src/paimon/core/io/data_file_path_factory.h b/src/paimon/core/io/data_file_path_factory.h index b49154f1..90ab01f3 100644 --- a/src/paimon/core/io/data_file_path_factory.h +++ b/src/paimon/core/io/data_file_path_factory.h @@ -64,6 +64,12 @@ class DataFilePathFactory : public PathFactory { return NewPathFromName(NewFileName(data_file_prefix_, ".blob")); } + /// Creates a new blob file path under the given external storage path for descriptor fields. + std::string NewExternalStorageBlobPath(const std::string& external_storage_path) const { + std::string file_name = NewFileName(data_file_prefix_, ".blob"); + return PathUtil::JoinPath(external_storage_path, file_name); + } + std::string NewPathFromName(const std::string& file_name) const { if (external_path_provider_ != nullptr) { return external_path_provider_->GetNextExternalDataPath(file_name); diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp index 293f14ab..50010ee5 100644 --- a/src/paimon/core/io/data_file_path_factory_test.cpp +++ b/src/paimon/core/io/data_file_path_factory_test.cpp @@ -24,6 +24,7 @@ #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/fs/external_path_provider.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/stats/simple_stats.h" @@ -58,6 +59,20 @@ TEST_F(DataFilePathFactoryTest, TestNewPath) { ASSERT_EQ(factory_.NewPathFromName("index-file"), "/tmp/index-file"); } +TEST_F(DataFilePathFactoryTest, TestNewExternalStorageBlobPath) { + std::string blob_path1 = factory_.NewExternalStorageBlobPath("/tmp/external_blob"); + std::string blob_path2 = factory_.NewExternalStorageBlobPath("/tmp/external_blob"); + + // Paths are unique (counter increments) + ASSERT_NE(blob_path1, blob_path2); + // Both start with the external storage path joined with the data file prefix + ASSERT_TRUE(StringUtils::StartsWith(blob_path1, "/tmp/external_blob/data-")); + ASSERT_TRUE(StringUtils::StartsWith(blob_path2, "/tmp/external_blob/data-")); + // Both end with .blob extension + ASSERT_TRUE(StringUtils::EndsWith(blob_path1, ".blob")); + ASSERT_TRUE(StringUtils::EndsWith(blob_path2, ".blob")); +} + TEST_F(DataFilePathFactoryTest, TestNewPathWithDataFilePrefixAndExternalPath) { DataFilePathFactory factory; ASSERT_OK_AND_ASSIGN( diff --git a/src/paimon/core/io/external_storage_blob_writer.cpp b/src/paimon/core/io/external_storage_blob_writer.cpp new file mode 100644 index 00000000..05b69e74 --- /dev/null +++ b/src/paimon/core/io/external_storage_blob_writer.cpp @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/external_storage_blob_writer.h" + +#include +#include + +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_binary.h" +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/data_file_writer.h" +#include "paimon/format/blob/blob_writer_builder.h" +#include "paimon/format/file_format.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" + +namespace paimon { + +ExternalStorageBlobWriter::ExternalStorageBlobWriter( + const std::shared_ptr& write_schema, + const std::set& external_storage_fields, const std::string& external_storage_path, + int64_t schema_id, const std::shared_ptr& seq_num_counter, + const std::shared_ptr& path_factory, const CoreOptions& options, + const std::shared_ptr& memory_pool) + : write_schema_(write_schema), + external_storage_fields_(external_storage_fields), + external_storage_path_(external_storage_path), + schema_id_(schema_id), + seq_num_counter_(seq_num_counter), + path_factory_(path_factory), + memory_pool_(memory_pool), + options_(options) {} + +Result> +ExternalStorageBlobWriter::CreateFieldRollingWriter(FieldWriter* field_writer) { + auto field = write_schema_->GetFieldByName(field_writer->field_name); + if (!field) { + return Status::Invalid("External storage field '{}' not found in write schema", + field_writer->field_name); + } + + auto single_field_schema = arrow::schema({field}); + ::ArrowSchema arrow_schema; + ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*single_field_schema, &arrow_schema)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, + FileFormatFactory::Get("blob", options_.ToMap())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr writer_builder, + format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); + writer_builder->WithMemoryPool(memory_pool_); + + // Inject WriteConsumer to capture BlobDescriptors during writes + auto blob_writer_builder = std::dynamic_pointer_cast(writer_builder); + if (!blob_writer_builder) { + return Status::Invalid( + "writer_builder cannot be casted to BlobWriterBuilder in ExternalStorageBlobWriter"); + } + blob_writer_builder->WithWriteConsumer( + [field_writer](std::unique_ptr descriptor) -> bool { + field_writer->captured_descriptors.push_back(std::move(descriptor)); + return true; // Always flush for single row. + }); + + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*single_field_schema, &arrow_schema)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, + format->CreateStatsExtractor(&arrow_schema)); + + std::vector write_cols = {field_writer->field_name}; + auto single_blob_file_writer_creator = [this, writer_builder, stats_extractor, write_cols]() + -> Result>>> { + auto writer = std::make_unique( + /*compression=*/"none", std::function(), schema_id_, + seq_num_counter_, FileSource::Append(), stats_extractor, + path_factory_->IsExternalPath(), write_cols, memory_pool_); + PAIMON_RETURN_NOT_OK(writer->Init( + options_.GetFileSystem(), + path_factory_->NewExternalStorageBlobPath(external_storage_path_), writer_builder)); + return writer; + }; + + return std::make_unique(options_.GetBlobTargetFileSize(), + single_blob_file_writer_creator); +} + +Status ExternalStorageBlobWriter::InitializeFieldWritersIfNeeded() { + if (initialized_) { + return Status::OK(); + } + for (int32_t i = 0; i < write_schema_->num_fields(); ++i) { + const auto& field = write_schema_->field(i); + if (external_storage_fields_.count(field->name()) > 0) { + FieldWriter fw; + fw.field_name = field->name(); + fw.field_index = i; + field_writers_.push_back(std::move(fw)); + } + } + // Create rolling writers after push_back so FieldWriter addresses are stable + // for the consumer lambda capture. + for (auto& fw : field_writers_) { + PAIMON_ASSIGN_OR_RAISE(fw.rolling_writer, CreateFieldRollingWriter(&fw)); + } + initialized_ = true; + return Status::OK(); +} + +Result> ExternalStorageBlobWriter::TransformField( + const std::shared_ptr& column, FieldWriter* field_writer) { + int64_t num_rows = column->length(); + + // Clear captured descriptors before processing this batch + field_writer->captured_descriptors.clear(); + + // Write each row via RollingFileWriter; the consumer captures the descriptor + for (int64_t row = 0; row < num_rows; ++row) { + std::shared_ptr slice = column->Slice(row, 1); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr single_row_struct, + arrow::StructArray::Make({slice}, {field_writer->field_name})); + + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*single_row_struct, &c_array)); + PAIMON_RETURN_NOT_OK(field_writer->rolling_writer->Write(&c_array)); + } + + // Validate captured descriptor count + if (static_cast(field_writer->captured_descriptors.size()) != num_rows) { + return Status::Invalid( + "Captured descriptor count {} does not match row count {} for field '{}'", + field_writer->captured_descriptors.size(), num_rows, field_writer->field_name); + } + + // Build descriptor column from captured descriptors + arrow::LargeBinaryBuilder descriptor_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(descriptor_builder.Reserve(num_rows)); + for (int64_t row = 0; row < num_rows; ++row) { + const auto& descriptor = field_writer->captured_descriptors[row]; + if (!descriptor) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(descriptor_builder.AppendNull()); + } else { + auto serialized = descriptor->Serialize(memory_pool_); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + descriptor_builder.Append(serialized->data(), serialized->size())); + } + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr descriptor_array, + descriptor_builder.Finish()); + return descriptor_array; +} + +Result> ExternalStorageBlobWriter::TransformBatch( + const std::shared_ptr& batch) { + if (external_storage_fields_.empty()) { + return batch; + } + + PAIMON_RETURN_NOT_OK(InitializeFieldWritersIfNeeded()); + + if (field_writers_.empty()) { + return batch; + } + + // Collect all arrays and field names from the original batch + std::vector> result_arrays; + std::vector result_names; + result_arrays.reserve(batch->num_fields()); + result_names.reserve(batch->num_fields()); + + for (int32_t col = 0; col < batch->num_fields(); ++col) { + result_names.push_back(batch->type()->field(col)->name()); + result_arrays.push_back(batch->field(col)); + } + + // Transform each external storage field and replace in result + for (FieldWriter& fw : field_writers_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr descriptor_array, + TransformField(batch->field(fw.field_index), &fw)); + result_arrays[fw.field_index] = descriptor_array; + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr result, + arrow::StructArray::Make(result_arrays, result_names)); + return result; +} + +Status ExternalStorageBlobWriter::Close() { + for (FieldWriter& fw : field_writers_) { + if (fw.rolling_writer) { + PAIMON_RETURN_NOT_OK(fw.rolling_writer->Close()); + } + } + return Status::OK(); +} + +void ExternalStorageBlobWriter::Abort() { + for (FieldWriter& fw : field_writers_) { + if (fw.rolling_writer) { + fw.rolling_writer->Abort(); + fw.rolling_writer.reset(); + } + } + field_writers_.clear(); +} + +} // namespace paimon diff --git a/src/paimon/core/io/external_storage_blob_writer.h b/src/paimon/core/io/external_storage_blob_writer.h new file mode 100644 index 00000000..f4319cac --- /dev/null +++ b/src/paimon/core/io/external_storage_blob_writer.h @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/rolling_file_writer.h" +#include "paimon/core/io/single_file_writer.h" +#include "paimon/logging.h" +#include "paimon/result.h" +#include "paimon/status.h" +namespace arrow { +class Schema; +class StructArray; +} // namespace arrow + +namespace paimon { + +class FileSystem; +class LongCounter; +class MemoryPool; +class DataFilePathFactory; + +/// Batch-oriented writer for descriptor BLOB fields that writes raw data to external storage. +/// +/// For each configured external_storage field, this writer: +/// 1. Uses RollingFileWriter (same infra as MultipleBlobFileWriter) with BlobFormatWriter +/// 2. Injects a WriteConsumer into BlobFormatWriter to capture each row's BlobDescriptor +/// 3. After writing a batch, constructs a descriptor column from captured descriptors +/// +/// After TransformBatch(), the returned StructArray has descriptor columns replaced with +/// serialized BlobDescriptor bytes (large_binary), ready to be written into the main data file. +class ExternalStorageBlobWriter { + public: + using BlobRollingWriter = RollingFileWriter<::ArrowArray*, std::shared_ptr>; + + ExternalStorageBlobWriter(const std::shared_ptr& write_schema, + const std::set& external_storage_fields, + const std::string& external_storage_path, int64_t schema_id, + const std::shared_ptr& seq_num_counter, + const std::shared_ptr& path_factory, + const CoreOptions& options, + const std::shared_ptr& memory_pool); + + /// Transforms a batch by writing external storage fields to .blob files and replacing + /// the BLOB values with serialized BlobDescriptor bytes. + Result> TransformBatch( + const std::shared_ptr& batch); + + /// Closes all internal blob writers and flushes pending data. + Status Close(); + + /// Aborts all internal blob writers. + void Abort(); + + private: + /// Per-field writer state for one external storage blob field. + struct FieldWriter { + std::string field_name; + int32_t field_index; + std::unique_ptr rolling_writer; + /// Descriptors captured by the WriteConsumer callback during writes. + std::vector> captured_descriptors; + }; + + /// Lazily initializes per-field writers on first call to TransformBatch. + Status InitializeFieldWritersIfNeeded(); + + /// Writes all rows of a single external blob field via RollingFileWriter and returns + /// a descriptor column (LargeBinary) built from captured BlobDescriptors. + Result> TransformField( + const std::shared_ptr& column, FieldWriter* field_writer); + + /// Creates a RollingFileWriter for one external storage blob field with consumer injected. + Result> CreateFieldRollingWriter(FieldWriter* field_writer); + + std::shared_ptr write_schema_; + std::set external_storage_fields_; + std::string external_storage_path_; + int64_t schema_id_; + std::shared_ptr seq_num_counter_; + std::shared_ptr path_factory_; + std::shared_ptr memory_pool_; + CoreOptions options_; + + std::vector field_writers_; + bool initialized_ = false; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/external_storage_blob_writer_test.cpp b/src/paimon/core/io/external_storage_blob_writer_test.cpp new file mode 100644 index 00000000..32950d26 --- /dev/null +++ b/src/paimon/core/io/external_storage_blob_writer_test.cpp @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/external_storage_blob_writer.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/utils/long_counter.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class ExternalStorageBlobWriterTest : public ::testing::Test { + protected: + void SetUp() override { + dir_ = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + + pool_ = GetDefaultPool(); + seq_num_counter_ = std::make_shared(0); + + // Create CoreOptions with blob format + ASSERT_OK_AND_ASSIGN(options_, CoreOptions::FromMap({})); + file_system_ = options_.GetFileSystem(); + + // Create external storage directory + external_storage_path_ = dir_->Str() + "/external_blob"; + ASSERT_OK(file_system_->Mkdirs(external_storage_path_)); + + // Initialize DataFilePathFactory + path_factory_ = std::make_shared(); + ASSERT_OK(path_factory_->Init(dir_->Str(), "blob", "data-", nullptr)); + + // Schema: int_col (int32) + blob_col (blob) + auto int_field = arrow::field("int_col", arrow::int32()); + auto blob_field = BlobUtils::ToArrowField("blob_col", false); + write_schema_ = arrow::schema({int_field, blob_field}); + } + + std::unique_ptr dir_; + std::shared_ptr pool_; + std::shared_ptr seq_num_counter_; + CoreOptions options_; + std::shared_ptr file_system_; + std::shared_ptr path_factory_; + std::shared_ptr write_schema_; + std::string external_storage_path_; +}; + +TEST_F(ExternalStorageBlobWriterTest, TestEmptyExternalFields) { + // No external storage fields -> TransformBatch returns original batch + ExternalStorageBlobWriter writer(write_schema_, /*external_storage_fields=*/{}, + external_storage_path_, /*schema_id=*/0, seq_num_counter_, + path_factory_, options_, pool_); + + auto input = std::static_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_schema_->fields()), + R"([[42, "hello"]])") + .ValueOrDie()); + + ASSERT_OK_AND_ASSIGN(auto result, writer.TransformBatch(input)); + ASSERT_TRUE(result->Equals(*input)); + + ASSERT_OK(writer.Close()); +} + +TEST_F(ExternalStorageBlobWriterTest, TestTransformBatchReplacesBlob) { + std::set external_fields = {"blob_col"}; + ExternalStorageBlobWriter writer(write_schema_, external_fields, external_storage_path_, + /*schema_id=*/0, seq_num_counter_, path_factory_, options_, + pool_); + + auto struct_type = arrow::struct_(write_schema_->fields()); + auto input = std::static_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(struct_type, R"([[10, "data1"], [20, "data2"]])") + .ValueOrDie()); + + auto original_int_col = input->field(0); + + ASSERT_OK_AND_ASSIGN(auto result, writer.TransformBatch(input)); + + // int_col should be unchanged + ASSERT_EQ(result->num_fields(), 2); + ASSERT_TRUE(result->field(0)->Equals(*original_int_col)); + + // blob_col should be replaced with serialized BlobDescriptors + auto descriptor_col = std::static_pointer_cast(result->field(1)); + ASSERT_EQ(descriptor_col->length(), 2); + + for (int64_t i = 0; i < 2; ++i) { + ASSERT_FALSE(descriptor_col->IsNull(i)); + auto view = descriptor_col->GetView(i); + ASSERT_OK_AND_ASSIGN(auto descriptor, + BlobDescriptor::Deserialize(view.data(), view.size())); + ASSERT_EQ(descriptor->Length(), 5); + ASSERT_TRUE(descriptor->Uri().find(external_storage_path_) != std::string::npos); + } + + ASSERT_OK(writer.Close()); +} + +TEST_F(ExternalStorageBlobWriterTest, TestAbort) { + std::set external_fields = {"blob_col"}; + ExternalStorageBlobWriter writer(write_schema_, external_fields, external_storage_path_, + /*schema_id=*/0, seq_num_counter_, path_factory_, options_, + pool_); + + auto input = std::static_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_schema_->fields()), + R"([[1, "abort_test"]])") + .ValueOrDie()); + + ASSERT_OK(writer.TransformBatch(input)); + + // Verify blob files exist before abort + std::vector> files_before; + ASSERT_OK(file_system_->ListDir(external_storage_path_, &files_before)); + ASSERT_FALSE(files_before.empty()); + + // Abort should clean up written blob files + writer.Abort(); + + std::vector> files_after; + ASSERT_OK(file_system_->ListDir(external_storage_path_, &files_after)); + ASSERT_TRUE(files_after.empty()); +} + +} // namespace paimon::test diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index 4b64a0b0..9eb6f20a 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -34,6 +34,7 @@ #include "arrow/ipc/json_simple.h" #include "arrow/util/checked_cast.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/defs.h" @@ -183,6 +184,9 @@ class FieldMappingReaderTest : public ::testing::Test { auto expected_chunk_array = std::make_shared(arrow::ArrayVector({expect_array})); + ASSERT_TRUE(result_array->type()->Equals(expected_chunk_array->type())) + << result_array->type()->ToString() << expected_chunk_array->type()->ToString(); + ASSERT_TRUE(result_array->Equals(expected_chunk_array)) << result_array->ToString() << expected_chunk_array->ToString(); } @@ -689,6 +693,33 @@ TEST_F(FieldMappingReaderTest, TestSchemaEvolutionWithDictType) { partition, expected_array); } +TEST_F(FieldMappingReaderTest, TestReadInlineBlobAsBinaryDataFile) { + // data_fields uses binary type because inline blob fields are stored as binary in data files + std::vector data_fields = { + DataField(0, arrow::field("descriptor", arrow::binary(), /*nullable=*/true)), + }; + auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields); + std::string json_str = R"([ + ["descriptor-1"], + [null], + ["descriptor-2"] + ])"; + auto data_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(data_schema->fields()), json_str) + .ValueOrDie()); + + std::vector read_fields = { + DataField(0, BlobUtils::ToArrowField("descriptor", /*nullable=*/true)), + }; + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), json_str) + .ValueOrDie()); + + CheckResult(data_schema, data_array, read_schema, /*predicate=*/nullptr, + /*partition_keys=*/{}, BinaryRow::EmptyRow(), expected); +} + TEST_F(FieldMappingReaderTest, TestReadWithSchemaEvolutionRenameCombinedCast) { // Test all 4 combinations of rename × cast: // f0: no rename, no cast (utf8 → utf8, name unchanged) diff --git a/src/paimon/core/io/rolling_blob_file_writer.cpp b/src/paimon/core/io/rolling_blob_file_writer.cpp index ccd6ca45..40c362f8 100644 --- a/src/paimon/core/io/rolling_blob_file_writer.cpp +++ b/src/paimon/core/io/rolling_blob_file_writer.cpp @@ -46,12 +46,13 @@ RollingBlobFileWriter::RollingBlobFileWriter( std::function>()> create_file_writer, const std::shared_ptr& blob_schema, MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator, - const std::shared_ptr& data_type) + const std::shared_ptr& data_type, const std::set& inline_fields) : RollingFileWriter<::ArrowArray*, std::shared_ptr>(target_file_size, create_file_writer), blob_schema_(blob_schema), blob_writer_creator_(std::move(blob_writer_creator)), data_type_(data_type), + inline_fields_(inline_fields), logger_(Logger::GetLogger("RollingBlobFileWriter")) {} Status RollingBlobFileWriter::Write(::ArrowArray* record) { @@ -69,7 +70,7 @@ Status RollingBlobFileWriter::Write(::ArrowArray* record) { auto struct_array = std::dynamic_pointer_cast(arrow_array); PAIMON_ASSIGN_OR_RAISE(BlobUtils::SeparatedStructArrays separated_arrays, - BlobUtils::SeparateBlobArray(struct_array)); + BlobUtils::SeparateBlobArray(struct_array, inline_fields_)); // Write main (non-blob) data ::ArrowArray c_main_array; PAIMON_RETURN_NOT_OK_FROM_ARROW( diff --git a/src/paimon/core/io/rolling_blob_file_writer.h b/src/paimon/core/io/rolling_blob_file_writer.h index f35f6960..a3f10383 100644 --- a/src/paimon/core/io/rolling_blob_file_writer.h +++ b/src/paimon/core/io/rolling_blob_file_writer.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "arrow/array/array_nested.h" @@ -64,7 +65,8 @@ class RollingBlobFileWriter std::function>()> create_file_writer, const std::shared_ptr& blob_schema, MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator, - const std::shared_ptr& data_type); + const std::shared_ptr& data_type, + const std::set& inline_fields); ~RollingBlobFileWriter() override = default; Status Write(::ArrowArray* record) override; @@ -87,6 +89,7 @@ class RollingBlobFileWriter MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator_; std::unique_ptr blob_writer_; std::shared_ptr data_type_; + std::set inline_fields_; std::unique_ptr logger_; }; diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index 30b2215a..e20b79a0 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -23,6 +23,7 @@ #include #include "arrow/type.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/reader/delegating_prefetch_reader.h" #include "paimon/common/reader/predicate_batch_reader.h" #include "paimon/common/reader/prefetch_file_batch_reader_impl.h" @@ -180,6 +181,10 @@ Result> AbstractSplitRead::CreateFieldMappingRe // load schema to get data schema PAIMON_ASSIGN_OR_RAISE(data_schema, schema_manager_->ReadSchema(file_meta->schema_id)); } + PAIMON_ASSIGN_OR_RAISE(CoreOptions data_options, + CoreOptions::FromMap(data_schema->Options(), options_.GetFileSystem())); + auto blob_inline_fields = data_options.GetBlobInlineFields(); + std::unique_ptr field_mapping; if (!data_schema->PrimaryKeys().empty()) { // for pk table, add special fields to file schema when field mapping @@ -193,8 +198,10 @@ Result> AbstractSplitRead::CreateFieldMappingRe PAIMON_ASSIGN_OR_RAISE( std::vector projected_data_fields, ProjectFieldsForRowTrackingAndDataEvolution(data_schema, file_meta->write_cols)); + auto converted_fields = + BlobUtils::ConvertBlobInlineDataFields(projected_data_fields, blob_inline_fields); PAIMON_ASSIGN_OR_RAISE(field_mapping, - field_mapping_builder->CreateFieldMapping(projected_data_fields)); + field_mapping_builder->CreateFieldMapping(converted_fields)); } auto read_schema = DataField::ConvertDataFieldsToArrowSchema( diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 427a0fa0..7f407cb5 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -73,10 +73,6 @@ AppendOnlyFileStoreWrite::AppendOnlyFileStoreWrite( is_streaming_mode, ignore_num_bucket_check, executor, pool), logger_(Logger::GetLogger("AppendOnlyFileStoreWrite")) { write_cols_ = write_schema->field_names(); - auto schemas = BlobUtils::SeparateBlobSchema(schema_); - if (schemas.blob_schema && schemas.blob_schema->num_fields() > 0) { - with_blob_ = true; - } // optimize write_cols to null in following cases: // 1. write_schema contains all columns // 2. TODO(xinyu.lxy) write_schema contains all columns and append _ROW_ID & _SEQUENCE_NUMBER @@ -174,9 +170,7 @@ Result> AppendOnlyFileStoreWrite::CreateWriter( file_store_path_factory_->CreateDataFilePathFactory(partition, bucket)); std::shared_ptr compact_manager; - auto schemas = BlobUtils::SeparateBlobSchema(write_schema_); - if (options_.WriteOnly() || options_.DataEvolutionEnabled() || options_.GetBucket() == -1 || - with_blob_) { + if (options_.WriteOnly() || options_.DataEvolutionEnabled() || options_.GetBucket() == -1) { compact_manager = std::make_shared(); } else { auto dv_factory = diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h index d1aacb5a..e3b729ec 100644 --- a/src/paimon/core/operation/append_only_file_store_write.h +++ b/src/paimon/core/operation/append_only_file_store_write.h @@ -117,7 +117,6 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { const std::vector>& files) const; std::optional> write_cols_; - bool with_blob_ = false; std::unique_ptr logger_; }; diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index 65acb7a6..b5275eed 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -29,6 +29,7 @@ #include "arrow/type.h" #include "fmt/format.h" #include "paimon/common/data/binary_array.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/executor/future.h" #include "paimon/common/predicate/literal_converter.h" #include "paimon/common/types/data_field.h" @@ -356,8 +357,12 @@ Status FileStoreScan::SplitAndSetFilter(const std::vector& partitio PAIMON_ASSIGN_OR_RAISE(std::unique_ptr mapping_builder, FieldMappingBuilder::Create(arrow_schema, partition_keys, scan_filters->GetPredicate())); + PAIMON_ASSIGN_OR_RAISE(std::vector data_fields, + DataField::ConvertArrowSchemaToDataFields(arrow_schema)); + auto converted_fields = BlobUtils::ConvertBlobInlineDataFields( + data_fields, core_options_.GetBlobInlineFields()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr mapping, - mapping_builder->CreateFieldMapping(arrow_schema)); + mapping_builder->CreateFieldMapping(converted_fields)); if (mapping->partition_info != std::nullopt) { const auto& partition_info = mapping->partition_info.value(); partition_schema_ = diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 17037ed9..eb0199a0 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -450,12 +450,8 @@ Status SchemaValidation::ValidateBlobFields(const TableSchema& schema, const Cor const auto& blob_descriptor_names = options.GetBlobDescriptorFields(); const auto& blob_view_names = options.GetBlobViewFields(); const auto& blob_external_storage_names = options.GetBlobExternalStorageFields(); - std::vector configured_blob_like_names = configured_blob_names; - configured_blob_like_names.insert(configured_blob_like_names.end(), - blob_descriptor_names.begin(), blob_descriptor_names.end()); - configured_blob_like_names.insert(configured_blob_like_names.end(), blob_view_names.begin(), - blob_view_names.end()); - if (configured_blob_like_names.empty() && blob_external_storage_names.empty()) { + if (configured_blob_names.empty() && blob_descriptor_names.empty() && blob_view_names.empty() && + blob_external_storage_names.empty()) { return Status::OK(); } diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index e12b82d4..5ab695ba 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -19,10 +19,8 @@ #include "paimon/core/utils/field_mapping.h" -#include -#include #include -#include +#include #include "arrow/type.h" #include "fmt/format.h" diff --git a/src/paimon/core/utils/field_mapping.h b/src/paimon/core/utils/field_mapping.h index ae7795c5..9445add3 100644 --- a/src/paimon/core/utils/field_mapping.h +++ b/src/paimon/core/utils/field_mapping.h @@ -18,8 +18,6 @@ */ #pragma once -#include -#include #include #include #include diff --git a/src/paimon/format/avro/avro_direct_encoder.cpp b/src/paimon/format/avro/avro_direct_encoder.cpp index 8a4d1b92..90ff7199 100644 --- a/src/paimon/format/avro/avro_direct_encoder.cpp +++ b/src/paimon/format/avro/avro_direct_encoder.cpp @@ -224,7 +224,14 @@ Status AvroDirectEncoder::EncodeArrowToAvro(const ::avro::NodePtr& avro_node, return Status::OK(); } - // Handle regular BYTES + // Handle regular BYTES (binary or large_binary) + if (array.type()->id() == arrow::Type::LARGE_BINARY) { + const auto& large_binary_array = + arrow::internal::checked_cast(array); + std::string_view value = large_binary_array.GetView(row_index); + encoder->encodeBytes(reinterpret_cast(value.data()), value.size()); + return Status::OK(); + } const auto& binary_array = arrow::internal::checked_cast(array); std::string_view value = binary_array.GetView(row_index); diff --git a/src/paimon/format/avro/avro_file_batch_reader_test.cpp b/src/paimon/format/avro/avro_file_batch_reader_test.cpp index fa506796..a4a26a88 100644 --- a/src/paimon/format/avro/avro_file_batch_reader_test.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader_test.cpp @@ -405,6 +405,56 @@ TEST_F(AvroFileBatchReaderTest, TestGetNumberOfRows) { } } +TEST_F(AvroFileBatchReaderTest, TestReadBinaryWrittenFromBinaryAndLargeBinary) { + auto check_binary_read_result = [&](const std::shared_ptr& write_type, + const std::string& file_name) { + std::string data_json = R"([ + ["descriptor-1"], + [""], + [null], + ["descriptor-2"] + ])"; + auto write_field = arrow::field("f0", write_type); + auto write_data_type = arrow::struct_({write_field}); + auto write_array = + arrow::ipc::internal::json::ArrayFromJSON(write_data_type, data_json).ValueOrDie(); + + std::string file_path = PathUtil::JoinPath(dir_->Str(), file_name); + WriteData(write_array, file_path, /*compression=*/"null"); + + // Read back with binary schema + auto read_field = arrow::field("f0", arrow::binary()); + auto read_data_type = arrow::struct_({read_field}); + + ASSERT_OK_AND_ASSIGN(auto reader_builder, + file_format_->CreateReaderBuilder(/*batch_size=*/1024)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, reader_builder->Build(in)); + + // Check GetFileSchema: regardless of write type, avro file schema is always binary + ASSERT_OK_AND_ASSIGN(auto c_file_schema, batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + arrow::Schema expected_file_schema({read_field}); + ASSERT_TRUE(file_schema->Equals(expected_file_schema)); + + auto read_schema = arrow::schema({read_field}); + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + EXPECT_OK(batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto result_array, ::paimon::test::ReadResultCollector::CollectResult( + batch_reader.get())); + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(read_data_type, data_json).ValueOrDie(); + auto expected_chunked_array = std::make_shared(expected_array); + ASSERT_TRUE(result_array->Equals(expected_chunked_array)); + }; + + check_binary_read_result(arrow::binary(), "binary.avro"); + check_binary_read_result(arrow::large_binary(), "large-binary.avro"); +} + INSTANTIATE_TEST_SUITE_P(TestParam, AvroFileBatchReaderTest, ::testing::Values(false, true)); } // namespace paimon::avro::test diff --git a/src/paimon/format/avro/avro_schema_converter.cpp b/src/paimon/format/avro/avro_schema_converter.cpp index ef905fff..d62bfc7e 100644 --- a/src/paimon/format/avro/avro_schema_converter.cpp +++ b/src/paimon/format/avro/avro_schema_converter.cpp @@ -269,6 +269,7 @@ Result<::avro::Schema> AvroSchemaConverter::ArrowTypeToAvroSchema( case arrow::Type::STRING: return nullable ? NullableSchema(::avro::StringSchema()) : ::avro::StringSchema(); case arrow::Type::BINARY: + case arrow::Type::LARGE_BINARY: return nullable ? NullableSchema(::avro::BytesSchema()) : ::avro::BytesSchema(); case arrow::Type::type::DATE32: { ::avro::IntSchema date_schema; diff --git a/src/paimon/format/avro/avro_stats_extractor.cpp b/src/paimon/format/avro/avro_stats_extractor.cpp index 104983d0..3bfbcf95 100644 --- a/src/paimon/format/avro/avro_stats_extractor.cpp +++ b/src/paimon/format/avro/avro_stats_extractor.cpp @@ -94,6 +94,7 @@ Result> AvroStatsExtractor::FetchColumnStatistics( case arrow::Type::type::DOUBLE: return ColumnStats::CreateDoubleColumnStats(std::nullopt, std::nullopt, std::nullopt); case arrow::Type::type::BINARY: + case arrow::Type::type::LARGE_BINARY: return ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt); case arrow::Type::type::STRING: return ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt); diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 3fc90514..0704fd5b 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -237,10 +237,9 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { file_system->Create(dir->Str() + "/file.blob", /*overwrite=*/true)); std::shared_ptr blob_field = BlobUtils::ToArrowField("blob_col"); auto struct_type = arrow::struct_({blob_field}); - bool blob_as_descriptor = GetParam(); ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor, output_stream, struct_type, - file_system, pool_)); + BlobFormatWriter::Create(output_stream, struct_type, + /*write_consumer=*/nullptr, file_system, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index d9e8f4dc..e5e3c8cf 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -23,6 +23,7 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/metrics/metrics_impl.h" @@ -33,23 +34,24 @@ namespace paimon::blob { -BlobFormatWriter::BlobFormatWriter(bool blob_as_descriptor, - const std::shared_ptr& out, +BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, + WriteConsumer write_consumer, const std::shared_ptr& fs, const std::shared_ptr& pool) - : blob_as_descriptor_(blob_as_descriptor), - out_(out), + : out_(out), + uri_(uri), data_type_(data_type), fs_(fs), - pool_(pool) { + pool_(pool), + write_consumer_(std::move(write_consumer)) { metrics_ = std::make_shared(); tmp_buffer_ = Bytes::AllocateBytes(kTmpBufferSize, pool_.get()); } Result> BlobFormatWriter::Create( - bool blob_as_descriptor, const std::shared_ptr& out, - const std::shared_ptr& data_type, const std::shared_ptr& fs, + const std::shared_ptr& out, const std::shared_ptr& data_type, + WriteConsumer write_consumer, const std::shared_ptr& fs, const std::shared_ptr& pool) { if (out == nullptr) { return Status::Invalid("blob format writer create failed. out is nullptr"); @@ -68,8 +70,9 @@ Result> BlobFormatWriter::Create( return Status::Invalid( fmt::format("field {} is not BLOB", data_type->field(0)->ToString())); } + PAIMON_ASSIGN_OR_RAISE(std::string uri, out->GetUri()); return std::unique_ptr( - new BlobFormatWriter(blob_as_descriptor, out, data_type, fs, pool)); + new BlobFormatWriter(out, uri, data_type, std::move(write_consumer), fs, pool)); } Status BlobFormatWriter::AddBatch(ArrowArray* batch) { @@ -93,6 +96,9 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { // Child-level null: record kNullBinLength, skip data writing (aligned with Java) if (child_array->IsNull(0)) { bin_lengths_.push_back(BlobDefs::kNullBinLength); + if (write_consumer_) { + write_consumer_(/*descriptor=*/nullptr); + } return Status::OK(); } @@ -105,7 +111,27 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { assert(blob_array.length() == 1); PAIMON_RETURN_NOT_OK(WriteBlob(blob_array.GetView(0))); - PAIMON_RETURN_NOT_OK(Flush()); + if (write_consumer_) { + // Construct BlobDescriptor from the blob just written. + // blob format: magic(4) + content + bin_length(8) + crc32(4) + // bin_length covers all of the above, so content_length = bin_length - 16. + // The stream is now positioned at the end of crc32, i.e., previous_pos + bin_length. + int64_t bin_length = bin_lengths_.back(); + PAIMON_ASSIGN_OR_RAISE(int64_t end_pos, out_->GetPos()); + int64_t blob_start_pos = end_pos - bin_length; + int64_t content_offset = blob_start_pos + BlobDefs::kContentStartOffset; + int64_t content_length = bin_length - BlobDefs::kTotalMetaLength; + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr descriptor, + BlobDescriptor::Create(uri_, content_offset, content_length)); + bool should_flush = write_consumer_(std::move(descriptor)); + if (should_flush) { + PAIMON_RETURN_NOT_OK(Flush()); + } + } else { + // Java does not flush when writeConsumer is null. + PAIMON_RETURN_NOT_OK(Flush()); + } return Status::OK(); } @@ -140,8 +166,13 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { PAIMON_RETURN_NOT_OK(WriteWithCrc32(kMagicNumberBytes->data(), kMagicNumberBytes->size())); // write blob content + // Dynamically check whether blob_data is a serialized BlobDescriptor (by magic header) + // rather than relying on blob_as_descriptor_ config. This is consistent with Java behavior: + // at write time, the input bytes are auto-detected as descriptor or raw data. std::unique_ptr in; - if (blob_as_descriptor_) { + PAIMON_ASSIGN_OR_RAISE(bool is_descriptor, + BlobDescriptor::IsBlobDescriptor(blob_data.data(), blob_data.size())); + if (is_descriptor) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, Blob::FromDescriptor(blob_data.data(), blob_data.size())); PAIMON_ASSIGN_OR_RAISE(in, blob->NewInputStream(fs_)); diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 542fe26c..7680611d 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include #include @@ -38,6 +39,7 @@ struct ArrowArray; namespace paimon { class Blob; +class BlobDescriptor; class FileSystem; class Metrics; class OutputStream; @@ -49,9 +51,14 @@ namespace paimon::blob { // https://cwiki.apache.org/confluence/display/PAIMON/PIP-35%3A+Introduce+Blob+to+store+multimodal+data class BlobFormatWriter : public FormatWriter { public: + /// Callback invoked after each blob row is written. + /// Receives the BlobDescriptor of the written blob (nullptr for null blobs). + /// Similar to Java's BlobConsumer. Returns true if the output stream should be flushed. + using WriteConsumer = std::function descriptor)>; + static Result> Create( - bool blob_as_descriptor, const std::shared_ptr& out, - const std::shared_ptr& data_type, const std::shared_ptr& fs, + const std::shared_ptr& out, const std::shared_ptr& data_type, + WriteConsumer write_consumer, const std::shared_ptr& fs, const std::shared_ptr& pool); Status AddBatch(ArrowArray* batch) override; @@ -67,9 +74,9 @@ class BlobFormatWriter : public FormatWriter { } private: - BlobFormatWriter(bool blob_as_descriptor, const std::shared_ptr& out, + BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, - const std::shared_ptr& fs, + WriteConsumer write_consumer, const std::shared_ptr& fs, const std::shared_ptr& pool); Status WriteBlob(std::string_view blob_data); @@ -85,15 +92,16 @@ class BlobFormatWriter : public FormatWriter { static constexpr uint32_t kTmpBufferSize = 1024 * 1024; private: - bool blob_as_descriptor_; uint32_t crc32_ = 0; std::vector bin_lengths_; std::shared_ptr out_; + std::string uri_; PAIMON_UNIQUE_PTR tmp_buffer_; std::shared_ptr data_type_; std::shared_ptr fs_; std::shared_ptr pool_; std::shared_ptr metrics_; + WriteConsumer write_consumer_; }; } // namespace paimon::blob diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index 99022259..9a5f013e 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -20,9 +20,11 @@ #include #include +#include #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/stream_utils.h" #include "paimon/data/blob.h" @@ -92,8 +94,8 @@ INSTANTIATE_TEST_SUITE_P(BlobAsDescriptor, BlobFormatWriterTest, ::testing::Valu TEST_P(BlobFormatWriterTest, TestSimple) { // write ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); std::vector> expected_blobs; std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; @@ -151,41 +153,82 @@ TEST_P(BlobFormatWriterTest, TestSimple) { } } +TEST_P(BlobFormatWriterTest, TestWriteConsumerReceivesDescriptors) { + std::vector> captured_descriptors; + BlobFormatWriter::WriteConsumer consumer = + [&captured_descriptors](std::unique_ptr descriptor) -> bool { + captured_descriptors.push_back(std::move(descriptor)); + return true; // request flush + }; + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, consumer, file_system_, pool_)); + + // Write a normal blob row + std::string file = paimon::test::GetDataDir() + "/xxhash.data"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, + Blob::FromPath(file, /*offset=*/0, /*length=*/91)); + ASSERT_OK_AND_ASSIGN(auto array, PrepareBlobArray(blob)); + ASSERT_OK(AddBatchOnce(writer, array)); + + ASSERT_EQ(captured_descriptors.size(), 1); + ASSERT_TRUE(captured_descriptors[0]); + ASSERT_EQ(captured_descriptors[0]->Uri(), dir_->Str() + "/file.blob"); + ASSERT_EQ(captured_descriptors[0]->Offset(), 4); // after magic(4) + ASSERT_EQ(captured_descriptors[0]->Length(), 91); + + // Write a null blob row — consumer should receive nullptr descriptor + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(blob_builder->AppendNull().ok()); + std::shared_ptr null_array; + ASSERT_TRUE(struct_builder.Finish(&null_array).ok()); + ASSERT_OK(AddBatchOnce(writer, null_array)); + + ASSERT_EQ(captured_descriptors.size(), 2); + ASSERT_FALSE(captured_descriptors[1]); + + ASSERT_OK(writer->Finish()); +} + TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) { // Test with nullptr output stream - ASSERT_NOK_WITH_MSG( - BlobFormatWriter::Create(blob_as_descriptor_, nullptr, struct_type_, file_system_, pool_), - "blob format writer create failed. out is nullptr"); + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(nullptr, struct_type_, /*write_consumer=*/nullptr, + file_system_, pool_), + "blob format writer create failed. out is nullptr"); // Test with nullptr data type - ASSERT_NOK_WITH_MSG( - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, nullptr, file_system_, pool_), - "blob format writer create failed. data_type is nullptr"); + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, nullptr, + /*write_consumer=*/nullptr, file_system_, pool_), + "blob format writer create failed. data_type is nullptr"); // Test with nullptr memory pool - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, nullptr), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, nullptr), "blob format writer create failed. pool is nullptr"); // Test with invalid field count (more than 1 field) auto multi_field_type = arrow::struct_( {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, - multi_field_type, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, multi_field_type, + /*write_consumer=*/nullptr, file_system_, pool_), "blob data type field number 2 is not 1"); // Test with non-blob field (missing blob metadata) auto non_blob_field = arrow::field("regular_col", arrow::binary()); auto non_blob_type = arrow::struct_({non_blob_field}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, non_blob_type, - file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, non_blob_type, + /*write_consumer=*/nullptr, file_system_, pool_), "field regular_col: binary is not BLOB"); } TEST_P(BlobFormatWriterTest, TestInvalidCase) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); // Test nullptr batch ASSERT_NOK_WITH_MSG(writer->AddBatch(nullptr), @@ -203,8 +246,8 @@ TEST_P(BlobFormatWriterTest, TestInvalidCase) { TEST_P(BlobFormatWriterTest, TestAddBatchWithInvalidBatchLength) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); // Test batch with wrong length (not 1) arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), @@ -231,8 +274,8 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithInvalidBatchLength) { TEST_P(BlobFormatWriterTest, TestReachTargetSize) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); // Initially should not reach target size ASSERT_OK_AND_ASSIGN(bool reached, writer->ReachTargetSize(true, 1000)); @@ -256,8 +299,8 @@ TEST_P(BlobFormatWriterTest, TestReachTargetSize) { TEST_P(BlobFormatWriterTest, TestGetWriterMetrics) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); auto metrics = writer->GetWriterMetrics(); ASSERT_TRUE(metrics); @@ -266,8 +309,8 @@ TEST_P(BlobFormatWriterTest, TestGetWriterMetrics) { TEST_P(BlobFormatWriterTest, TestEmptyWriter) { // Test creating a writer and finishing without adding any data ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -287,8 +330,8 @@ TEST_P(BlobFormatWriterTest, TestEmptyWriter) { TEST_P(BlobFormatWriterTest, TestLargeBlob) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); // Create a temporary large file for testing std::string large_file_path = dir_->Str() + "/large_test_file.bin"; @@ -342,8 +385,8 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); // Write one row with child-level null blob arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), @@ -390,8 +433,8 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { auto null_c_array = std::make_unique(); ASSERT_TRUE(arrow::ExportArray(*null_struct_array, null_c_array.get()).ok()); ASSERT_OK_AND_ASSIGN(std::shared_ptr writer2, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); ASSERT_NOK_WITH_MSG(writer2->AddBatch(null_c_array.get()), "BlobFormatWriter does not support struct-level null."); ArrowArrayRelease(null_c_array.get()); @@ -399,8 +442,8 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(blob_as_descriptor_, output_stream_, struct_type_, - file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_consumer=*/nullptr, file_system_, pool_)); // Create a zero-length file std::string zero_file_path = dir_->Str() + "/zero_length_file.bin"; diff --git a/src/paimon/format/blob/blob_writer_builder.h b/src/paimon/format/blob/blob_writer_builder.h index 1fc1a58e..d7aae78c 100644 --- a/src/paimon/format/blob/blob_writer_builder.h +++ b/src/paimon/format/blob/blob_writer_builder.h @@ -26,8 +26,6 @@ #include #include "arrow/api.h" -#include "paimon/common/utils/options_utils.h" -#include "paimon/defs.h" #include "paimon/format/blob/blob_format_writer.h" #include "paimon/format/format_writer.h" #include "paimon/format/writer_builder.h" @@ -63,16 +61,19 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { return this; } + /// Sets a write consumer that will be called after each blob row is written. + BlobWriterBuilder* WithWriteConsumer(BlobFormatWriter::WriteConsumer consumer) { + write_consumer_ = std::move(consumer); + return this; + } + Result> Build(const std::shared_ptr& out, const std::string& compression) override { assert(out); if (fs_ == nullptr) { return Status::Invalid("File system is nullptr. Please call WithFileSystem() first."); } - PAIMON_ASSIGN_OR_RAISE( - bool blob_as_descriptor, - OptionsUtils::GetValueFromMap(options_, Options::BLOB_AS_DESCRIPTOR, false)); - return BlobFormatWriter::Create(blob_as_descriptor, out, data_type_, fs_, pool_); + return BlobFormatWriter::Create(out, data_type_, write_consumer_, fs_, pool_); } private: @@ -80,6 +81,7 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { std::shared_ptr data_type_; std::map options_; std::shared_ptr fs_; + BlobFormatWriter::WriteConsumer write_consumer_; }; } // namespace paimon::blob diff --git a/src/paimon/format/blob/blob_writer_builder_test.cpp b/src/paimon/format/blob/blob_writer_builder_test.cpp index 2350eba2..8052a97e 100644 --- a/src/paimon/format/blob/blob_writer_builder_test.cpp +++ b/src/paimon/format/blob/blob_writer_builder_test.cpp @@ -18,9 +18,16 @@ #include "paimon/format/blob/blob_writer_builder.h" +#include + #include "arrow/api.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/defs.h" +#include "paimon/format/format_writer.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/testing/utils/testharness.h" @@ -54,4 +61,34 @@ TEST_F(BlobWriterBuilderTest, TestSimple) { ASSERT_OK(builder.Build(output_stream_, "none")); } +TEST_F(BlobWriterBuilderTest, TestWithWriteConsumer) { + std::vector> captured; + BlobWriterBuilder builder(struct_type_, {{Options::BLOB_AS_DESCRIPTOR, "false"}}); + builder.WithFileSystem(file_system_); + builder.WithWriteConsumer([&captured](std::unique_ptr descriptor) -> bool { + captured.push_back(std::move(descriptor)); + return true; + }); + + ASSERT_OK_AND_ASSIGN(auto writer, builder.Build(output_stream_, "none")); + + // Build a single-row struct array with raw blob data + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(blob_builder->Append("hello", 5).ok()); + std::shared_ptr array; + ASSERT_TRUE(struct_builder.Finish(&array).ok()); + + auto c_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*array, c_array.get()).ok()); + ASSERT_OK(writer->AddBatch(c_array.get())); + + ASSERT_EQ(captured.size(), 1); + ASSERT_TRUE(captured[0]); + ASSERT_EQ(captured[0]->Length(), 5); + ASSERT_OK(writer->Finish()); +} + } // namespace paimon::blob::test diff --git a/src/paimon/format/orc/orc_adapter.cpp b/src/paimon/format/orc/orc_adapter.cpp index 5f76419f..b4f28fb6 100644 --- a/src/paimon/format/orc/orc_adapter.cpp +++ b/src/paimon/format/orc/orc_adapter.cpp @@ -1316,6 +1316,9 @@ arrow::Status WriteBatch(const arrow::Array& array, ::orc::ColumnVectorBatch* co case arrow::Type::type::BINARY: return WriteGenericBatch( array, column_vector_batch); + case arrow::Type::type::LARGE_BINARY: + return WriteGenericBatch( + array, column_vector_batch); case arrow::Type::type::STRING: return WriteGenericBatch( array, column_vector_batch); @@ -1379,6 +1382,7 @@ arrow::Result> GetOrcType(const arrow::DataType& ty case arrow::Type::type::STRING: return ::orc::createPrimitiveType(::orc::TypeKind::STRING); case arrow::Type::type::BINARY: + case arrow::Type::type::LARGE_BINARY: return ::orc::createPrimitiveType(::orc::TypeKind::BINARY); case arrow::Type::type::DATE32: return ::orc::createPrimitiveType(::orc::TypeKind::DATE); diff --git a/src/paimon/format/orc/orc_adapter_test.cpp b/src/paimon/format/orc/orc_adapter_test.cpp index 2678a286..1a43de5f 100644 --- a/src/paimon/format/orc/orc_adapter_test.cpp +++ b/src/paimon/format/orc/orc_adapter_test.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -184,12 +185,13 @@ TEST_F(OrcAdapterTest, TestGetOrcType) { auto col21_field = arrow::field("col21", arrow::timestamp(arrow::TimeUnit::MILLI, timezone)); auto col22_field = arrow::field("col22", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)); auto col23_field = arrow::field("col23", arrow::timestamp(arrow::TimeUnit::NANO, timezone)); + auto col24_field = arrow::field("col24", arrow::large_binary()); auto arrow_schema = std::make_shared(arrow::FieldVector( {col1_field, col2_field, col3_field, col4_field, col5_field, col6_field, col7_field, col8_field, col9_field, col10_field, col11_field, col12_field, col13_field, col14_field, col15_field, col16_field, col17_field, col18_field, - col19_field, col20_field, col21_field, col22_field, col23_field})); + col19_field, col20_field, col21_field, col22_field, col23_field, col24_field})); ASSERT_OK_AND_ASSIGN(std::unique_ptr<::orc::Type> orc_type, OrcAdapter::GetOrcType(*arrow_schema)); ASSERT_TRUE(orc_type); @@ -199,7 +201,7 @@ TEST_F(OrcAdapterTest, TestGetOrcType) { "array,col14:map,col15:timestamp,col16:struct,col17:timestamp,col18:timestamp,col19:timestamp,col20:timestamp " "with local time zone,col21:timestamp with local time zone,col22:timestamp with local time " - "zone,col23:timestamp with local time zone>", + "zone,col23:timestamp with local time zone,col24:binary>", orc_type->toString()); } @@ -209,11 +211,6 @@ TEST_F(OrcAdapterTest, TestGetOrcTypeWithInvalidArrowType) { auto arrow_schema = arrow::schema(arrow::FieldVector({col1_field})); ASSERT_NOK(OrcAdapter::GetOrcType(*arrow_schema)); } - { - auto col1_field = arrow::field("col1", arrow::large_binary()); - auto arrow_schema = arrow::schema(arrow::FieldVector({col1_field})); - ASSERT_NOK(OrcAdapter::GetOrcType(*arrow_schema)); - } { auto col1_field = arrow::field("col1", arrow::uint32()); auto arrow_schema = arrow::schema(arrow::FieldVector({col1_field})); @@ -570,6 +567,39 @@ TEST_P(OrcAdapterTest, TestAppendBatchWithBinaryForAllNull) { ASSERT_TRUE(converted_array->Equals(src_array)) << converted_array->ToString(); } +TEST_P(OrcAdapterTest, TestWriteBatchWithLargeBinary) { + arrow::FieldVector fields = {arrow::field("f0", arrow::large_binary())}; + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + ["descriptor-1"], + [""], + [null], + ["descriptor-2"] + ])") + .ValueOrDie()); + + auto [orc_reader_holder, read_batch] = GenerateOrcReadBatch(src_array); + auto* struct_batch = dynamic_cast<::orc::StructVectorBatch*>(read_batch.get()); + ASSERT_TRUE(struct_batch); + ASSERT_EQ(1, struct_batch->fields.size()); + + auto* large_binary_batch = dynamic_cast<::orc::StringVectorBatch*>(struct_batch->fields[0]); + ASSERT_TRUE(large_binary_batch); + ASSERT_EQ(4, large_binary_batch->numElements); + + std::vector expected_values = {"descriptor-1", "", "descriptor-2"}; + ASSERT_TRUE(large_binary_batch->notNull[0]); + ASSERT_EQ(expected_values[0], + std::string(large_binary_batch->data[0], large_binary_batch->length[0])); + ASSERT_TRUE(large_binary_batch->notNull[1]); + ASSERT_EQ(expected_values[1], + std::string(large_binary_batch->data[1], large_binary_batch->length[1])); + ASSERT_FALSE(large_binary_batch->notNull[2]); + ASSERT_TRUE(large_binary_batch->notNull[3]); + ASSERT_EQ(expected_values[2], + std::string(large_binary_batch->data[3], large_binary_batch->length[3])); +} + TEST_P(OrcAdapterTest, TestDecimalAndTimestamp) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); arrow::FieldVector fields = { diff --git a/src/paimon/format/orc/orc_file_batch_reader_test.cpp b/src/paimon/format/orc/orc_file_batch_reader_test.cpp index afb407ba..a9fe5db9 100644 --- a/src/paimon/format/orc/orc_file_batch_reader_test.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader_test.cpp @@ -196,6 +196,50 @@ INSTANTIATE_TEST_SUITE_P(TestParam, OrcFileBatchReaderTest, ::testing::Values(TestParam{128 * 1024, false}, TestParam{16, false}, TestParam{16, true})); +TEST_F(OrcFileBatchReaderTest, TestReadBinaryWrittenFromBinaryAndLargeBinary) { + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto file_system = dir->GetFileSystem(); + + auto check_binary_read_result = [&](const std::shared_ptr& write_type, + const std::string& file_name) { + std::string data_json = R"([ + ["descriptor-1"], + [""], + [null], + ["descriptor-2"] + ])"; + auto write_field = arrow::field("f0", write_type); + auto write_schema = arrow::schema({write_field}); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({write_field}), data_json) + .ValueOrDie()); + + std::string file_path = dir->Str() + "/" + file_name; + WriteArray(file_system, file_path, write_array, write_schema, /*options=*/{}); + + auto read_field = arrow::field("f0", arrow::binary()); + arrow::Schema read_schema({read_field}); + auto orc_batch_reader = PrepareOrcFileBatchReader(file_path, &read_schema, batch_size_, + DEFAULT_NATURAL_READ_SIZE); + + ASSERT_OK_AND_ASSIGN(auto c_file_schema, orc_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + ASSERT_TRUE(file_schema->Equals(read_schema)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_field}), data_json) + .ValueOrDie()); + auto expected_chunked_array = std::make_shared(expected_array); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + ASSERT_TRUE(result_array->Equals(expected_chunked_array)); + }; + + check_binary_read_result(arrow::binary(), "binary.orc"); + check_binary_read_result(arrow::large_binary(), "large-binary.orc"); +} + TEST_F(OrcFileBatchReaderTest, TestSetReadSchema) { std::string file_name = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/f1=10/bucket-1/" diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index 836c291b..d324a54b 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -163,6 +163,48 @@ class ParquetFileBatchReaderTest : public ::testing::Test, std::shared_ptr struct_array_; }; +TEST_F(ParquetFileBatchReaderTest, TestReadBinaryWrittenFromBinaryAndLargeBinary) { + auto check_binary_read_result = [&](const std::shared_ptr& write_type, + const std::string& file_name) { + std::string data_json = R"([ + ["descriptor-1"], + [""], + [null], + ["descriptor-2"] + ])"; + auto write_field = arrow::field("f0", write_type); + auto write_schema = arrow::schema({write_field}); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({write_field}), data_json) + .ValueOrDie()); + + std::string file_path = PathUtil::JoinPath(dir_->Str(), file_name); + WriteArray(file_path, write_array, write_schema, /*write_batch_size=*/write_array->length(), + /*enable_dictionary=*/false, /*max_row_group_length=*/write_array->length()); + + auto read_field = arrow::field("f0", arrow::binary()); + auto read_schema = arrow::schema({read_field}); + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path, read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, batch_size_); + + ASSERT_OK_AND_ASSIGN(auto c_file_schema, parquet_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + ASSERT_TRUE(file_schema->Equals(*read_schema)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_field}), data_json) + .ValueOrDie()); + auto expected_chunked_array = std::make_shared(expected_array); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + ASSERT_TRUE(result_array->Equals(expected_chunked_array)); + }; + + check_binary_read_result(arrow::binary(), "binary.parquet"); + check_binary_read_result(arrow::large_binary(), "large-binary.parquet"); +} + TEST_F(ParquetFileBatchReaderTest, TestSimple) { std::string file_name = paimon::test::GetDataDir() + "/parquet/parquet_append_table.db/parquet_append_table/bucket-0/" diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index c6a05092..b18a1016 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -241,74 +241,6 @@ class TestHelper { return result_blobs; } - // need to reconstruct the blob array, because the array in read result do not have blob meta - Result> ReconstructBlobArray( - const std::shared_ptr& array, const std::shared_ptr& schema) { - ::ArrowArray c_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); - ::ArrowSchema new_c_schema; - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &new_c_schema)); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto new_array, - arrow::ImportArray(&c_array, &new_c_schema)); - return new_array; - } - - Result ReadAndCheckResultForBlobTable( - const std::shared_ptr& all_columns_schema, - const std::vector>& splits, const std::string& main_expected_json, - const std::vector>& expected_blob_descriptors) { - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options_); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, - read_context_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); - PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); - PAIMON_ASSIGN_OR_RAISE(auto read_result, - ReadResultCollector::CollectResult(batch_reader.get())); - - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto concat_array, - arrow::Concatenate(read_result->chunks())); - PAIMON_ASSIGN_OR_RAISE(auto reconstruct_array, - ReconstructBlobArray(concat_array, all_columns_schema)); - PAIMON_ASSIGN_OR_RAISE( - auto separated_array, - BlobUtils::SeparateBlobArray( - std::dynamic_pointer_cast(reconstruct_array))); - - arrow::EqualOptions equal_options = arrow::EqualOptions::Defaults(); - - // check main columns - auto separated_schema = BlobUtils::SeparateBlobSchema(all_columns_schema); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - auto main_expected_array, - arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_(separated_schema.main_schema->fields()), main_expected_json)); - auto main_expected_chunk_array = std::make_shared(main_expected_array); - bool main_equal = main_expected_chunk_array->Equals( - arrow::ChunkedArray(separated_array.main_array), equal_options.diff_sink(&std::cout)); - if (!main_equal) { - std::cout << "[expected_data_type]" << main_expected_chunk_array->type()->ToString() - << std::endl; - std::cout << "[actual_data_type]" << separated_array.main_array->type()->ToString() - << std::endl; - std::cout << "[expected]:" << main_expected_chunk_array->ToString() << std::endl; - std::cout << "[actual]: " << separated_array.main_array->ToString() << std::endl; - } - - // check blob column - std::vector> expected_blobs; - for (const auto& descriptor : expected_blob_descriptors) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr blob, - Blob::FromDescriptor(descriptor->data(), descriptor->size())); - expected_blobs.emplace_back(blob); - } - PAIMON_ASSIGN_OR_RAISE(auto result_blobs, ToBlobs(separated_array.blob_array)); - PAIMON_ASSIGN_OR_RAISE(bool blob_equal, CheckBlobsEqual(result_blobs, expected_blobs, fs_)); - - table_read.reset(); - return main_equal && blob_equal; - } - Result ReadAndCheckResult(const std::shared_ptr& data_type, const std::vector>& splits, const std::string& expected_result) { diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index ca6df4f1..30e439e6 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -78,11 +79,18 @@ class RecordBatch; } // namespace paimon namespace paimon::test { + +struct ReadResult { + std::unique_ptr batch_reader; + std::shared_ptr chunked_array; +}; + class BlobTableInteTest : public testing::Test, public ::testing::WithParamInterface { public: void SetUp() override { pool_ = GetDefaultPool(); dir_ = UniqueTestDirectory::Create("local"); + blob_dir_ = UniqueTestDirectory::Create("local"); } void TearDown() override { @@ -91,7 +99,13 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter void CreateTable(const std::vector& partition_keys, const std::map& options) const { - auto schema = arrow::schema(fields_); + CreateTable(fields_, partition_keys, options); + } + + void CreateTable(const arrow::FieldVector& fields, + const std::vector& partition_keys, + const std::map& options) const { + auto schema = arrow::schema(fields); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -162,11 +176,10 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter return file_store_commit->Commit(commit_msgs); } - Status ScanAndRead(const std::string& table_path, const std::vector& read_schema, - const std::shared_ptr& expected_array, - const std::shared_ptr& predicate = nullptr, - const std::vector& row_ranges = {}) const { - // scan + /// Scan table and return the plan (without reading data). + Result> ScanTable(const std::string& table_path, + const std::shared_ptr& predicate = nullptr, + const std::vector& row_ranges = {}) const { ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPredicate(predicate); if (!row_ranges.empty()) { @@ -176,47 +189,72 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); - if (!expected_array) { - EXPECT_TRUE(result_plan->Splits().empty()); - } + return result_plan; + } - // read - auto splits = result_plan->Splits(); + /// Read from table using a pre-scanned plan, returning the ChunkedArray and batch_reader. + /// The batch_reader must outlive the returned ChunkedArray (array memory depends on reader). + Result ReadTable(const std::string& table_path, + const std::vector& read_schema, + const std::shared_ptr& plan, + const std::shared_ptr& predicate = nullptr, + const std::map& options = {}) const { + auto splits = plan->Splits(); ReadContextBuilder read_context_builder(table_path); read_context_builder.SetReadSchema(read_schema).SetPredicate(predicate); + if (!options.empty()) { + read_context_builder.SetOptions(options); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); PAIMON_ASSIGN_OR_RAISE(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + return ReadResult{std::move(batch_reader), std::move(read_result)}; + } - if (!expected_array) { - EXPECT_FALSE(read_result); - return Status::OK(); - } - // add row kind array for expected array + /// Convenience: scan + read in one call. + Result ScanAndReadResult(const std::string& table_path, + const std::vector& read_schema, + const std::shared_ptr& predicate = nullptr, + const std::vector& row_ranges = {}) const { + PAIMON_ASSIGN_OR_RAISE(auto result_plan, ScanTable(table_path, predicate, row_ranges)); + return ReadTable(table_path, read_schema, result_plan, predicate); + } + + /// Prepend a _VALUE_KIND (Insert) column to a StructArray. + static Result> PrependRowKindColumn( + const std::shared_ptr& array) { auto row_kind_scalar = std::make_shared(RowKind::Insert()->ToByteValue()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - auto row_kind_array, - arrow::MakeArrayFromScalar(*row_kind_scalar, expected_array->length())); - arrow::ArrayVector expected_with_row_kind_fields = expected_array->fields(); - std::vector expected_with_row_kind_field_names = - arrow::schema(expected_array->type()->fields())->field_names(); - expected_with_row_kind_fields.insert(expected_with_row_kind_fields.begin(), row_kind_array); - expected_with_row_kind_field_names.insert(expected_with_row_kind_field_names.begin(), - "_VALUE_KIND"); - - // check read result + auto row_kind_array, arrow::MakeArrayFromScalar(*row_kind_scalar, array->length())); + arrow::ArrayVector fields_with_row_kind = array->fields(); + std::vector names_with_row_kind = + arrow::schema(array->type()->fields())->field_names(); + fields_with_row_kind.insert(fields_with_row_kind.begin(), row_kind_array); + names_with_row_kind.insert(names_with_row_kind.begin(), "_VALUE_KIND"); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - auto expected_with_row_kind_array, - arrow::StructArray::Make(expected_with_row_kind_fields, - expected_with_row_kind_field_names)); - auto expected_chunk_array = - std::make_shared(expected_with_row_kind_array); - EXPECT_TRUE(expected_chunk_array->Equals(read_result)) - << "result:" << read_result->ToString() << std::endl + auto result, arrow::StructArray::Make(fields_with_row_kind, names_with_row_kind)); + return std::dynamic_pointer_cast(result); + } + + Status ScanAndRead(const std::string& table_path, const std::vector& read_schema, + const std::shared_ptr& expected_array, + const std::shared_ptr& predicate = nullptr, + const std::vector& row_ranges = {}) const { + PAIMON_ASSIGN_OR_RAISE(auto scan_read, + ScanAndReadResult(table_path, read_schema, predicate, row_ranges)); + + if (!expected_array) { + EXPECT_FALSE(scan_read.chunked_array); + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE(auto expected_with_row_kind, PrependRowKindColumn(expected_array)); + auto expected_chunk_array = std::make_shared(expected_with_row_kind); + EXPECT_TRUE(expected_chunk_array->Equals(scan_read.chunked_array)) + << "result:" << scan_read.chunked_array->ToString() << std::endl << "expected:" << expected_chunk_array->ToString(); return Status::OK(); } @@ -238,9 +276,133 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter .ValueOrDie()); } + /// Convert a StructArray with raw blob bytes into a StructArray with serialized + /// BlobDescriptor bytes. Each raw blob value is written to a temporary file, and + /// the corresponding cell is replaced with the serialized BlobDescriptor pointing + /// to that file. + /// Common framework for transforming blob fields in a StructArray. + /// Non-blob fields are kept as-is; blob fields are processed row-by-row via `transform_row`. + /// `transform_row` receives (binary_value_view) and returns the transformed bytes via builder. + using BlobRowTransform = + std::function; + + Result> TransformBlobFields( + const std::shared_ptr& input_array, + const std::set& blob_fields, BlobRowTransform transform_row) const { + auto fields = input_array->type()->fields(); + arrow::ArrayVector child_arrays; + + for (const auto& field : fields) { + auto col = input_array->GetFieldByName(field->name()); + if (blob_fields.count(field->name()) == 0) { + child_arrays.push_back(col); + continue; + } + const auto& binary_array = + arrow::internal::checked_cast(*col); + arrow::LargeBinaryBuilder builder; + for (int64_t i = 0; i < binary_array.length(); ++i) { + if (binary_array.IsNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + continue; + } + PAIMON_RETURN_NOT_OK(transform_row(binary_array.GetView(i), &builder)); + } + std::shared_ptr result_col; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&result_col)); + child_arrays.push_back(result_col); + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto result, + arrow::StructArray::Make(child_arrays, fields)); + return result; + } + + Result> ConvertRawBlobToDescriptor( + const std::shared_ptr& raw_array, + const std::set& blob_fields) { + auto fs = std::make_shared(); + return TransformBlobFields( + raw_array, blob_fields, + [&](const std::string_view& raw_value, arrow::LargeBinaryBuilder* builder) -> Status { + std::string file_path = + blob_dir_->Str() + "/blob_" + std::to_string(blob_file_counter_++) + ".bin"; + PAIMON_ASSIGN_OR_RAISE(auto out, fs->Create(file_path, /*overwrite=*/true)); + PAIMON_ASSIGN_OR_RAISE( + auto written, + out->Write(raw_value.data(), static_cast(raw_value.size()))); + PAIMON_RETURN_NOT_OK(out->Flush()); + PAIMON_RETURN_NOT_OK(out->Close()); + if (static_cast(written) != raw_value.size()) { + return Status::Invalid("Short write: expected {}, wrote {}", raw_value.size(), + written); + } + PAIMON_ASSIGN_OR_RAISE(auto blob, Blob::FromPath(file_path)); + auto descriptor = blob->ToDescriptor(pool_); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + builder->Append(descriptor->data(), descriptor->size())); + return Status::OK(); + }); + } + + /// Convert a StructArray with serialized BlobDescriptor bytes back to a StructArray + /// with raw blob bytes. Only blob fields are resolved; other columns (including + /// _VALUE_KIND) are kept as-is. + Result> ConvertDescriptorToRawBlob( + const std::shared_ptr& desc_array, + const std::set& blob_fields) const { + auto fs = std::make_shared(); + return TransformBlobFields( + desc_array, blob_fields, + [&](const std::string_view& descriptor_bytes, + arrow::LargeBinaryBuilder* builder) -> Status { + PAIMON_ASSIGN_OR_RAISE(auto blob, Blob::FromDescriptor(descriptor_bytes.data(), + descriptor_bytes.size())); + PAIMON_ASSIGN_OR_RAISE(auto data, blob->ToData(fs, pool_)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(data->data(), data->size())); + return Status::OK(); + }); + } + + /// Verify DataFileMeta properties from a scan plan. + /// Each vector element corresponds to one expected DataFileMeta (ordered by file index). + static void VerifyDataFileMetas( + const std::shared_ptr& plan, size_t expected_file_count, + const std::vector& expected_row_counts, + const std::vector& expected_min_seqs, + const std::vector& expected_max_seqs, + const std::vector& expected_first_row_ids, + const std::vector>>& expected_write_cols) { + std::vector> all_files; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + for (const auto& file : data_split->DataFiles()) { + all_files.push_back(file); + } + } + ASSERT_EQ(all_files.size(), expected_file_count); + ASSERT_EQ(expected_row_counts.size(), expected_file_count); + ASSERT_EQ(expected_min_seqs.size(), expected_file_count); + ASSERT_EQ(expected_max_seqs.size(), expected_file_count); + ASSERT_EQ(expected_first_row_ids.size(), expected_file_count); + ASSERT_EQ(expected_write_cols.size(), expected_file_count); + for (size_t i = 0; i < all_files.size(); ++i) { + const auto& file = all_files[i]; + EXPECT_EQ(file->row_count, expected_row_counts[i]); + EXPECT_EQ(file->min_sequence_number, expected_min_seqs[i]); + EXPECT_EQ(file->max_sequence_number, expected_max_seqs[i]); + ASSERT_TRUE(file->first_row_id.has_value()); + EXPECT_EQ(file->first_row_id.value(), expected_first_row_ids[i]); + EXPECT_EQ(file->write_cols, expected_write_cols[i]); + } + } + private: std::shared_ptr pool_; std::unique_ptr dir_; + std::unique_ptr blob_dir_; + int blob_file_counter_ = 0; arrow::FieldVector fields_ = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("f1"), arrow::field("f2", arrow::utf8())}; }; @@ -260,142 +422,75 @@ INSTANTIATE_TEST_SUITE_P(FileFormat, BlobTableInteTest, ::testing::ValuesIn(GetTestValuesForBlobTableInteTest())); TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorTrue) { - auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - auto schema = arrow::schema(fields); - auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - ASSERT_OK_AND_ASSIGN( - auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); - int64_t commit_identifier = 0; - - auto generate_blob_array = [&](const std::vector>& blob_descriptors) - -> std::shared_ptr { - arrow::StructBuilder struct_builder( - arrow::struct_(fields), arrow::default_memory_pool(), - {std::make_shared(), std::make_shared(), - std::make_shared()}); - auto string_builder = static_cast(struct_builder.field_builder(0)); - auto int_builder = static_cast(struct_builder.field_builder(1)); - auto binary_builder = - static_cast(struct_builder.field_builder(2)); - for (size_t i = 0; i < blob_descriptors.size(); ++i) { - EXPECT_TRUE(struct_builder.Append().ok()); - EXPECT_TRUE(string_builder->Append("str_" + std::to_string(i)).ok()); - if (i % 3 == 0) { - // test null - EXPECT_TRUE(int_builder->AppendNull().ok()); - } else { - EXPECT_TRUE(int_builder->Append(i).ok()); - } - EXPECT_TRUE( - binary_builder->Append(blob_descriptors[i]->data(), blob_descriptors[i]->size()) - .ok()); - } - std::shared_ptr array; - EXPECT_TRUE(struct_builder.Finish(&array).ok()); - return array; - }; - - // prepare data - std::vector> expected_blob_descriptors; - std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; - ASSERT_OK_AND_ASSIGN(auto blob1, Blob::FromPath(file1)); - expected_blob_descriptors.emplace_back(blob1->ToDescriptor(pool_)); - - std::string file2 = paimon::test::GetDataDir() + "/xxhash.data"; - ASSERT_OK_AND_ASSIGN(auto blob2, Blob::FromPath(file2, /*offset=*/0, /*length=*/91)); - expected_blob_descriptors.emplace_back(blob2->ToDescriptor(pool_)); - ASSERT_OK_AND_ASSIGN(auto blob3, Blob::FromPath(file2, /*offset=*/92, /*length=*/85)); - expected_blob_descriptors.emplace_back(blob3->ToDescriptor(pool_)); - ASSERT_OK_AND_ASSIGN(auto blob4, Blob::FromPath(file2, /*offset=*/300, /*length=*/3000)); - expected_blob_descriptors.emplace_back(blob4->ToDescriptor(pool_)); - - auto array = generate_blob_array(expected_blob_descriptors); - ::ArrowArray arrow_array; - ASSERT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); - RecordBatchBuilder batch_builder(&arrow_array); - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, batch_builder.Finish()); + // prepare data: input uses plain raw blob bytes for readability + std::string raw_json = R"([ + ["str_0", null, "hello_blob_0"], + ["str_1", 1, "blob_data_1"], + ["str_2", 2, "blob_data_2"], + ["str_3", null, "blob_data_3"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + // write descriptor array + auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), commit_identifier++, - /*expected_commit_messages=*/std::nullopt)); - - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - auto schema_with_row_kind = arrow::schema(fields_with_row_kind); - ASSERT_OK_AND_ASSIGN(std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - std::string expected_data = R"([ - [0, "str_0", null], - [0, "str_1", 1], - [0, "str_2", 2], - [0, "str_3", null] - ])"; - ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResultForBlobTable( - schema_with_row_kind, data_splits, expected_data, - expected_blob_descriptors)); - ASSERT_TRUE(success); + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // read result contains descriptors pointing to paimon internal blob files + // resolve descriptors back to raw bytes, then prepend _VALUE_KIND and compare + ASSERT_OK_AND_ASSIGN(auto result, ScanAndReadResult(table_path, schema->field_names())); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"blob"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); } TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorFalse) { - auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - auto schema = arrow::schema(fields); - auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_AS_DESCRIPTOR, "false"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - ASSERT_OK_AND_ASSIGN( - auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); - int64_t commit_identifier = 0; - - std::string data = R"([ + std::string data_json = R"([ ["str_0", null, "apple"], ["str_1", 1, "banana"], ["str_2", 2, "cat"], ["str_3", null, "dog"] ])"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), data, - /*partition_map=*/{}, /*bucket=*/0, {})); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), data_json).ValueOrDie()); + auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), commit_identifier++, - /*expected_commit_messages=*/std::nullopt)); - - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - auto data_type = arrow::struct_(fields_with_row_kind); - ASSERT_OK_AND_ASSIGN(std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - std::string expected_data = R"([ - [0, "str_0", null, "apple"], - [0, "str_1", 1, "banana"], - [0, "str_2", 2, "cat"], - [0, "str_3", null, "dog"] - ])"; - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(data_type, data_splits, expected_data)); - ASSERT_TRUE(success); + WriteArray(table_path, {}, schema->field_names(), {write_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // BLOB_AS_DESCRIPTOR=false: blob data is stored inline, read result should match input + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), write_array)); } TEST_P(BlobTableInteTest, TestBasic) { @@ -590,7 +685,7 @@ TEST_P(BlobTableInteTest, TestOnlySomeColumns) { ])") .ValueOrDie()); ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, write_cols1, {src_array1}), - "Can't infer struct array length with 0 child arrays"); + "SeparateBlobArray expects at least one main field, but got none."); } TEST_P(BlobTableInteTest, TestMultipleAppendsDifferentFirstRowIds) { @@ -1291,7 +1386,6 @@ TEST_P(BlobTableInteTest, TestWithRowIdsForMultipleBlobFiles) { {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "false"}, {Options::FILE_SYSTEM, "local"}}; CreateTable(/*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -1391,107 +1485,60 @@ TEST_P(BlobTableInteTest, TestWithRowIdsForMultipleBlobFiles) { } TEST_P(BlobTableInteTest, TestAppendTableWriteWithMultipleBlobFields) { - auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = { arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob1", true), BlobUtils::ToArrowField("blob2", true)}; - auto schema = arrow::schema(fields); - auto file_format = GetParam(); std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format}, + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "false"}, {Options::FILE_SYSTEM, "local"}}; - - ASSERT_OK_AND_ASSIGN( - auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); - int64_t commit_identifier = 0; + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - std::string data = R"([ + std::string data_json = R"([ ["str_0", null, "apple", "red"], ["str_1", 1, "banana", "yellow"], ["str_2", 2, "cat", "black"] ])"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), data, - /*partition_map=*/{}, /*bucket=*/0, {})); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), data_json).ValueOrDie()); + auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), commit_identifier++, - /*expected_commit_messages=*/std::nullopt)); - ASSERT_EQ(commit_msgs.size(), 1); - - ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); - ASSERT_TRUE(snapshot); - ASSERT_EQ(1, snapshot.value().Id()); - ASSERT_EQ(3, snapshot.value().NextRowId().value()); - - // Scan and read: verify all fields including multiple blob fields - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - auto data_type = arrow::struct_(fields_with_row_kind); - ASSERT_OK_AND_ASSIGN(std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - std::string expected_data = R"([ - [0, "str_0", null, "apple", "red"], - [0, "str_1", 1, "banana", "yellow"], - [0, "str_2", 2, "cat", "black"] - ])"; - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(data_type, data_splits, expected_data)); - ASSERT_TRUE(success); + WriteArray(table_path, {}, schema->field_names(), {write_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), write_array)); } TEST_P(BlobTableInteTest, TestAppendWriteWithNullBlob) { - auto dir = UniqueTestDirectory::Create(); arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; - auto schema = arrow::schema(fields); - auto file_format = GetParam(); std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, + {Options::FILE_FORMAT, GetParam()}, {Options::BUCKET, "-1"}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_AS_DESCRIPTOR, "false"}}; - - ASSERT_OK_AND_ASSIGN( - auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); // Write: row 0 non-null blob, row 1 null blob, row 2 non-null blob - std::string data = R"([ + std::string data_json = R"([ [1, "hello"], [2, null], [3, "world"] ])"; - ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, - TestHelper::MakeRecordBatch(arrow::struct_(fields), data, - /*partition_map=*/{}, /*bucket=*/0, {})); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), data_json).ValueOrDie()); + + auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN(auto commit_msgs, - helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, - /*expected_commit_messages=*/std::nullopt)); - - // Read and verify - arrow::FieldVector fields_with_row_kind = fields; - fields_with_row_kind.insert(fields_with_row_kind.begin(), - arrow::field("_VALUE_KIND", arrow::int8())); - auto data_type = arrow::struct_(fields_with_row_kind); - ASSERT_OK_AND_ASSIGN(std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - std::string expected_data = R"([ - [0, 1, "hello"], - [0, 2, null], - [0, 3, "world"] - ])"; - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(data_type, data_splits, expected_data)); - ASSERT_TRUE(success); + WriteArray(table_path, {}, schema->field_names(), {write_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), write_array)); } TEST_P(BlobTableInteTest, TestReadTableWithMultiBlobFields) { @@ -1567,4 +1614,767 @@ TEST_P(BlobTableInteTest, TestReadTableWithMultiBlobFields) { } } +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWithoutExternalStorage) { + if (GetParam() == "lance") { + return; + } + // Two blob fields configured via BLOB_DESCRIPTOR_FIELD, no external storage. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Input uses plain raw bytes for readability + std::string raw_json = R"([ + [1, "image_data_0", "video_data_0"], + [2, "image_data_1", "video_data_1"], + [3, "image_data_2", "video_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"b0", "b1"})); + + // write descriptor array + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Scan and verify DataFileMeta: no external storage -> write_cols should be nullopt + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/1, /*expected_row_counts=*/{3}, + /*expected_min_seqs=*/{1}, /*expected_max_seqs=*/{1}, + /*expected_first_row_ids=*/{0}, + /*expected_write_cols=*/{std::nullopt}); + + // Read and resolve descriptors back to raw bytes + std::map read_options = {}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + + // Descriptor bytes should be unchanged (inline, not repacked) + ASSERT_TRUE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); + ASSERT_TRUE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWithExternalStorage) { + if (GetParam() == "lance") { + return; + } + // Two blob fields configured via BLOB_DESCRIPTOR_FIELD + BLOB_EXTERNAL_STORAGE_FIELD + // with BLOB_EXTERNAL_STORAGE_PATH pointing to blob_dir_. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Input uses plain raw bytes for readability + std::string raw_json = R"([ + [1, "image_data_0", "video_data_0"], + [2, "image_data_1", "video_data_1"], + [3, "image_data_2", "video_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"b0", "b1"})); + + // write descriptor array + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Scan and verify DataFileMeta: with external storage -> write_cols should be explicit + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/1, /*expected_row_counts=*/{3}, + /*expected_min_seqs=*/{1}, /*expected_max_seqs=*/{1}, + /*expected_first_row_ids=*/{0}, + /*expected_write_cols=*/{std::vector{"f0", "b0", "b1"}}); + + // Read and resolve descriptors back to raw bytes + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + + // Descriptor bytes should differ (repacked by external storage) + ASSERT_FALSE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); + ASSERT_FALSE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialExternalStorage) { + if (GetParam() == "lance") { + return; + } + // 4 blob fields: b0,b1 have external storage, b2,b3 are descriptor-only (no external storage). + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1,b2,b3"}, + {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Input uses plain raw bytes for readability; some blob fields are null + std::string raw_json = R"([ + [1, "img_0", null, "doc_0", "log_0"], + [2, null, "vid_1", null, "log_1"], + [3, "img_2", "vid_2", "doc_2", null ] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, + ConvertRawBlobToDescriptor(raw_array, {"b0", "b1", "b2", "b3"})); + + // write descriptor array + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Scan and verify DataFileMeta: external storage on b0,b1 -> write_cols should be explicit + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas( + plan, /*expected_file_count=*/1, /*expected_row_counts=*/{3}, + /*expected_min_seqs=*/{1}, /*expected_max_seqs=*/{1}, + /*expected_first_row_ids=*/{0}, + /*expected_write_cols=*/{std::vector{"f0", "b0", "b1", "b2", "b3"}}); + + // Read and resolve all descriptors back to raw bytes + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, + ConvertDescriptorToRawBlob(read_struct, {"b0", "b1", "b2", "b3"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + + // b0,b1 repacked by external storage, should differ + ASSERT_FALSE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); + ASSERT_FALSE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); + // b2,b3 inline descriptor, should match + ASSERT_TRUE(read_struct->GetFieldByName("b2")->Equals(desc_array->GetFieldByName("b2"))); + ASSERT_TRUE(read_struct->GetFieldByName("b3")->Equals(desc_array->GetFieldByName("b3"))); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) { + if (GetParam() == "lance") { + return; + } + // 4 blob fields: b0,b1 are descriptor (inline), b2,b3 are regular blob (written to .blob + // files). No external storage. + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Input uses plain raw bytes: + // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls + std::string raw_json = R"([ + [1, "img_0", null, "raw_2_0", "raw_3_0"], + [2, "img_1", "vid_1", "raw_2_1", null ], + [3, "img_2", null, "raw_2_2", "raw_3_2" ] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, + ConvertRawBlobToDescriptor(raw_array, {"b0", "b1", "b2", "b3"})); + + // write: b0,b1 as descriptor bytes; b2,b3 as raw bytes (paimon writes them to .blob files) + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Scan and verify DataFileMeta: b2,b3 go to .blob files, "f0", "b0", "b1" go to main files. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/3, /*expected_row_counts=*/{3, 3, 3}, + /*expected_min_seqs=*/{1, 1, 1}, /*expected_max_seqs=*/{1, 1, 1}, + /*expected_first_row_ids=*/{0, 0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "b0", "b1"}, std::vector{"b2"}, + std::vector{"b3"}}); + + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // b0,b1 inline descriptor (not repacked), should match input + ASSERT_TRUE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); + ASSERT_TRUE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); + + // Resolve b0,b1 descriptors back to raw bytes, then compare full struct + ASSERT_OK_AND_ASSIGN(auto resolved, + ConvertDescriptorToRawBlob(read_struct, {"b0", "b1", "b2", "b3"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialExternalStorageRepack) { + if (GetParam() == "lance") { + return; + } + // 4 blob fields: b0,b1 are descriptor + external-storage-field WITH external-storage-path. + // b2,b3 are regular blob (written to .blob files). + // All blob descriptors get repacked by external storage or .blob writer. + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls + std::string raw_json = R"([ + [1, "img_0", null, "raw_2_0", "raw_3_0"], + [2, "img_1", "vid_1", "raw_2_1", null ], + [3, "img_2", null, "raw_2_2", "raw_3_2" ] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, + ConvertRawBlobToDescriptor(raw_array, {"b0", "b1", "b2", "b3"})); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // b0,b1 repacked to external storage; b2,b3 go to .blob files. + // Main file contains f0,b0,b1; .blob files for b2 and b3. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/3, /*expected_row_counts=*/{3, 3, 3}, + /*expected_min_seqs=*/{1, 1, 1}, /*expected_max_seqs=*/{1, 1, 1}, + /*expected_first_row_ids=*/{0, 0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "b0", "b1"}, std::vector{"b2"}, + std::vector{"b3"}}); + + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // Resolve descriptors back to raw bytes and compare + ASSERT_OK_AND_ASSIGN(auto resolved, + ConvertDescriptorToRawBlob(read_struct, {"b0", "b1", "b2", "b3"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + + // All blob columns should differ from input desc_array (all repacked) + ASSERT_FALSE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); + ASSERT_FALSE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); + ASSERT_FALSE(read_struct->GetFieldByName("b2")->Equals(desc_array->GetFieldByName("b2"))); + ASSERT_FALSE(read_struct->GetFieldByName("b3")->Equals(desc_array->GetFieldByName("b3"))); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialExternalStorageSingleField) { + if (GetParam() == "lance") { + return; + } + // 4 blob fields: b0,b1 are descriptor; only b1 has external storage. + // b2,b3 are regular blob (written to .blob files). + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, + {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls + std::string raw_json = R"([ + [1, "img_0", null, "raw_2_0", "raw_3_0"], + [2, "img_1", "vid_1", "raw_2_1", null ], + [3, "img_2", null, "raw_2_2", "raw_3_2" ] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, + ConvertRawBlobToDescriptor(raw_array, {"b0", "b1", "b2", "b3"})); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // b1 repacked to external storage; b2,b3 go to .blob files; b0 stays inline in main file. + // Main file contains f0,b0,b1; .blob files for b2 and b3. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/3, /*expected_row_counts=*/{3, 3, 3}, + /*expected_min_seqs=*/{1, 1, 1}, /*expected_max_seqs=*/{1, 1, 1}, + /*expected_first_row_ids=*/{0, 0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "b0", "b1"}, std::vector{"b2"}, + std::vector{"b3"}}); + + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // Resolve all descriptors back to raw bytes and compare + ASSERT_OK_AND_ASSIGN(auto resolved, + ConvertDescriptorToRawBlob(read_struct, {"b0", "b1", "b2", "b3"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + + // b0 is inline descriptor (not repacked), should match input + ASSERT_TRUE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); + // b1 is repacked by external storage, should differ + ASSERT_FALSE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); + // b2,b3 are repacked by .blob writer, should differ + ASSERT_FALSE(read_struct->GetFieldByName("b2")->Equals(desc_array->GetFieldByName("b2"))); + ASSERT_FALSE(read_struct->GetFieldByName("b3")->Equals(desc_array->GetFieldByName("b3"))); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialExternalStorageNoAsDescriptor) { + if (GetParam() == "lance") { + return; + } + // Same as TestBlobDescriptorFieldPartialExternalStorageSingleField but without + // BLOB_AS_DESCRIPTOR in table options. Only b0 is explicitly converted to descriptor before + // write. b1 is written as raw bytes but still configured as descriptor field, so paimon should + // auto-convert it to descriptor internally (write auto-detects descriptor via magic header). + // After read with BLOB_AS_DESCRIPTOR=true, b0 and b1 are both stored as descriptor. + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, + {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls + std::string raw_json = R"([ + [1, "img_0", null, "raw_2_0", "raw_3_0"], + [2, "img_1", "vid_1", "raw_2_1", null ], + [3, "img_2", null, "raw_2_2", "raw_3_2" ] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + // Only convert b0 to descriptor; b1,b2,b3 remain as raw bytes + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"b0"})); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // b1 repacked to external storage; b2,b3 go to .blob files; b0 stays inline in main file. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/3, /*expected_row_counts=*/{3, 3, 3}, + /*expected_min_seqs=*/{1, 1, 1}, /*expected_max_seqs=*/{1, 1, 1}, + /*expected_first_row_ids=*/{0, 0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "b0", "b1"}, std::vector{"b2"}, + std::vector{"b3"}}); + + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // After read, b0 and b1 are both descriptor-stored; resolve all back to raw bytes + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + + // b0 is inline descriptor (not repacked), should match input desc_array + ASSERT_TRUE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { + if (GetParam() == "lance") { + return; + } + // Similar to TestBlobDescriptorFieldPartialExternalStorageNoAsDescriptor but: + // 1. Multiple write+commit rounds + // 2. Read schema is shuffled: b3, b2, b1, b0, f0 + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, + {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // --- First write+commit --- + std::string raw_json_1 = R"([ + [1, "img_0", null, "raw_2_0", "raw_3_0"], + [2, "img_1", "vid_1", "raw_2_1", null ] + ])"; + auto raw_array_1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json_1).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array_1, ConvertRawBlobToDescriptor(raw_array_1, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, + WriteArray(table_path, {}, schema->field_names(), {desc_array_1})); + ASSERT_OK(Commit(table_path, commit_msgs_1)); + + // --- Second write+commit --- + std::string raw_json_2 = R"([ + [3, "img_2", "vid_2", "raw_2_2", "raw_3_2"], + [4, null, "vid_3", "raw_2_3", "raw_3_3"] + ])"; + auto raw_array_2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json_2).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array_2, ConvertRawBlobToDescriptor(raw_array_2, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, + WriteArray(table_path, {}, schema->field_names(), {desc_array_2})); + ASSERT_OK(Commit(table_path, commit_msgs_2)); + + // --- Third write+commit --- + std::string raw_json_3 = R"([ + [5, "img_4", null, "raw_2_4", null ], + [6, "img_5", "vid_5", null, "raw_3_5"] + ])"; + auto raw_array_3 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json_3).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array_3, ConvertRawBlobToDescriptor(raw_array_3, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_3, + WriteArray(table_path, {}, schema->field_names(), {desc_array_3})); + ASSERT_OK(Commit(table_path, commit_msgs_3)); + + // test read + { + // --- Read with shuffled schema: b3, b2, b1, b0, f0 --- + std::vector shuffled_read_schema = {"b3", "b2", "b1", "b0", "f0"}; + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, shuffled_read_schema, plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // Build expected array in shuffled order from all 3 batches + arrow::FieldVector shuffled_fields = { + BlobUtils::ToArrowField("b3", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b0", true), + arrow::field("f0", arrow::int32())}; + std::string expected_json = R"([ + ["raw_3_0", "raw_2_0", null, "img_0", 1], + [null, "raw_2_1", "vid_1", "img_1", 2], + ["raw_3_2", "raw_2_2", "vid_2", "img_2", 3], + ["raw_3_3", "raw_2_3", "vid_3", null, 4], + [null, "raw_2_4", null, "img_4", 5], + ["raw_3_5", null, "vid_5", "img_5", 6] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(shuffled_fields), + expected_json) + .ValueOrDie()); + + // Resolve descriptors (b0, b1 are descriptor fields) back to raw bytes + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + } + { + // test scan and read with GlobalIndexResult + std::vector shuffled_read_schema = {"b3", "b2", "b1", "b0", "f0"}; + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path, /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 3), Range(5, 5)})); + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, shuffled_read_schema, plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // Build expected array in shuffled order from all 3 batches + arrow::FieldVector shuffled_fields = { + BlobUtils::ToArrowField("b3", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b0", true), + arrow::field("f0", arrow::int32())}; + std::string expected_json = R"([ + [null, "raw_2_1", "vid_1", "img_1", 2], + ["raw_3_2", "raw_2_2", "vid_2", "img_2", 3], + ["raw_3_3", "raw_2_3", "vid_3", null, 4], + ["raw_3_5", null, "vid_5", "img_5", 6] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(shuffled_fields), + expected_json) + .ValueOrDie()); + + // Resolve descriptors (b0, b1 are descriptor fields) back to raw bytes + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { + if (GetParam() == "lance") { + return; + } + // Test DataEvolution (split-column write) combined with blob descriptor fields. + // Schema: f0(int32), b0(blob descriptor inline), b1(blob descriptor+external), b2(blob), + // b3(blob) + // Commit 1: file A writes (f0, b2, b3) + // Commit 2: file B writes (f0, b0, b1) with SetFirstRowId(0) + // -> merges with commit 1 + // Commit 3: file A writes (f0, b0, b1, b3) + // Commit 4: file B writes (b0, b1, b3) with SetFirstRowId(3) + // -> merges with commit 3 + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, + {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // --- Commit 1: file A (f0, b2, b3), Commit 2: file B (f0, b0, b1) SetFirstRowId(0) --- + std::string file_a1_json = R"([ + [1, "raw_2_0", "raw_3_0"], + [2, "raw_2_1", null ], + [3, null, "raw_3_2"] + ])"; + arrow::FieldVector file_a1_fields = {fields[0], fields[3], fields[4]}; + auto file_a1_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_a1_fields), file_a1_json) + .ValueOrDie()); + + std::string file_b1_json = R"([ + [1, "img_0", "vid_0"], + [2, "img_1", null ], + [3, "img_2", "vid_2"] + ])"; + arrow::FieldVector file_b1_fields = {fields[0], fields[1], fields[2]}; + auto file_b1_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_b1_fields), file_b1_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto file_b1_desc, ConvertRawBlobToDescriptor(file_b1_array, {"b0"})); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs_a1, + WriteArray(table_path, {}, {"f0", "b2", "b3"}, {file_a1_array})); + ASSERT_OK(Commit(table_path, commit_msgs_a1)); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs_b1, + WriteArray(table_path, {}, {"f0", "b0", "b1"}, {file_b1_desc})); + SetFirstRowId(0, commit_msgs_b1); + ASSERT_OK(Commit(table_path, commit_msgs_b1)); + + // --- Commit 3: file A (f0, b0, b1, b3), Commit 4: file B (b0, b1, b3) SetFirstRowId(3) --- + // Duplicate cols b0, b1, b3: file B (commit 4, newer) takes precedence. + std::string file_a2_json = R"([ + [4, "img_3_old", "vid_3_old", "raw_3_3_old"], + [5, null, "vid_4_old", "raw_3_4_old"], + [6, "img_5_old", null, null ] + ])"; + arrow::FieldVector file_a2_fields = {fields[0], fields[1], fields[2], fields[4]}; + auto file_a2_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_a2_fields), file_a2_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto file_a2_desc, ConvertRawBlobToDescriptor(file_a2_array, {"b0"})); + + std::string file_b2_json = R"([ + ["img_3", "vid_3", "raw_3_3"], + [null, "vid_4", "raw_3_4"], + ["img_5", null, null ] + ])"; + arrow::FieldVector file_b2_fields = {fields[1], fields[2], fields[4]}; + auto file_b2_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_b2_fields), file_b2_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto file_b2_desc, ConvertRawBlobToDescriptor(file_b2_array, {"b0"})); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs_a2, + WriteArray(table_path, {}, {"f0", "b0", "b1", "b3"}, {file_a2_desc})); + ASSERT_OK(Commit(table_path, commit_msgs_a2)); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs_b2, + WriteArray(table_path, {}, {"b0", "b1", "b3"}, {file_b2_desc})); + SetFirstRowId(3, commit_msgs_b2); + ASSERT_OK(Commit(table_path, commit_msgs_b2)); + + // --- Read all data with full schema --- + std::vector read_schema = {"f0", "b0", "b1", "b2", "b3"}; + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, read_schema, plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 6); + + // Expected: round1 all columns present; round2 b2=null, b0/b1/b3 from file B (newer) + std::string expected_json = R"([ + [1, "img_0", "vid_0", "raw_2_0", "raw_3_0"], + [2, "img_1", null, "raw_2_1", null ], + [3, "img_2", "vid_2", null, "raw_3_2" ], + [4, "img_3", "vid_3", null, "raw_3_3" ], + [5, null, "vid_4", null, "raw_3_4" ], + [6, "img_5", null, null, null ] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + + // Resolve descriptors back to raw bytes + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->type()->Equals(expected_with_rk->type())); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); +} + +TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) { + if (GetParam() == "lance") { + return; + } + // Similar to TestBlobDescriptorFieldWithoutExternalStorage but writes raw bytes directly + // without converting to descriptor first. The writer should auto-detect that the data + // is NOT a descriptor (no magic header) and handle it accordingly. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Write raw bytes directly (no ConvertRawBlobToDescriptor) + std::string raw_json = R"([ + [1, "image_data_0", "video_data_0"], + [2, "image_data_1", "video_data_1"], + [3, "image_data_2", "video_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + + auto schema = arrow::schema(fields); + ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, schema->field_names(), {raw_array}), + "BLOB inline field b0 configured by blob-descriptor-field or " + "blob-view-field require values " + "to be a BlobDescriptor or BlobViewStruct."); +} + } // namespace paimon::test From 93db3e3b64b6f84722fa8f783a583ff02293a244 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:37:11 +0800 Subject: [PATCH 019/138] fix: Like::TestString to align with Java LIKE semantics From 5ea204d29a0389d2d01e0046fbd657018e4c552b Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Mon, 1 Jun 2026 21:43:56 +0800 Subject: [PATCH 020/138] chore: update LICENSE with additional PyTorch copyright information --- LICENSE | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/LICENSE b/LICENSE index 8ef61972..dd128552 100644 --- a/LICENSE +++ b/LICENSE @@ -426,6 +426,54 @@ License: BSD-3-Clause, see licenses/LICENSE-pytorch.txt -------------------------------------------------------------------------------- +This product includes code derived from PyTorch TH simd.h. + +* SIMD detection code in third_party/roaring_bitmap/roaring.cpp + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, +Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute +(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, +Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +License: BSD-3-Clause + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories +America and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + This product includes code from cppjieba. * cppjieba patch in cmake_modules/jieba.diff From a15fc72127bf80883c4130fdb465b476908ccd89 Mon Sep 17 00:00:00 2001 From: dalingmeng <49717204+dalingmeng@users.noreply.github.com> Date: Tue, 2 Jun 2026 08:53:35 +0800 Subject: [PATCH 021/138] fix: date validation reject invalid dates instead of silent normalization From c394b0c95d606c8b95c49a72d10275abebc75c16 Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Tue, 2 Jun 2026 09:16:32 +0800 Subject: [PATCH 022/138] fix: fix thread-safe problem for file store path factory From b412c2d534ad44369d1ca6eb98b74a6f3dd1b340 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:59:09 +0800 Subject: [PATCH 023/138] feat: Improve PK MOR read performance with batch queue and cached column metadata --- .../data/columnar/columnar_batch_context.h | 99 ++++++++++++++++++- .../common/data/columnar/columnar_row_ref.h | 35 ++++--- .../async_key_value_producer_and_consumer.cpp | 66 ++++++------- .../async_key_value_producer_and_consumer.h | 10 +- .../io/key_value_data_file_record_reader.cpp | 2 +- .../io/key_value_in_memory_record_reader.cpp | 2 +- src/paimon/core/mergetree/spill_reader.cpp | 2 +- 7 files changed, 153 insertions(+), 63 deletions(-) diff --git a/src/paimon/common/data/columnar/columnar_batch_context.h b/src/paimon/common/data/columnar/columnar_batch_context.h index 2d35c0dd..adbe5416 100644 --- a/src/paimon/common/data/columnar/columnar_batch_context.h +++ b/src/paimon/common/data/columnar/columnar_batch_context.h @@ -17,10 +17,13 @@ */ #pragma once +#include #include +#include #include #include "arrow/array/array_base.h" +#include "arrow/util/bit_util.h" namespace arrow { class StructArray; @@ -29,12 +32,106 @@ class StructArray; namespace paimon { class MemoryPool; +/// Pre-cached column metadata for fast access without virtual function calls or checked_cast. +struct CachedColumnMeta { + /// Null bitmap pointer. nullptr means no nulls (all valid). + const uint8_t* null_bitmap = nullptr; + /// Arrow array offset (for sliced arrays). + int64_t array_offset = 0; + /// For fixed-width types: raw pointer to values buffer (buffer[1]). + /// For variable-length types (STRING/BINARY): raw pointer to data buffer (buffer[2]). + const uint8_t* values_data = nullptr; + /// For variable-length types (STRING/BINARY): raw pointer to offsets buffer (buffer[1]). + const int32_t* offsets = nullptr; + + /// Fast null check: directly reads validity bitmap bit. + inline bool IsNull(int64_t row_id) const { + return null_bitmap != nullptr && + !arrow::bit_util::GetBit(null_bitmap, array_offset + row_id); + } + + /// Fast fixed-width value access (INT8/INT16/INT32/INT64/FLOAT/DOUBLE/DATE32/TIMESTAMP). + template + inline T GetFixed(int64_t row_id) const { + return reinterpret_cast(values_data)[array_offset + row_id]; + } + + /// Fast boolean value access (bit-packed in values buffer). + inline bool GetBool(int64_t row_id) const { + return arrow::bit_util::GetBit(values_data, array_offset + row_id); + } + + /// Fast string_view access for non-dictionary STRING/BINARY columns. + inline std::string_view GetVarLenView(int64_t row_id) const { + int64_t idx = array_offset + row_id; + int32_t start = offsets[idx]; + int32_t length = offsets[idx + 1] - start; + return {reinterpret_cast(values_data) + start, static_cast(length)}; + } +}; + struct ColumnarBatchContext { ColumnarBatchContext(const arrow::ArrayVector& array_vec_in, const std::shared_ptr& pool_in) - : pool(pool_in), array_vec(array_vec_in) {} + : pool(pool_in), array_vec(array_vec_in) { + BuildCachedMeta(); + } std::shared_ptr pool; arrow::ArrayVector array_vec; + /// Pre-cached metadata per column for fast access. + std::vector cached_meta; + + private: + void BuildCachedMeta() { + cached_meta.resize(array_vec.size()); + for (size_t i = 0; i < array_vec.size(); i++) { + const auto* array = array_vec[i].get(); + auto& meta = cached_meta[i]; + meta.array_offset = array->offset(); + + // Cache null bitmap + if (array->null_count() != 0) { + meta.null_bitmap = array->null_bitmap_data(); + } + + // Cache data pointers based on type + const auto& array_data = array->data(); + switch (array->type_id()) { + case arrow::Type::BOOL: + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::DATE32: + case arrow::Type::INT64: + case arrow::Type::FLOAT: + case arrow::Type::DOUBLE: { + // Fixed-width: values in buffer[1] + if (array_data->buffers.size() > 1 && array_data->buffers[1]) { + meta.values_data = array_data->buffers[1]->data(); + } + break; + } + case arrow::Type::STRING: + case arrow::Type::BINARY: { + // Variable-length: offsets in buffer[1], data in buffer[2] + if (array_data->buffers.size() > 2) { + if (array_data->buffers[1]) { + meta.offsets = + reinterpret_cast(array_data->buffers[1]->data()); + } + if (array_data->buffers[2]) { + meta.values_data = array_data->buffers[2]->data(); + } + } + break; + } + default: + // TIMESTAMP, DECIMAL, DICTIONARY, LIST, MAP, STRUCT — not cached, use array_vec + // fallback + break; + } + } + } }; } // namespace paimon diff --git a/src/paimon/common/data/columnar/columnar_row_ref.h b/src/paimon/common/data/columnar/columnar_row_ref.h index b08fdf6e..45582c26 100644 --- a/src/paimon/common/data/columnar/columnar_row_ref.h +++ b/src/paimon/common/data/columnar/columnar_row_ref.h @@ -38,10 +38,11 @@ namespace paimon { class Bytes; /// Columnar row view which shares batch-level context to reduce per-row overhead. +/// Uses pre-cached column metadata for fast field access without virtual function calls. class ColumnarRowRef : public InternalRow { public: ColumnarRowRef(std::shared_ptr ctx, int64_t row_id) - : ctx_(std::move(ctx)), row_id_(row_id) {} + : ctx_(std::move(ctx)), cached_meta_ptr_(ctx_->cached_meta.data()), row_id_(row_id) {} Result GetRowKind() const override { return row_kind_; @@ -56,47 +57,39 @@ class ColumnarRowRef : public InternalRow { } bool IsNullAt(int32_t pos) const override { - return ctx_->array_vec[pos]->IsNull(row_id_); + return cached_meta_ptr_[pos].IsNull(row_id_); } bool GetBoolean(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetBool(row_id_); } char GetByte(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return static_cast(cached_meta_ptr_[pos].GetFixed(row_id_)); } int16_t GetShort(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } int32_t GetInt(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } int32_t GetDate(int32_t pos) const override { - return ColumnarUtils::GetGenericValue( - ctx_->array_vec[pos].get(), row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } int64_t GetLong(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } float GetFloat(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } double GetDouble(int32_t pos) const override { - return ColumnarUtils::GetGenericValue(ctx_->array_vec[pos].get(), - row_id_); + return cached_meta_ptr_[pos].GetFixed(row_id_); } BinaryString GetString(int32_t pos) const override { @@ -106,6 +99,11 @@ class ColumnarRowRef : public InternalRow { } std::string_view GetStringView(int32_t pos) const override { + auto& meta = cached_meta_ptr_[pos]; + if (meta.values_data && meta.offsets) { + return meta.GetVarLenView(row_id_); + } + // Fallback for dictionary-encoded or uncached types return ColumnarUtils::GetView(ctx_->array_vec[pos].get(), row_id_); } @@ -130,6 +128,7 @@ class ColumnarRowRef : public InternalRow { private: std::shared_ptr ctx_; + const CachedColumnMeta* cached_meta_ptr_; const RowKind* row_kind_ = RowKind::Insert(); int64_t row_id_; }; diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp index fc8b54a6..1792b43c 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.cpp +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.cpp @@ -40,7 +40,7 @@ AsyncKeyValueProducerAndConsumer::AsyncKeyValueProducerAndConsumer( pool_(pool), sort_merge_reader_(std::move(sort_merge_reader)), create_consumer_(std::move(create_consumer)) { - kv_queue_.set_capacity(batch_size); + kv_queue_.set_capacity(consumer_thread_num * 2); result_queue_.set_capacity(RESULT_BATCH_COUNT); } @@ -81,8 +81,8 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { Result>> consumer = create_consumer_(); PAIMON_RETURN_NOT_OK(consumer.status()); auto async_consumer = std::make_unique>( - batch_size_ / consumer_thread_num_, std::move(consumer).value(), consume_finished_, - consumer_finished_count_, kv_queue_, result_queue_); + std::move(consumer).value(), consume_finished_, consumer_finished_count_, kv_queue_, + result_queue_); consumers_.push_back(std::move(async_consumer)); } } @@ -109,24 +109,33 @@ Result AsyncKeyValueProducerAndConsumer::NextBatch() { template Status AsyncKeyValueProducerAndConsumer::ProduceLoop() { + std::vector batch; + batch.reserve(batch_size_); while (!consume_finished_) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, sort_merge_reader_->NextBatch()); if (iterator == nullptr) { - // all iterator is all visited - kv_queue_.push(std::nullopt); break; } while (!consume_finished_) { PAIMON_ASSIGN_OR_RAISE(bool has_next, iterator->HasNext()); if (!has_next) { - // current iterator is all visited break; } - std::optional kv = std::move(iterator->Next()); - kv_queue_.push(std::move(kv)); + batch.push_back(std::move(iterator->Next())); + if (static_cast(batch.size()) >= batch_size_) { + kv_queue_.push(std::move(batch)); + batch = std::vector(); + batch.reserve(batch_size_); + } } } + // Push remaining rows + if (!batch.empty()) { + kv_queue_.push(std::move(batch)); + } + // Push empty batch as EOF signal + kv_queue_.push(std::vector()); return Status::OK(); } @@ -159,8 +168,8 @@ void AsyncKeyValueProducerAndConsumer::CleanUpQueue() { } } - std::optional kv; - while (kv_queue_.try_pop(kv)) { + std::vector kv_batch; + while (kv_queue_.try_pop(kv_batch)) { } } @@ -169,12 +178,11 @@ template class AsyncKeyValueProducerAndConsumer; template AsyncKeyValueConsumer::AsyncKeyValueConsumer( - int32_t batch_size, std::unique_ptr>&& key_value_consumer, + std::unique_ptr>&& key_value_consumer, std::atomic& consume_finished, std::atomic& consumer_finished_count, - tbb::concurrent_bounded_queue>& kv_queue, + tbb::concurrent_bounded_queue>& kv_queue, tbb::concurrent_bounded_queue& result_queue) - : batch_size_(batch_size), - key_value_consumer_(std::move(key_value_consumer)), + : key_value_consumer_(std::move(key_value_consumer)), consume_finished_(consume_finished), consumer_finished_count_(consumer_finished_count), kv_queue_(kv_queue), @@ -197,30 +205,18 @@ Status AsyncKeyValueConsumer::GetStatus() const { template Status AsyncKeyValueConsumer::ConsumeLoop() { while (!consume_finished_) { - int32_t cur_batch_size = 0; std::vector key_value_vec; - key_value_vec.reserve(batch_size_); - while (!consume_finished_) { - std::optional kv; - if (!kv_queue_.try_pop(kv)) { - usleep(1); - continue; - } - if (!kv) { - consume_finished_ = true; - break; - } - key_value_vec.push_back(std::move(kv).value()); - cur_batch_size++; - if (cur_batch_size >= batch_size_) { - break; - } + if (!kv_queue_.try_pop(key_value_vec)) { + usleep(100); + continue; } - - if (cur_batch_size > 0) { - PAIMON_ASSIGN_OR_RAISE(R result, key_value_consumer_->NextBatch(key_value_vec)); - result_queue_.push(std::move(result)); + if (key_value_vec.empty()) { + // Empty batch is EOF signal; re-push for other consumers + kv_queue_.push(std::move(key_value_vec)); + break; } + PAIMON_ASSIGN_OR_RAISE(R result, key_value_consumer_->NextBatch(key_value_vec)); + result_queue_.push(std::move(result)); } consumer_finished_count_++; return Status::OK(); diff --git a/src/paimon/core/io/async_key_value_producer_and_consumer.h b/src/paimon/core/io/async_key_value_producer_and_consumer.h index 235b2912..af8bbed6 100644 --- a/src/paimon/core/io/async_key_value_producer_and_consumer.h +++ b/src/paimon/core/io/async_key_value_producer_and_consumer.h @@ -94,18 +94,17 @@ class AsyncKeyValueProducerAndConsumer { std::shared_future producer_future_; std::vector>> consumers_; std::atomic consumer_finished_count_ = 0; - tbb::concurrent_bounded_queue> kv_queue_; + tbb::concurrent_bounded_queue> kv_queue_; tbb::concurrent_bounded_queue result_queue_; }; template class AsyncKeyValueConsumer { public: - AsyncKeyValueConsumer(int32_t batch_size, - std::unique_ptr>&& key_value_consumer, + AsyncKeyValueConsumer(std::unique_ptr>&& key_value_consumer, std::atomic& consume_finished, std::atomic& consumer_finished_count, - tbb::concurrent_bounded_queue>& kv_queue, + tbb::concurrent_bounded_queue>& kv_queue, tbb::concurrent_bounded_queue& result_queue); ~AsyncKeyValueConsumer() { @@ -119,12 +118,11 @@ class AsyncKeyValueConsumer { Status ConsumeLoop(); private: - int32_t batch_size_; std::unique_ptr> key_value_consumer_; std::shared_future consumer_future_; std::atomic& consume_finished_; std::atomic& consumer_finished_count_; - tbb::concurrent_bounded_queue>& kv_queue_; + tbb::concurrent_bounded_queue>& kv_queue_; tbb::concurrent_bounded_queue& result_queue_; }; diff --git a/src/paimon/core/io/key_value_data_file_record_reader.cpp b/src/paimon/core/io/key_value_data_file_record_reader.cpp index bccd730f..0ad68585 100644 --- a/src/paimon/core/io/key_value_data_file_record_reader.cpp +++ b/src/paimon/core/io/key_value_data_file_record_reader.cpp @@ -70,7 +70,7 @@ Result KeyValueDataFileRecordReader::Iterator::HasNext() const { Result KeyValueDataFileRecordReader::Iterator::Next() { // key is only used in merge sort; key context does not hold parent struct array - auto key = std::make_unique(reader_->key_ctx_, cursor_); + std::shared_ptr key = std::make_shared(reader_->key_ctx_, cursor_); // value is used in merge sort and projection (maybe async and multi-thread), so value context // holds parent struct array to ensure data remains valid auto value = std::make_unique(reader_->value_ctx_, cursor_); diff --git a/src/paimon/core/io/key_value_in_memory_record_reader.cpp b/src/paimon/core/io/key_value_in_memory_record_reader.cpp index 65aa5cba..43af1b54 100644 --- a/src/paimon/core/io/key_value_in_memory_record_reader.cpp +++ b/src/paimon/core/io/key_value_in_memory_record_reader.cpp @@ -47,7 +47,7 @@ Result KeyValueInMemoryRecordReader::Iterator::Next() { } // key must hold value_struct_array as min/max key may be used after projection - auto key = std::make_unique(reader_->key_ctx_, index); + std::shared_ptr key = std::make_shared(reader_->key_ctx_, index); auto value = std::make_unique(reader_->value_ctx_, index); return KeyValue(row_kind, reader_->last_sequence_num_ + index, /*level=*/KeyValue::UNKNOWN_LEVEL, std::move(key), std::move(value)); diff --git a/src/paimon/core/mergetree/spill_reader.cpp b/src/paimon/core/mergetree/spill_reader.cpp index 0b338c29..f4bb9c1d 100644 --- a/src/paimon/core/mergetree/spill_reader.cpp +++ b/src/paimon/core/mergetree/spill_reader.cpp @@ -78,7 +78,7 @@ Result SpillReader::Iterator::Next() { PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind, RowKind::FromByteValue(reader_->row_kind_array_->Value(cursor_))); int64_t sequence_number = reader_->sequence_number_array_->Value(cursor_); - auto key = std::make_unique(reader_->key_ctx_, cursor_); + std::shared_ptr key = std::make_shared(reader_->key_ctx_, cursor_); auto value = std::make_unique(reader_->value_ctx_, cursor_); cursor_++; return KeyValue(row_kind, sequence_number, KeyValue::UNKNOWN_LEVEL, std::move(key), From 17b7c2a3453c0731eee1f0c5d984de8e652179bf Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Tue, 2 Jun 2026 20:58:45 +0800 Subject: [PATCH 024/138] fix: avoid signed integer overflow UB in BloomFilter and DeltaVarintCompressor From 996be78af48a753e2bf3cdffb1525e0a51d64ee8 Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Wed, 3 Jun 2026 14:39:05 +0800 Subject: [PATCH 025/138] feat: optimize count(*) for pk/append table Co-authored-by: dalingmeng --- include/paimon/reader/count_reader.h | 36 ++ include/paimon/table/source/table_read.h | 7 + src/paimon/CMakeLists.txt | 5 + .../core/deletionvectors/deletion_vector.cpp | 17 + .../core/deletionvectors/deletion_vector.h | 8 + .../deletionvectors/deletion_vector_test.cpp | 34 ++ .../core/mergetree/row_count_accumulator.cpp | 69 ++++ .../core/mergetree/row_count_accumulator.h | 50 +++ .../core/operation/abstract_split_read.cpp | 23 -- .../core/operation/abstract_split_read.h | 10 - .../core/operation/internal_read_context.cpp | 9 + .../core/operation/internal_read_context.h | 8 + .../core/operation/merge_file_split_read.cpp | 12 +- .../core/operation/merge_file_split_read.h | 5 + .../core/operation/raw_file_split_read.cpp | 3 +- .../core/table/source/append_count_reader.cpp | 81 +++++ .../core/table/source/append_count_reader.h | 55 +++ .../table/source/append_count_reader_test.cpp | 146 ++++++++ .../table/source/append_only_table_read.cpp | 16 +- .../table/source/append_only_table_read.h | 4 + .../core/table/source/data_split_impl.cpp | 78 +++- .../core/table/source/data_split_impl.h | 27 +- .../core/table/source/data_split_test.cpp | 335 ++++++++++++++++-- .../table/source/data_table_batch_scan.cpp | 9 +- .../table/source/key_value_table_read.cpp | 34 +- .../core/table/source/key_value_table_read.h | 11 +- .../core/table/source/pk_count_reader.cpp | 143 ++++++++ .../core/table/source/pk_count_reader.h | 71 ++++ .../table/source/pk_count_reader_test.cpp | 175 +++++++++ src/paimon/core/table/source/table_read.cpp | 6 + test/inte/scan_and_read_inte_test.cpp | 131 ++++++- 31 files changed, 1535 insertions(+), 83 deletions(-) create mode 100644 include/paimon/reader/count_reader.h create mode 100644 src/paimon/core/mergetree/row_count_accumulator.cpp create mode 100644 src/paimon/core/mergetree/row_count_accumulator.h create mode 100644 src/paimon/core/table/source/append_count_reader.cpp create mode 100644 src/paimon/core/table/source/append_count_reader.h create mode 100644 src/paimon/core/table/source/append_count_reader_test.cpp create mode 100644 src/paimon/core/table/source/pk_count_reader.cpp create mode 100644 src/paimon/core/table/source/pk_count_reader.h create mode 100644 src/paimon/core/table/source/pk_count_reader_test.cpp diff --git a/include/paimon/reader/count_reader.h b/include/paimon/reader/count_reader.h new file mode 100644 index 00000000..88603e34 --- /dev/null +++ b/include/paimon/reader/count_reader.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// Reader abstraction for count queries. +class PAIMON_EXPORT CountReader { + public: + virtual ~CountReader() = default; + + /// Count rows for splits bound by the corresponding CreateCountReader call. + virtual Result CountRows() = 0; +}; + +} // namespace paimon diff --git a/include/paimon/table/source/table_read.h b/include/paimon/table/source/table_read.h index 157561f9..3bb6610a 100644 --- a/include/paimon/table/source/table_read.h +++ b/include/paimon/table/source/table_read.h @@ -26,6 +26,7 @@ #include "paimon/memory/memory_pool.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" +#include "paimon/reader/count_reader.h" #include "paimon/result.h" #include "paimon/table/source/split.h" #include "paimon/visibility.h" @@ -66,6 +67,12 @@ class PAIMON_EXPORT TableRead { virtual Result> CreateReader( const std::shared_ptr& split) = 0; + /// Creates a `CountReader` for count queries on the specified splits. + /// + /// Implementations may override this to provide a more efficient count path. + virtual Result> CreateCountReader( + const std::vector>& splits); + protected: explicit TableRead(const std::shared_ptr& memory_pool); diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index b1702a9b..a47e8c00 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -266,6 +266,7 @@ set(PAIMON_CORE_SRCS core/mergetree/lookup_file.cpp core/mergetree/lookup_levels.cpp core/mergetree/lookup/remote_lookup_file_manager.cpp + core/mergetree/row_count_accumulator.cpp core/migrate/file_meta_utils.cpp core/operation/data_evolution_file_store_scan.cpp core/operation/data_evolution_split_read.cpp @@ -307,6 +308,7 @@ set(PAIMON_CORE_SRCS core/table/sink/commit_message.cpp core/table/sink/commit_message_impl.cpp core/table/sink/commit_message_serializer.cpp + core/table/source/append_count_reader.cpp core/table/source/append_only_table_read.cpp core/table/source/split.cpp core/table/source/data_split_impl.cpp @@ -314,6 +316,7 @@ set(PAIMON_CORE_SRCS core/table/source/data_table_stream_scan.cpp core/table/source/fallback_table_read.cpp core/table/source/key_value_table_read.cpp + core/table/source/pk_count_reader.cpp core/table/source/merge_tree_split_generator.cpp core/table/source/data_evolution_split_generator.cpp core/table/source/plan_impl.cpp @@ -705,6 +708,8 @@ if(PAIMON_BUILD_TESTS) core/table/sink/commit_message_impl_test.cpp core/table/source/fallback_data_split_test.cpp core/table/source/table_read_test.cpp + core/table/source/append_count_reader_test.cpp + core/table/source/pk_count_reader_test.cpp core/table/source/data_split_test.cpp core/table/source/deletion_file_test.cpp core/table/source/split_generator_test.cpp diff --git a/src/paimon/core/deletionvectors/deletion_vector.cpp b/src/paimon/core/deletionvectors/deletion_vector.cpp index 82e347d7..b34a1a32 100644 --- a/src/paimon/core/deletionvectors/deletion_vector.cpp +++ b/src/paimon/core/deletionvectors/deletion_vector.cpp @@ -18,6 +18,7 @@ */ #include "paimon/core/deletionvectors/deletion_vector.h" +#include #include #include @@ -61,6 +62,22 @@ DeletionVector::Factory DeletionVector::CreateFactory( }; } +std::unordered_map DeletionVector::CreateDeletionFileMap( + const std::vector>& data_files, + const std::vector>& deletion_files) { + std::unordered_map deletion_file_map; + if (deletion_files.empty()) { + return deletion_file_map; + } + assert(deletion_files.size() == data_files.size()); + for (size_t i = 0; i < deletion_files.size(); i++) { + if (deletion_files[i] != std::nullopt) { + deletion_file_map.emplace(data_files[i]->file_name, deletion_files[i].value()); + } + } + return deletion_file_map; +} + Result> DeletionVector::DeserializeFromBytes(const Bytes* bytes, MemoryPool* pool) { return BitmapDeletionVector::Deserialize(bytes->data(), bytes->size(), pool); diff --git a/src/paimon/core/deletionvectors/deletion_vector.h b/src/paimon/core/deletionvectors/deletion_vector.h index bc53af2e..e2757c63 100644 --- a/src/paimon/core/deletionvectors/deletion_vector.h +++ b/src/paimon/core/deletionvectors/deletion_vector.h @@ -51,6 +51,14 @@ class DeletionVector { static Factory CreateFactory(const std::shared_ptr& dv_maintainer); + /// Builds a map from data file name to its deletion file. + /// + /// Entries whose deletion file is absent are skipped. Returns an empty map when + /// `deletion_files` is empty. + static std::unordered_map CreateDeletionFileMap( + const std::vector>& data_files, + const std::vector>& deletion_files); + virtual ~DeletionVector() = default; /// Marks the row at the specified position as deleted. diff --git a/src/paimon/core/deletionvectors/deletion_vector_test.cpp b/src/paimon/core/deletionvectors/deletion_vector_test.cpp index e8742cd3..194a5c33 100644 --- a/src/paimon/core/deletionvectors/deletion_vector_test.cpp +++ b/src/paimon/core/deletionvectors/deletion_vector_test.cpp @@ -21,12 +21,16 @@ #include #include #include +#include +#include #include #include #include "gtest/gtest.h" #include "paimon/core/deletionvectors/bitmap64_deletion_vector.h" #include "paimon/core/deletionvectors/bitmap_deletion_vector.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/table/source/deletion_file.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/io/byte_order.h" #include "paimon/io/data_input_stream.h" @@ -43,6 +47,16 @@ void AppendInt32BigEndian(std::vector* bytes, int32_t value) { bytes->push_back(static_cast(value & 0xFF)); } +std::shared_ptr CreateDataFileMeta(const std::string& file_name) { + return std::make_shared( + file_name, /*file_size=*/100, /*row_count=*/10, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0, + DataFileMeta::DUMMY_LEVEL, std::vector>{}, Timestamp(0, 0), + std::nullopt, nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, + std::nullopt); +} + } // namespace TEST(DeletionVectorTest, TestSimple) { @@ -163,4 +177,24 @@ TEST(DeletionVectorTest, ReadFromDataInputStreamInvalidMagicNumber) { "Invalid magic number"); } +TEST(DeletionVectorTest, CreateDeletionFileMap) { + std::vector> data_files = {CreateDataFileMeta("file-0.orc"), + CreateDataFileMeta("file-1.orc"), + CreateDataFileMeta("file-2.orc")}; + + auto empty_map = DeletionVector::CreateDeletionFileMap(data_files, {}); + ASSERT_TRUE(empty_map.empty()); + + DeletionFile deletion_file_0("dv-0", /*offset=*/10, /*length=*/20, /*cardinality=*/3); + DeletionFile deletion_file_2("dv-2", /*offset=*/30, /*length=*/40, std::nullopt); + std::vector> deletion_files = {deletion_file_0, std::nullopt, + deletion_file_2}; + + auto deletion_file_map = DeletionVector::CreateDeletionFileMap(data_files, deletion_files); + ASSERT_EQ(deletion_file_map.size(), 2); + ASSERT_EQ(deletion_file_map.at("file-0.orc"), deletion_file_0); + ASSERT_EQ(deletion_file_map.count("file-1.orc"), 0); + ASSERT_EQ(deletion_file_map.at("file-2.orc"), deletion_file_2); +} + } // namespace paimon::test diff --git a/src/paimon/core/mergetree/row_count_accumulator.cpp b/src/paimon/core/mergetree/row_count_accumulator.cpp new file mode 100644 index 00000000..8cbc04fb --- /dev/null +++ b/src/paimon/core/mergetree/row_count_accumulator.cpp @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/mergetree/row_count_accumulator.h" + +#include + +#include "paimon/core/key_value.h" + +namespace paimon { + +RowCountAccumulator::RowCountAccumulator(std::unique_ptr&& merged_reader) + : merged_reader_(std::move(merged_reader)) {} + +Result RowCountAccumulator::CountAll() { + int64_t count = 0; + + while (true) { + // Get next batch of merged KV iterators + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iter, + merged_reader_->NextBatch()); + if (iter == nullptr) { + // No more data + break; + } + + // Iterate through all KV objects in this batch + while (true) { + PAIMON_ASSIGN_OR_RAISE(bool has_next, iter->HasNext()); + if (!has_next) { + break; + } + + iter->Next(); + + // At this point: + // - kv has passed through SortMergeReader (deduplicated, merged) + // - kv has passed through DropDeleteReader (kind is guaranteed IsAdd()) + // - kv represents a final, valid, non-deleted row + count++; + } + } + + return count; +} + +void RowCountAccumulator::Close() { + if (merged_reader_) { + merged_reader_->Close(); + } +} + +} // namespace paimon diff --git a/src/paimon/core/mergetree/row_count_accumulator.h b/src/paimon/core/mergetree/row_count_accumulator.h new file mode 100644 index 00000000..7e5bf98e --- /dev/null +++ b/src/paimon/core/mergetree/row_count_accumulator.h @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/mergetree/compact/sort_merge_reader.h" +#include "paimon/result.h" + +namespace paimon { + +/// Counts rows from a merged KeyValue stream after delete rows are dropped. +class RowCountAccumulator { + public: + /// @param merged_reader The merged reader. Must be wrapped with DropDeleteReader + /// so that only valid (non-deleted) KeyValue objects are output. + explicit RowCountAccumulator(std::unique_ptr&& merged_reader); + + ~RowCountAccumulator() = default; + + /// Count all valid rows from the merge reader. + /// Iterates through all merged+deduplicated+non-deleted KeyValue objects. + Result CountAll(); + + /// Close underlying readers and release resources. + void Close(); + + private: + std::unique_ptr merged_reader_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index e20b79a0..b00a08ef 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -104,29 +104,6 @@ bool AbstractSplitRead::NeedCompleteRowTrackingFields( } return false; } - -std::unordered_map AbstractSplitRead::CreateDeletionFileMap( - const DataSplitImpl& data_split) { - return CreateDeletionFileMap(data_split.DataFiles(), data_split.DeletionFiles()); -} - -std::unordered_map AbstractSplitRead::CreateDeletionFileMap( - const std::vector>& data_files, - const std::vector>& deletion_files) { - std::unordered_map deletion_file_map; - if (deletion_files.empty()) { - return deletion_file_map; - } - assert(deletion_files.size() == data_files.size()); - size_t file_count = deletion_files.size(); - for (size_t i = 0; i < file_count; i++) { - if (deletion_files[i] != std::nullopt) { - deletion_file_map.emplace(data_files[i]->file_name, deletion_files[i].value()); - } - } - return deletion_file_map; -} - Result> AbstractSplitRead::ApplyPredicateFilterIfNeeded( std::unique_ptr&& reader, const std::shared_ptr& predicate) const { if (!context_->EnablePredicateFilter() || predicate == nullptr) { diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index ed5dda98..276b5277 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include "arrow/type_fwd.h" @@ -32,7 +31,6 @@ #include "paimon/core/operation/split_read.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/table/source/data_split_impl.h" -#include "paimon/core/table/source/deletion_file.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/format/reader_builder.h" #include "paimon/reader/batch_reader.h" @@ -75,14 +73,6 @@ class AbstractSplitRead : public SplitRead { std::unique_ptr&& schema_manager, const std::shared_ptr& memory_pool, const std::shared_ptr& executor); - - static std::unordered_map CreateDeletionFileMap( - const DataSplitImpl& data_split); - - static std::unordered_map CreateDeletionFileMap( - const std::vector>& data_files, - const std::vector>& deletion_files); - Result> ApplyPredicateFilterIfNeeded( std::unique_ptr&& reader, const std::shared_ptr& predicate) const; diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index fb0f36b4..26f5f27e 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -118,4 +118,13 @@ InternalReadContext::InternalReadContext(const std::shared_ptr& rea read_schema_(read_schema), options_(options) {} +Result> InternalReadContext::CreateWithSchema( + const std::shared_ptr& original, + const std::shared_ptr& new_read_schema) { + // Create a new InternalReadContext sharing all properties except read_schema. + // The new read_schema is the minimal column set for COUNT(*). + return std::shared_ptr(new InternalReadContext( + original->read_context_, original->table_schema_, new_read_schema, original->options_)); +} + } // namespace paimon diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index c1990264..0b60d8ce 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -100,6 +100,14 @@ class InternalReadContext { return read_context_->GetCacheConfig(); } + /// Create a new InternalReadContext with a different read schema. + /// Useful for creating a context with a minimal column set for specialized reads. + /// All other settings (predicate, options, table_schema, etc.) are inherited + /// from the original context. + static Result> CreateWithSchema( + const std::shared_ptr& original, + const std::shared_ptr& new_read_schema); + private: InternalReadContext(const std::shared_ptr& read_context, const std::shared_ptr& table_schema, diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp index ca6c1d57..09c2e0d5 100644 --- a/src/paimon/core/operation/merge_file_split_read.cpp +++ b/src/paimon/core/operation/merge_file_split_read.cpp @@ -235,8 +235,10 @@ Result> MergeFileSplitRead::ApplyIndexAndDvRead Result> MergeFileSplitRead::CreateMergeReader( const std::shared_ptr& data_split, const std::shared_ptr& data_file_path_factory) { - auto dv_factory = DeletionVector::CreateFactory(options_.GetFileSystem(), - CreateDeletionFileMap(*data_split), pool_); + auto dv_factory = DeletionVector::CreateFactory( + options_.GetFileSystem(), + DeletionVector::CreateDeletionFileMap(data_split->DataFiles(), data_split->DeletionFiles()), + pool_); std::vector> sections = IntervalPartition(data_split->DataFiles(), key_comparator_).Partition(); @@ -257,8 +259,10 @@ Result> MergeFileSplitRead::CreateMergeReader( Result> MergeFileSplitRead::CreateNoMergeReader( const std::shared_ptr& data_split, bool only_filter_key, const std::shared_ptr& data_file_path_factory) const { - auto dv_factory = DeletionVector::CreateFactory(options_.GetFileSystem(), - CreateDeletionFileMap(*data_split), pool_); + auto dv_factory = DeletionVector::CreateFactory( + options_.GetFileSystem(), + DeletionVector::CreateDeletionFileMap(data_split->DataFiles(), data_split->DeletionFiles()), + pool_); // create read schema without extra fields (e.g., completed key, sequence fields) auto row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind()); diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h index f00883b0..89145f80 100644 --- a/src/paimon/core/operation/merge_file_split_read.h +++ b/src/paimon/core/operation/merge_file_split_read.h @@ -107,6 +107,11 @@ class MergeFileSplitRead : public AbstractSplitRead { return path_factory_; } + /// Get the key comparator (needed by count readers for IntervalPartition). + std::shared_ptr GetKeyComparator() const { + return key_comparator_; + } + std::shared_ptr GetValueSchema() const { return value_schema_; } diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index c06fb055..b6c9f25a 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -98,7 +98,8 @@ Result> RawFileSplitRead::CreateReader( const std::vector>& data_files, const std::vector>& deletion_files) { auto dv_factory = DeletionVector::CreateFactory( - options_.GetFileSystem(), CreateDeletionFileMap(data_files, deletion_files), pool_); + options_.GetFileSystem(), DeletionVector::CreateDeletionFileMap(data_files, deletion_files), + pool_); return CreateReader(partition, bucket, data_files, dv_factory); } diff --git a/src/paimon/core/table/source/append_count_reader.cpp b/src/paimon/core/table/source/append_count_reader.cpp new file mode 100644 index 00000000..8f9af90a --- /dev/null +++ b/src/paimon/core/table/source/append_count_reader.cpp @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/append_count_reader.h" + +#include "paimon/core/deletionvectors/deletion_vector.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/status.h" + +namespace paimon { + +Result AppendCountReader::CountRows() { + int64_t total = 0; + for (const auto& split : splits_) { + PAIMON_ASSIGN_OR_RAISE(int64_t split_count, CountSingleSplit(split)); + total += split_count; + } + return total; +} + +Result AppendCountReader::CountSingleSplit(const std::shared_ptr& split) const { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("split cannot be cast to DataSplitImpl"); + } + + if (data_split->DataFiles().empty()) { + return 0; + } + + return MetadataCount(data_split); +} + +Result AppendCountReader::MetadataCount( + const std::shared_ptr& split) const { + if (split->RawConvertible()) { + if (!file_system_ || !pool_) { + return Status::Invalid( + "file_system or memory_pool is null for DV-based append count fallback"); + } + + DeletionVector::Factory dv_factory = DeletionVector::CreateFactory( + file_system_, + DeletionVector::CreateDeletionFileMap(split->DataFiles(), split->DeletionFiles()), + pool_); + + PAIMON_ASSIGN_OR_RAISE(std::optional merged_count, + split->MergedRowCount(dv_factory)); + if (merged_count.has_value()) { + return merged_count.value(); + } + } else { + // Non-raw-convertible splits are typically produced by data evolution when multiple files + // overlap on row-id ranges. Count them through data-evolution metadata instead of using a + // deletion-vector factory for raw file counts. + PAIMON_ASSIGN_OR_RAISE(std::optional merged_count, split->MergedRowCount()); + if (merged_count.has_value()) { + return merged_count.value(); + } + } + + return Status::Invalid("not support split in append count fallback"); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/append_count_reader.h b/src/paimon/core/table/source/append_count_reader.h new file mode 100644 index 00000000..e6c6a096 --- /dev/null +++ b/src/paimon/core/table/source/append_count_reader.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/count_reader.h" +#include "paimon/result.h" + +namespace paimon { +class DataSplitImpl; +class Split; + +class AppendCountReader : public CountReader { + public: + explicit AppendCountReader(std::vector> splits, + const std::shared_ptr& file_system, + const std::shared_ptr& pool) + : splits_(std::move(splits)), file_system_(file_system), pool_(pool) {} + + Result CountRows() override; + + private: + Result CountSingleSplit(const std::shared_ptr& split) const; + Result MetadataCount(const std::shared_ptr& split) const; + + private: + std::vector> splits_; + std::shared_ptr file_system_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/append_count_reader_test.cpp b/src/paimon/core/table/source/append_count_reader_test.cpp new file mode 100644 index 00000000..3b191a3d --- /dev/null +++ b/src/paimon/core/table/source/append_count_reader_test.cpp @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/append_count_reader.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/scan_context.h" +#include "paimon/status.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/split.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +class DummySplit final : public Split {}; + +} // namespace + +class AppendCountReaderTest : public testing::Test { + protected: + Result>> CreateSplits( + const std::string& table_path, const std::optional& snapshot_id) { + ScanContextBuilder scan_context_builder(table_path); + if (snapshot_id.has_value()) { + scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, + std::to_string(snapshot_id.value())); + } + + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); + if (snapshot_id.has_value() && + (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id)) { + return Status::Invalid("snapshot id mismatch"); + } + return plan->Splits(); + } + + std::shared_ptr file_system_ = std::make_shared(); + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(AppendCountReaderTest, TestCountRowsSnapshot1) { + std::string table_path = GetDataDir() + "/orc/append_09.db/append_09"; + + ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/1)); + AppendCountReader count_reader(splits, file_system_, pool_); + + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader.CountRows()); + ASSERT_EQ(count, 5); +} + +TEST_F(AppendCountReaderTest, TestCountRowsSnapshot5) { + std::string table_path = GetDataDir() + "/orc/append_09.db/append_09"; + + ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5)); + AppendCountReader count_reader(splits, file_system_, pool_); + + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader.CountRows()); + ASSERT_EQ(count, 11); +} + +TEST_F(AppendCountReaderTest, TestCountRowsDataEvolutionTable) { + std::string table_path = + GetDataDir() + "/orc/data_evolution_with_dense_stats.db/data_evolution_with_dense_stats"; + + ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/2)); + ASSERT_FALSE(splits.empty()); + + bool has_non_raw_convertible_split = false; + for (const auto& split : splits) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + has_non_raw_convertible_split |= !data_split->RawConvertible(); + } + ASSERT_TRUE(has_non_raw_convertible_split); + + AppendCountReader count_reader(splits, file_system_, pool_); + + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader.CountRows()); + ASSERT_EQ(count, 2); +} + +TEST_F(AppendCountReaderTest, TestCountRowsWithEmptySplits) { + std::vector> empty_splits; + AppendCountReader count_reader(empty_splits, file_system_, pool_); + + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader.CountRows()); + ASSERT_EQ(count, 0); +} + +TEST_F(AppendCountReaderTest, TestCountRowsWithInvalidSplit) { + std::vector> splits = {std::make_shared()}; + AppendCountReader count_reader(splits, file_system_, pool_); + + ASSERT_NOK_WITH_MSG(count_reader.CountRows(), "split cannot be cast to DataSplitImpl"); +} + +TEST_F(AppendCountReaderTest, TestAppendOnlyTableReadCreateCountReaderPredicateNotSupported) { + std::string table_path = GetDataDir() + "/orc/append_09.db/append_09"; + + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", + FieldType::DOUBLE, Literal(13.0)); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + + ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5)); + ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(splits), + "predicate pushdown is not supported"); +} + +} // namespace paimon::test diff --git a/src/paimon/core/table/source/append_only_table_read.cpp b/src/paimon/core/table/source/append_only_table_read.cpp index b51dc839..eb6498c1 100644 --- a/src/paimon/core/table/source/append_only_table_read.cpp +++ b/src/paimon/core/table/source/append_only_table_read.cpp @@ -19,10 +19,13 @@ #include "paimon/core/table/source/append_only_table_read.h" +#include + #include "paimon/core/core_options.h" #include "paimon/core/operation/data_evolution_split_read.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/core/operation/raw_file_split_read.h" +#include "paimon/core/table/source/append_count_reader.h" #include "paimon/status.h" namespace paimon { @@ -35,7 +38,7 @@ AppendOnlyTableRead::AppendOnlyTableRead(const std::shared_ptr& context, const std::shared_ptr& memory_pool, const std::shared_ptr& executor) - : TableRead(memory_pool) { + : TableRead(memory_pool), context_(context) { const auto& core_options = context->GetCoreOptions(); if (core_options.DataEvolutionEnabled()) { // add data evolution first @@ -58,4 +61,15 @@ Result> AppendOnlyTableRead::CreateReader( return Status::Invalid("create reader failed, not read match with split."); } +Result> AppendOnlyTableRead::CreateCountReader( + const std::vector>& splits) { + if (context_->GetPredicate() != nullptr) { + return Status::NotImplemented( + "CreateCountReader with predicate pushdown is not supported yet"); + } + + return std::make_unique(splits, context_->GetCoreOptions().GetFileSystem(), + GetMemoryPool()); +} + } // namespace paimon diff --git a/src/paimon/core/table/source/append_only_table_read.h b/src/paimon/core/table/source/append_only_table_read.h index b911deca..e26b8613 100644 --- a/src/paimon/core/table/source/append_only_table_read.h +++ b/src/paimon/core/table/source/append_only_table_read.h @@ -47,8 +47,12 @@ class AppendOnlyTableRead : public TableRead { Result> CreateReader( const std::shared_ptr& data_split) override; + Result> CreateCountReader( + const std::vector>& splits) override; + private: std::vector> split_reads_; + std::shared_ptr context_; }; } // namespace paimon diff --git a/src/paimon/core/table/source/data_split_impl.cpp b/src/paimon/core/table/source/data_split_impl.cpp index 020b5018..67abe964 100644 --- a/src/paimon/core/table/source/data_split_impl.cpp +++ b/src/paimon/core/table/source/data_split_impl.cpp @@ -24,6 +24,9 @@ #include #include +#include "paimon/common/utils/range_helper.h" +#include "paimon/core/deletionvectors/deletion_vector.h" + namespace paimon { bool DataSplit::SimpleDataFileMeta::operator==(const SimpleDataFileMeta& other) const { @@ -112,18 +115,83 @@ bool DataSplitImpl::TEST_Equal(const DataSplitImpl& other) const { is_streaming_ == other.is_streaming_ && raw_convertible_ == other.raw_convertible_; } -int64_t DataSplitImpl::PartialMergedRowCount() const { +Result> DataSplitImpl::MergedRowCount() const { + return MergedRowCount(nullptr); +} + +Result> DataSplitImpl::MergedRowCount( + DeletionVector::Factory dv_factory) const { + PAIMON_ASSIGN_OR_RAISE(std::optional raw_merged_row_count, + RawMergedRowCount(dv_factory)); + if (raw_merged_row_count.has_value()) { + return raw_merged_row_count; + } + if (DataEvolutionRowCountAvailable()) { + PAIMON_ASSIGN_OR_RAISE(int64_t merged_row_count, DataEvolutionMergedRowCount()); + return std::optional(merged_row_count); + } + return std::optional(); +} + +Result> DataSplitImpl::RawMergedRowCount( + DeletionVector::Factory dv_factory) const { if (!raw_convertible_) { - return 0; + return std::optional(); } + int64_t sum = 0; for (size_t i = 0; i < data_files_.size(); i++) { const auto& data_file = data_files_[i]; - if (data_deletion_files_.empty() || data_deletion_files_[i] == std::nullopt) { + const std::optional deletion_file = + data_deletion_files_.empty() ? std::nullopt : data_deletion_files_[i]; + if (deletion_file == std::nullopt) { sum += data_file->row_count; - } else if (data_deletion_files_[i].value().cardinality != std::nullopt) { - sum += data_file->row_count - data_deletion_files_[i].value().cardinality.value(); + } else if (deletion_file.value().cardinality != std::nullopt) { + sum += data_file->row_count - deletion_file.value().cardinality.value(); + } else { + if (!dv_factory) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr deletion_vector, + dv_factory(data_file->file_name)); + if (!deletion_vector) { + return Status::Invalid( + "deletion vector not found for file with missing cardinality"); + } + sum += data_file->row_count - deletion_vector->GetCardinality(); + } + } + + return std::optional(sum); +} + +bool DataSplitImpl::DataEvolutionRowCountAvailable() const { + return std::all_of(data_files_.begin(), data_files_.end(), + [](const std::shared_ptr& file) { + return file->first_row_id != std::nullopt; + }); +} + +Result DataSplitImpl::DataEvolutionMergedRowCount() const { + std::vector> files = data_files_; + RangeHelper> range_helper( + [](const std::shared_ptr& meta) -> Result { + return meta->first_row_id.value(); + }, + [](const std::shared_ptr& meta) -> Result { + return meta->first_row_id.value() + meta->row_count - 1; + }); + + PAIMON_ASSIGN_OR_RAISE(std::vector>> ranges, + range_helper.MergeOverlappingRanges(std::move(files))); + + int64_t sum = 0; + for (const auto& group : ranges) { + int64_t max_count = 0; + for (const auto& file : group) { + max_count = std::max(max_count, file->row_count); } + sum += max_count; } return sum; } diff --git a/src/paimon/core/table/source/data_split_impl.h b/src/paimon/core/table/source/data_split_impl.h index ce1124ab..05076ab7 100644 --- a/src/paimon/core/table/source/data_split_impl.h +++ b/src/paimon/core/table/source/data_split_impl.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -28,6 +29,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/preconditions.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/deletionvectors/deletion_vector.h" #include "paimon/core/io/data_file_meta_09_serializer.h" #include "paimon/core/io/data_file_meta_10_serializer.h" #include "paimon/core/io/data_file_meta_12_serializer.h" @@ -37,6 +39,7 @@ #include "paimon/table/source/data_split.h" namespace paimon { + /// Input splits. Needed by most batch computation engines. class DataSplitImpl : public DataSplit { public: @@ -96,14 +99,22 @@ class DataSplitImpl : public DataSplit { bool operator==(const DataSplitImpl& other) const; bool TEST_Equal(const DataSplitImpl& other) const; - /// Obtain merged row count as much as possible. There are two scenarios where accurate row - /// count - /// can be calculated: + /// Obtain merged row count when metadata is sufficient. + /// + /// This method follows Java DataSplit#mergedRowCount behavior: /// - /// 1. raw file and no deletion file. + /// 1. Prefer raw merged row count when split is raw-convertible and deletion cardinality is + /// available. + /// 2. Fallback to data-evolution merged row count when all files have first_row_id. /// - /// 2. raw file + deletion file with cardinality. - int64_t PartialMergedRowCount() const; + /// Obtain merged row count without reading deletion vector files for missing cardinality. + Result> MergedRowCount() const; + + /// Obtain merged row count with a deletion vector factory for missing cardinality. + /// + /// When a deletion file exists but its cardinality metadata is missing, the factory can be + /// used to read the deletion vector file and provide exact cardinality. + Result> MergedRowCount(DeletionVector::Factory dv_factory) const; // Builder /// Builder for `DataSplitImpl`. @@ -177,6 +188,10 @@ class DataSplitImpl : public DataSplit { bucket_path_(bucket_path), data_files_(std::move(data_files)) {} + Result> RawMergedRowCount(DeletionVector::Factory dv_factory) const; + bool DataEvolutionRowCountAvailable() const; + Result DataEvolutionMergedRowCount() const; + private: int64_t snapshot_id_ = 0; BinaryRow partition_ = BinaryRow::EmptyRow(); diff --git a/src/paimon/core/table/source/data_split_test.cpp b/src/paimon/core/table/source/data_split_test.cpp index 528e19cd..6e69b038 100644 --- a/src/paimon/core/table/source/data_split_test.cpp +++ b/src/paimon/core/table/source/data_split_test.cpp @@ -28,6 +28,8 @@ #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" +#include "paimon/core/deletionvectors/bitmap_deletion_vector.h" +#include "paimon/core/deletionvectors/deletion_vector.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/stats/simple_stats.h" @@ -415,7 +417,9 @@ TEST(DataSplitTest, TestDeserializeVersion5PkWithEmptyExternalPath) { builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value()); ASSERT_EQ(*result_data_split, *expected_data_split); - ASSERT_EQ(5, expected_data_split->PartialMergedRowCount()); + ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, + expected_data_split->MergedRowCount()); + ASSERT_EQ(std::optional(5), merged_row_count); ASSERT_OK(Split::Serialize(result_data_split, pool)); } @@ -477,7 +481,9 @@ TEST(DataSplitTest, TestDeserializeVersion4PkWithSnapshot4WithDvCardinality) { .value()); ASSERT_EQ(*result_data_split, *expected_data_split); - ASSERT_EQ(5, expected_data_split->PartialMergedRowCount()); + ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, + expected_data_split->MergedRowCount()); + ASSERT_EQ(std::optional(5), merged_row_count); } TEST(DataSplitTest, TestDeserializeVersion3AppendWithSnapshot1) { @@ -816,7 +822,9 @@ TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfSingleFile) { .Build() .value()); ASSERT_EQ(*result_data_split, *expected_data_split); - ASSERT_EQ(0, expected_data_split->PartialMergedRowCount()); + ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, + expected_data_split->MergedRowCount()); + ASSERT_EQ(std::nullopt, merged_row_count); } TEST(DataSplitTest, TestDeserializePkWithSnapshot6OfMultiFiles) { @@ -1051,16 +1059,8 @@ TEST(DataSplitTest, TestDeserializePk10WithSnapshot6) { TEST(DataSplitTest, TestPartialMergedRowCount) { auto pool = GetDefaultPool(); auto file_meta = std::make_shared( - "data-0.orc", /*file_size=*/100, /*row_count=*/2, - /*min_key=*/ - BinaryRowGenerator::GenerateRow({std::string("Alice"), 1}, pool.get()), /*max_key=*/ - BinaryRowGenerator::GenerateRow({std::string("David"), 1}, pool.get()), - /*key_stats=*/ - BinaryRowGenerator::GenerateStats({std::string("Alice"), 1}, {std::string("David"), 1}, - {0, 0}, pool.get()), /*value_stats=*/ - BinaryRowGenerator::GenerateStats({std::string("Alice"), 10, 1, 11.0}, - {std::string("David"), 10, 1, 11.1}, {0, 0, 0, 0}, - pool.get()), + "data-0.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_sequence_number=*/0, /*max_sequence_number=*/1, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1725562946338ll, 0), @@ -1069,16 +1069,8 @@ TEST(DataSplitTest, TestPartialMergedRowCount) { /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); auto file_meta2 = std::make_shared( - "data-1.orc", /*file_size=*/100, /*row_count=*/2, - /*min_key=*/ - BinaryRowGenerator::GenerateRow({std::string("Bob"), 1}, pool.get()), /*max_key=*/ - BinaryRowGenerator::GenerateRow({std::string("David"), 1}, pool.get()), - /*key_stats=*/ - BinaryRowGenerator::GenerateStats({std::string("Bob"), 1}, {std::string("David"), 1}, - {0, 0}, pool.get()), /*value_stats=*/ - BinaryRowGenerator::GenerateStats({std::string("Bob"), 10, 1, 11.0}, - {std::string("David"), 10, 1, 11.1}, {0, 0, 0, 0}, - pool.get()), + "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_sequence_number=*/2, /*max_sequence_number=*/3, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1725562947338ll, 0), @@ -1098,7 +1090,9 @@ TEST(DataSplitTest, TestPartialMergedRowCount) { .RawConvertible(false) .Build() .value()); - ASSERT_EQ(0, expected_data_split->PartialMergedRowCount()); + ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, + expected_data_split->MergedRowCount()); + ASSERT_EQ(std::nullopt, merged_row_count); ASSERT_EQ(4, expected_data_split->RowCount()); ASSERT_OK_AND_ASSIGN(auto latest_epoch, expected_data_split->LatestFileCreationEpochMillis()); @@ -1106,6 +1100,299 @@ TEST(DataSplitTest, TestPartialMergedRowCount) { ASSERT_EQ(file_meta2->CreationTimeEpochMillis().value(), latest_epoch.value()); } +TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithoutDeletionFiles) { + auto pool = GetDefaultPool(); + auto file_meta1 = std::make_shared( + "data-0.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/2, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562946338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + auto file_meta2 = std::make_shared( + "data-1.orc", /*file_size=*/100, /*row_count=*/4, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/3, /*max_sequence_number=*/6, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562947338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + + DataSplitImpl::Builder builder( + /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), + /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", {file_meta1, file_meta2}); + + auto data_split = std::dynamic_pointer_cast( + builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value()); + + // Java parity: rawConvertible and deletion files absent should return row_count sum. + ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount()); + ASSERT_EQ(std::optional(7), merged_row_count); +} + +TEST(DataSplitTest, TestPartialMergedRowCountRawConvertibleWithCardinality) { + auto pool = GetDefaultPool(); + auto file_meta1 = std::make_shared( + "data-0.orc", /*file_size=*/100, /*row_count=*/7, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/6, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562946338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + auto file_meta2 = std::make_shared( + "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/7, /*max_sequence_number=*/8, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562947338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + auto file_meta3 = std::make_shared( + "data-2.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/9, /*max_sequence_number=*/11, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562948338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + + DataSplitImpl::Builder builder( + /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), + /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", + {file_meta1, file_meta2, file_meta3}); + + auto data_split = std::dynamic_pointer_cast( + builder.WithSnapshot(1) + .WithDataDeletionFiles({std::nullopt, + DeletionFile("fake/index-0", /*offset=*/1, /*length=*/22, + /*cardinality=*/2), + DeletionFile("fake/index-0", /*offset=*/31, /*length=*/22, + /*cardinality=*/1)}) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + // Java parity: null + cardinality + cardinality => (7 + (2 - 2) + (3 - 1)) = 9. + ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount()); + ASSERT_EQ(std::optional(9), merged_row_count); +} + +TEST(DataSplitTest, TestPartialMergedRowCountMixedCardinalityReturnsNullopt) { + auto pool = GetDefaultPool(); + auto file_meta1 = std::make_shared( + "data-0.orc", /*file_size=*/100, /*row_count=*/7, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/6, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562946338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + auto file_meta2 = std::make_shared( + "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/7, /*max_sequence_number=*/8, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562947338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + + DataSplitImpl::Builder builder( + /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), + /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", {file_meta1, file_meta2}); + + auto data_split = std::dynamic_pointer_cast( + builder.WithSnapshot(1) + .WithDataDeletionFiles({DeletionFile("fake/index-0", /*offset=*/1, /*length=*/22, + /*cardinality=*/2), + DeletionFile("fake/index-0", /*offset=*/31, /*length=*/22, + /*cardinality=*/std::nullopt)}) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + // If any deletion file misses cardinality, metadata count is unknown and must not be partial. + ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount()); + ASSERT_EQ(std::nullopt, merged_row_count); +} + +TEST(DataSplitTest, TestPartialMergedRowCountUnknownDeleteRowCountDoesNotBlockRawCount) { + auto pool = GetDefaultPool(); + auto file_meta1 = std::make_shared( + "data-0.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/2, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562946338ll, 0), + /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + auto file_meta2 = std::make_shared( + "data-1.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/3, /*max_sequence_number=*/4, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562947338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + + DataSplitImpl::Builder builder( + /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), + /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", {file_meta1, file_meta2}); + + auto data_split = std::dynamic_pointer_cast( + builder.WithSnapshot(1) + .WithDataDeletionFiles({std::nullopt, std::nullopt}) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount()); + ASSERT_EQ(std::optional(5), merged_row_count); +} + +TEST(DataSplitTest, TestPartialMergedRowCountFallsBackToDataEvolution) { + auto pool = GetDefaultPool(); + auto file_meta1 = std::make_shared( + "data-0.orc", /*file_size=*/100, /*row_count=*/3, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/2, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562946338ll, 0), + /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/100, + /*write_cols=*/std::nullopt); + auto file_meta2 = std::make_shared( + "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/3, /*max_sequence_number=*/7, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562947338ll, 0), + /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/100, + /*write_cols=*/std::nullopt); + auto file_meta3 = std::make_shared( + "data-2.orc", /*file_size=*/100, /*row_count=*/2, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/8, /*max_sequence_number=*/9, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562948338ll, 0), + /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/200, + /*write_cols=*/std::nullopt); + + DataSplitImpl::Builder builder( + /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), + /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", + {file_meta1, file_meta2, file_meta3}); + + auto data_split = std::dynamic_pointer_cast( + builder.WithSnapshot(1) + .WithDataDeletionFiles({std::nullopt, std::nullopt, std::nullopt}) + .IsStreaming(false) + .RawConvertible(false) + .Build() + .value()); + + // [100,102] and [100,104] overlap -> max row_count is 5; [200,201] is separate -> 2. + ASSERT_OK_AND_ASSIGN(std::optional merged_row_count, data_split->MergedRowCount()); + ASSERT_EQ(std::optional(7), merged_row_count); +} + +// Covers the refactored path where a deletion file exists but its cardinality metadata is +// missing (nullopt). In that case MergedRowCount must call the provided dv_factory to read the +// deletion vector and derive the exact cardinality, instead of returning nullopt. +TEST(DataSplitTest, TestPartialMergedRowCountResolvesMissingCardinalityViaFactory) { + auto pool = GetDefaultPool(); + auto file_meta1 = std::make_shared( + "data-0.orc", /*file_size=*/100, /*row_count=*/7, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/6, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562946338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + auto file_meta2 = std::make_shared( + "data-1.orc", /*file_size=*/100, /*row_count=*/5, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/7, /*max_sequence_number=*/11, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(1725562947338ll, 0), + /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + + DataSplitImpl::Builder builder( + /*partition=*/BinaryRowGenerator::GenerateRow({10}, pool.get()), + /*bucket=*/0, /*bucket_path=*/"fake_table/f1=10/bucket-0", {file_meta1, file_meta2}); + + // The second data file has a deletion file whose cardinality metadata is missing (nullopt), + // which forces resolution through the dv_factory. + auto data_split = std::dynamic_pointer_cast( + builder.WithSnapshot(1) + .WithDataDeletionFiles({std::nullopt, DeletionFile("fake/index-0", /*offset=*/0, + /*length=*/22, /*cardinality=*/ + std::nullopt)}) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + // Without a factory, the missing cardinality keeps the result unknown. + ASSERT_OK_AND_ASSIGN(std::optional merged_without_factory, + data_split->MergedRowCount()); + ASSERT_EQ(std::nullopt, merged_without_factory); + + // Build a deletion vector with 2 deleted rows; the factory returns it for the data file whose + // cardinality is missing. + RoaringBitmap32 roaring; + roaring.Add(1); + roaring.Add(3); + auto deletion_vector = std::make_shared(roaring); + ASSERT_EQ(2, deletion_vector->GetCardinality()); + + DeletionVector::Factory dv_factory = + [&deletion_vector]( + const std::string& file_name) -> Result> { + if (file_name == "data-1.orc") { + return std::static_pointer_cast(deletion_vector); + } + return std::shared_ptr(); + }; + + // file_meta1: 7 (no deletion file) + file_meta2: 5 - factory_cardinality(2) = 10. + ASSERT_OK_AND_ASSIGN(std::optional merged_with_factory, + data_split->MergedRowCount(dv_factory)); + ASSERT_EQ(std::optional(10), merged_with_factory); +} + TEST(DataSplitTest, TestRowCountAndLatestFileCreationEpochMillisEmpty) { DataSplitImpl::Builder builder( /*partition=*/BinaryRow::EmptyRow(), diff --git a/src/paimon/core/table/source/data_table_batch_scan.cpp b/src/paimon/core/table/source/data_table_batch_scan.cpp index 9cb816f5..5ca4b2ae 100644 --- a/src/paimon/core/table/source/data_table_batch_scan.cpp +++ b/src/paimon/core/table/source/data_table_batch_scan.cpp @@ -83,9 +83,14 @@ Result> DataTableBatchScan::ApplyPushDownLimit( return Status::Invalid("DataSplit cannot cast to DataSplitImpl"); } if (data_split->RawConvertible()) { - int64_t partial_merged_row_count = data_split->PartialMergedRowCount(); + PAIMON_ASSIGN_OR_RAISE(std::optional partial_merged_row_count, + data_split->MergedRowCount()); + if (!partial_merged_row_count.has_value()) { + // Cannot safely estimate split rows from metadata; skip push-down limit. + return current_scan_result->GetPlan(); + } limited_data_splits.emplace_back(data_split); - scanned_row_count += partial_merged_row_count; + scanned_row_count += partial_merged_row_count.value(); if (scanned_row_count >= push_down_limit_.value()) { PAIMON_ASSIGN_OR_RAISE(int64_t snapshot_id, current_scan_result->SnapshotId()); return std::make_shared(snapshot_id, limited_data_splits); diff --git a/src/paimon/core/table/source/key_value_table_read.cpp b/src/paimon/core/table/source/key_value_table_read.cpp index 0d709094..23b90226 100644 --- a/src/paimon/core/table/source/key_value_table_read.cpp +++ b/src/paimon/core/table/source/key_value_table_read.cpp @@ -23,6 +23,8 @@ #include "paimon/core/operation/merge_file_split_read.h" #include "paimon/core/operation/raw_file_split_read.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/pk_count_reader.h" #include "paimon/status.h" namespace paimon { @@ -33,8 +35,15 @@ class InternalReadContext; class MemoryPool; KeyValueTableRead::KeyValueTableRead(std::vector>&& split_reads, - const std::shared_ptr& memory_pool) - : TableRead(memory_pool), split_reads_(std::move(split_reads)) {} + const std::shared_ptr& path_factory, + const std::shared_ptr& context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor) + : TableRead(memory_pool), + split_reads_(std::move(split_reads)), + path_factory_(path_factory), + context_(context), + executor_(executor) {} Result> KeyValueTableRead::Create( const std::shared_ptr& path_factory, @@ -49,7 +58,8 @@ Result> KeyValueTableRead::Create( MergeFileSplitRead::Create(path_factory, context, memory_pool, executor)); split_reads.emplace_back(std::move(merge_file_split_read)); - return std::unique_ptr(new KeyValueTableRead(std::move(split_reads), memory_pool)); + return std::unique_ptr(new KeyValueTableRead(std::move(split_reads), path_factory, + context, memory_pool, executor)); } void KeyValueTableRead::ForceKeepDelete(bool force_keep_delete) { @@ -77,4 +87,22 @@ Result> KeyValueTableRead::CreateReader( return Status::Invalid("create reader failed, not read match with data split."); } +Result> KeyValueTableRead::CreateCountReader( + const std::vector>& splits) { + if (context_->GetPredicate() != nullptr) { + return Status::NotImplemented( + "CreateCountReader with predicate pushdown is not supported yet"); + } + + if (force_keep_delete_) { + return Status::NotImplemented("CreateCountReader with force_keep_delete is not supported"); + } + + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr pk_count_reader, + PKCountReader::Create(splits, path_factory_, context_, GetMemoryPool(), executor_)); + + return pk_count_reader; +} + } // namespace paimon diff --git a/src/paimon/core/table/source/key_value_table_read.h b/src/paimon/core/table/source/key_value_table_read.h index 7751e599..d6a1c83d 100644 --- a/src/paimon/core/table/source/key_value_table_read.h +++ b/src/paimon/core/table/source/key_value_table_read.h @@ -45,13 +45,22 @@ class KeyValueTableRead : public TableRead { Result> CreateReader(const std::shared_ptr& split) override; + Result> CreateCountReader( + const std::vector>& splits) override; + void ForceKeepDelete(bool force_keep_delete); private: KeyValueTableRead(std::vector>&& split_reads, - const std::shared_ptr& memory_pool); + const std::shared_ptr& path_factory, + const std::shared_ptr& context, + const std::shared_ptr& memory_pool, + const std::shared_ptr& executor); std::vector> split_reads_; + std::shared_ptr path_factory_; + std::shared_ptr context_; + std::shared_ptr executor_; bool force_keep_delete_ = false; }; diff --git a/src/paimon/core/table/source/pk_count_reader.cpp b/src/paimon/core/table/source/pk_count_reader.cpp new file mode 100644 index 00000000..2e492ad7 --- /dev/null +++ b/src/paimon/core/table/source/pk_count_reader.cpp @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/pk_count_reader.h" + +#include +#include +#include + +#include "arrow/c/abi.h" +#include "arrow/type.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/deletionvectors/deletion_vector.h" +#include "paimon/core/mergetree/compact/interval_partition.h" +#include "paimon/core/mergetree/row_count_accumulator.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/operation/merge_file_split_read.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/status.h" + +namespace paimon { + +PKCountReader::~PKCountReader() = default; + +Result> PKCountReader::Create( + std::vector> splits, + const std::shared_ptr& path_factory, + const std::shared_ptr& context, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor) { + const auto& table_schema = context->GetTableSchema(); + PAIMON_ASSIGN_OR_RAISE(std::vector pk_fields, + table_schema->TrimmedPrimaryKeyFields()); + std::shared_ptr count_read_schema = + DataField::ConvertDataFieldsToArrowSchema(pk_fields); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr count_context, + InternalReadContext::CreateWithSchema(context, count_read_schema)); + + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr merge_read, + MergeFileSplitRead::Create(path_factory, count_context, memory_pool, executor)); + + return std::unique_ptr( + new PKCountReader(std::move(splits), count_context, std::move(merge_read), memory_pool)); +} + +Result PKCountReader::CountRows() { + int64_t total = 0; + for (const auto& split : splits_) { + PAIMON_ASSIGN_OR_RAISE(int64_t split_count, CountSingleSplit(split)); + total += split_count; + } + return total; +} + +PKCountReader::PKCountReader(std::vector> splits, + const std::shared_ptr& context, + std::unique_ptr&& merge_read, + const std::shared_ptr& memory_pool) + : splits_(std::move(splits)), + context_(context), + merge_read_(std::move(merge_read)), + pool_(memory_pool) {} + +Result PKCountReader::CountSingleSplit(const std::shared_ptr& split) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("split cannot be cast to DataSplitImpl"); + } + + if (data_split->DataFiles().empty()) { + return 0; + } + + if (data_split->RawConvertible()) { + return MetadataCount(data_split); + } + + return MergeCount(data_split); +} + +Result PKCountReader::MetadataCount(const std::shared_ptr& split) { + DeletionVector::Factory dv_factory = DeletionVector::CreateFactory( + context_->GetCoreOptions().GetFileSystem(), + DeletionVector::CreateDeletionFileMap(split->DataFiles(), split->DeletionFiles()), pool_); + + PAIMON_ASSIGN_OR_RAISE(std::optional count, split->MergedRowCount(dv_factory)); + if (count.has_value()) { + return count.value(); + } + + return Status::Invalid("not support split in pk count metadata fallback"); +} + +Result PKCountReader::MergeCount(const std::shared_ptr& split) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, + merge_read_->GetPathFactory()->CreateDataFilePathFactory( + split->Partition(), split->Bucket())); + + auto dv_factory = DeletionVector::CreateFactory( + context_->GetCoreOptions().GetFileSystem(), + DeletionVector::CreateDeletionFileMap(split->DataFiles(), split->DeletionFiles()), pool_); + + std::vector> sections = + IntervalPartition(split->DataFiles(), merge_read_->GetKeyComparator()).Partition(); + + int64_t total_count = 0; + + for (const auto& section : sections) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr merged_reader, + merge_read_->CreateSortMergeReaderForSection( + section, split->Partition(), dv_factory, + /*predicate=*/nullptr, data_file_path_factory, + /*drop_delete=*/true)); + + RowCountAccumulator accumulator(std::move(merged_reader)); + PAIMON_ASSIGN_OR_RAISE(int64_t section_count, accumulator.CountAll()); + total_count += section_count; + accumulator.Close(); + } + + return total_count; +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/pk_count_reader.h b/src/paimon/core/table/source/pk_count_reader.h new file mode 100644 index 00000000..701c6121 --- /dev/null +++ b/src/paimon/core/table/source/pk_count_reader.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/reader/count_reader.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { +class DataSplitImpl; +class Executor; +class FileStorePathFactory; +class InternalReadContext; +class MemoryPool; +class MergeFileSplitRead; +class Split; + +class PKCountReader : public CountReader { + public: + ~PKCountReader() override; + + static Result> Create( + std::vector> splits, + const std::shared_ptr& path_factory, + const std::shared_ptr& context, + const std::shared_ptr& memory_pool, const std::shared_ptr& executor); + + Result CountRows() override; + + private: + PKCountReader(std::vector> splits, + const std::shared_ptr& context, + std::unique_ptr&& merge_read, + const std::shared_ptr& memory_pool); + + Result CountSingleSplit(const std::shared_ptr& split); + Result MetadataCount(const std::shared_ptr& split); + Result MergeCount(const std::shared_ptr& split); + + private: + std::vector> splits_; + std::shared_ptr context_; + std::unique_ptr merge_read_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/pk_count_reader_test.cpp b/src/paimon/core/table/source/pk_count_reader_test.cpp new file mode 100644 index 00000000..72507fee --- /dev/null +++ b/src/paimon/core/table/source/pk_count_reader_test.cpp @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/pk_count_reader.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/split.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +class DummySplit final : public Split {}; + +} // namespace + +class PKCountReaderTest : public testing::Test { + protected: + Result>> CreateSplits( + const std::string& table_path, const std::optional& snapshot_id) { + ScanContextBuilder scan_context_builder(table_path); + if (snapshot_id.has_value()) { + scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, + std::to_string(snapshot_id.value())); + } + + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); + if (snapshot_id.has_value()) { + if (!plan->SnapshotId().has_value() || plan->SnapshotId().value() != snapshot_id) { + return Status::Invalid("snapshot id mismatch"); + } + } + return plan->Splits(); + } + + Result> CreateInternalContext( + const std::string& table_path) { + ReadContextBuilder read_context_builder(table_path); + PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); + + SchemaManager schema_manager(std::make_shared(), table_path); + PAIMON_ASSIGN_OR_RAISE(auto table_schema, schema_manager.ReadSchema(0)); + PAIMON_ASSIGN_OR_RAISE(auto internal_context, + InternalReadContext::Create(std::move(read_context), table_schema, + table_schema->Options())); + return std::shared_ptr(std::move(internal_context)); + } + + Result> CreatePathFactory( + const std::shared_ptr& internal_context) { + const auto& core_options = internal_context->GetCoreOptions(); + auto arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(internal_context->GetTableSchema()->Fields()); + + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, + core_options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + core_options.CreateGlobalIndexExternalPath()); + + PAIMON_ASSIGN_OR_RAISE( + auto path_factory, + FileStorePathFactory::Create( + internal_context->GetPath(), arrow_schema, + internal_context->GetTableSchema()->PartitionKeys(), + core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), + core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), + pool_)); + + return std::shared_ptr(std::move(path_factory)); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(PKCountReaderTest, TestCountRowsWithMORSnapshot5) { + std::string table_path = + GetDataDir() + "/orc/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/5)); + ASSERT_OK_AND_ASSIGN(auto internal_context, CreateInternalContext(table_path)); + ASSERT_OK_AND_ASSIGN(auto path_factory, CreatePathFactory(internal_context)); + + ASSERT_OK_AND_ASSIGN(auto pk_count_reader, + PKCountReader::Create(splits, path_factory, internal_context, pool_, + internal_context->GetExecutor())); + ASSERT_OK_AND_ASSIGN(int64_t count, pk_count_reader->CountRows()); + + ASSERT_EQ(count, 11); +} + +TEST_F(PKCountReaderTest, TestCountRowsWithDVSnapshot6) { + std::string table_path = + GetDataDir() + "/orc/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; + + ASSERT_OK_AND_ASSIGN(auto splits, CreateSplits(table_path, /*snapshot_id=*/6)); + ASSERT_OK_AND_ASSIGN(auto internal_context, CreateInternalContext(table_path)); + ASSERT_OK_AND_ASSIGN(auto path_factory, CreatePathFactory(internal_context)); + + ASSERT_OK_AND_ASSIGN(auto pk_count_reader, + PKCountReader::Create(splits, path_factory, internal_context, pool_, + internal_context->GetExecutor())); + ASSERT_OK_AND_ASSIGN(int64_t count, pk_count_reader->CountRows()); + + ASSERT_EQ(count, 8); +} + +TEST_F(PKCountReaderTest, TestCountRowsWithEmptySplits) { + std::string table_path = + GetDataDir() + "/orc/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + ASSERT_OK_AND_ASSIGN(auto internal_context, CreateInternalContext(table_path)); + ASSERT_OK_AND_ASSIGN(auto path_factory, CreatePathFactory(internal_context)); + + std::vector> empty_splits; + ASSERT_OK_AND_ASSIGN(auto pk_count_reader, + PKCountReader::Create(empty_splits, path_factory, internal_context, pool_, + internal_context->GetExecutor())); + ASSERT_OK_AND_ASSIGN(int64_t count, pk_count_reader->CountRows()); + + ASSERT_EQ(count, 0); +} + +TEST_F(PKCountReaderTest, TestCountRowsWithInvalidSplit) { + std::string table_path = + GetDataDir() + "/orc/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + ASSERT_OK_AND_ASSIGN(auto internal_context, CreateInternalContext(table_path)); + ASSERT_OK_AND_ASSIGN(auto path_factory, CreatePathFactory(internal_context)); + + std::vector> splits = {std::make_shared()}; + ASSERT_OK_AND_ASSIGN(auto pk_count_reader, + PKCountReader::Create(splits, path_factory, internal_context, pool_, + internal_context->GetExecutor())); + + ASSERT_NOK_WITH_MSG(pk_count_reader->CountRows(), "split cannot be cast to DataSplitImpl"); +} + +} // namespace paimon::test diff --git a/src/paimon/core/table/source/table_read.cpp b/src/paimon/core/table/source/table_read.cpp index 5af23cf6..e7e3f02f 100644 --- a/src/paimon/core/table/source/table_read.cpp +++ b/src/paimon/core/table/source/table_read.cpp @@ -181,4 +181,10 @@ Result> TableRead::CreateReader( return std::make_unique(std::move(batch_readers), pool_); } +Result> TableRead::CreateCountReader( + const std::vector>& splits) { + (void)splits; + return Status::NotImplemented("CreateCountReader is not implemented for this table type"); +} + } // namespace paimon diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 2c2c9f31..491f4628 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -40,6 +40,7 @@ #include "paimon/core/io/data_file_meta.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/deletion_file.h" +#include "paimon/core/table/source/key_value_table_read.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" @@ -336,6 +337,10 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { .ValueOrDie()); ASSERT_TRUE(expected); ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { @@ -421,6 +426,10 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { .ValueOrDie()); ASSERT_TRUE(expected); ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { @@ -599,6 +608,12 @@ TEST_F(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { .ValueOrDie()); ASSERT_TRUE(expected); ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + // CountRows should match the number of visible rows returned by CreateReader. + ASSERT_OK_AND_ASSIGN(auto count_reader, + table_read->CreateCountReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); }; for (auto [file_format, enable_prefetch] : GetTestValuesForScanAndReadInteTest()) { check_result(file_format); @@ -1069,7 +1084,8 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + auto splits = result_plan->Splits(); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); // check result @@ -1090,6 +1106,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { .ValueOrDie()); ASSERT_TRUE(expected); ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { @@ -1111,7 +1131,8 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + auto splits = result_plan->Splits(); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(splits)); ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); // check result @@ -1129,6 +1150,10 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { .ValueOrDie()); ASSERT_TRUE(expected); ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndBucketFilter) { @@ -1794,7 +1819,9 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { [0, "Emily", 10, 0, 13.1], [0, "Alice", 10, 1, 21.1], [0, "Two roads diverged in a wood, and I took the one less traveled by, And that has made all the difference.", 10, 1, 11.0], -[0, "Whether I shall turn out to be the hero of my own life.", 10, 1, 19.1] +[0, "Whether I shall turn out to be the hero of my own life.", 10, 1, 19.1], +[0, "Lucy", 20, 1, 14.1], +[0, "Paul", 20, 1, 18.1] ])") .ValueOrDie()); ASSERT_TRUE(expected); @@ -2717,4 +2744,102 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } +TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { + auto [file_format, enable_prefetch] = GetParam(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + + // CreateCountReader with empty splits should return 0 rows. + std::vector> empty_splits; + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(empty_splits)); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, 0); +} + +TEST_P(ScanAndReadInteTest, TestCountRowsConsistencyWithCreateReader) { + auto [file_format, enable_prefetch] = GetParam(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + // Scan latest snapshot + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + // Method 1: CreateCountReader + iterate batches + ReadContextBuilder count_context_builder(table_path); + AddReadOptionsForPrefetch(&count_context_builder); + ASSERT_OK_AND_ASSIGN(auto count_read_context, count_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto count_table_read, TableRead::Create(std::move(count_read_context))); + ASSERT_OK_AND_ASSIGN(auto count_reader, + count_table_read->CreateCountReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(int64_t count_result, count_reader->CountRows()); + + // Method 2: CreateReader + iterate batches + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + int64_t iterate_count = read_result ? read_result->length() : 0; + + // Both methods should return the same count + ASSERT_EQ(count_result, iterate_count); +} + +TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithPredicateNotSupported) { + auto [file_format, enable_prefetch] = GetParam(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + // Create splits from latest snapshot. + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + // Set predicate in read context. CountReader currently does not support predicate pushdown. + auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f2", FieldType::INT, + Literal(static_cast(0))); + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + read_context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + + ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(result_plan->Splits()), + "predicate pushdown is not supported"); +} + +TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithForceKeepDeleteNotSupported) { + auto [file_format, enable_prefetch] = GetParam(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; + + // Create splits from latest snapshot. + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + + auto* key_value_table_read = dynamic_cast(table_read.get()); + ASSERT_TRUE(key_value_table_read != nullptr); + key_value_table_read->ForceKeepDelete(true); + + ASSERT_NOK_WITH_MSG(table_read->CreateCountReader(result_plan->Splits()), + "force_keep_delete is not supported"); +} + } // namespace paimon::test From e88a8d47eae7eb18be5d0fbb2c01914d836dc2f7 Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Wed, 3 Jun 2026 17:28:42 +0800 Subject: [PATCH 026/138] =?UTF-8?q?feat(blob):=20support=20blob-view-field?= =?UTF-8?q?=20for=20cross-table=20blob=20reference=20re=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/paimon/catalog/identifier.h | 17 +- src/paimon/CMakeLists.txt | 6 + src/paimon/common/catalog/catalog_context.h | 38 +++ src/paimon/common/data/blob_descriptor.cpp | 2 +- .../common/data/blob_descriptor_test.cpp | 4 + src/paimon/common/data/blob_view_struct.cpp | 116 +++++++++ src/paimon/common/data/blob_view_struct.h | 88 +++++++ .../common/data/blob_view_struct_test.cpp | 112 ++++++++ .../blob_view_resolving_batch_reader.cpp | 129 +++++++++ .../reader/blob_view_resolving_batch_reader.h | 65 +++++ .../blob_view_resolving_batch_reader_test.cpp | 214 +++++++++++++++ src/paimon/core/catalog/identifier.cpp | 37 ++- src/paimon/core/catalog/identifier_test.cpp | 41 +++ .../operation/data_evolution_split_read.cpp | 158 +++++++++++- .../operation/data_evolution_split_read.h | 18 +- src/paimon/core/utils/blob_view_lookup.cpp | 244 ++++++++++++++++++ src/paimon/core/utils/blob_view_lookup.h | 83 ++++++ .../core/utils/blob_view_lookup_test.cpp | 194 ++++++++++++++ test/inte/blob_table_inte_test.cpp | 163 ++++++++++++ 19 files changed, 1717 insertions(+), 12 deletions(-) create mode 100644 src/paimon/common/catalog/catalog_context.h create mode 100644 src/paimon/common/data/blob_view_struct.cpp create mode 100644 src/paimon/common/data/blob_view_struct.h create mode 100644 src/paimon/common/data/blob_view_struct_test.cpp create mode 100644 src/paimon/common/reader/blob_view_resolving_batch_reader.cpp create mode 100644 src/paimon/common/reader/blob_view_resolving_batch_reader.h create mode 100644 src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp create mode 100644 src/paimon/core/utils/blob_view_lookup.cpp create mode 100644 src/paimon/core/utils/blob_view_lookup.h create mode 100644 src/paimon/core/utils/blob_view_lookup_test.cpp diff --git a/include/paimon/catalog/identifier.h b/include/paimon/catalog/identifier.h index 62d6ab8f..82ea2e3a 100644 --- a/include/paimon/catalog/identifier.h +++ b/include/paimon/catalog/identifier.h @@ -38,7 +38,7 @@ class PAIMON_EXPORT Identifier { explicit Identifier(const std::string& table); Identifier(const std::string& database, const std::string& table); - bool operator==(const Identifier& other); + bool operator==(const Identifier& other) const; const std::string& GetDatabaseName() const; const std::string& GetTableName() const; Result GetDataTableName() const; @@ -46,7 +46,12 @@ class PAIMON_EXPORT Identifier { Result GetBranchNameOrDefault() const; Result> GetSystemTableName() const; Result IsSystemTable() const; + std::string GetFullName() const; std::string ToString() const; + int32_t HashCode() const; + + public: + static Result FromString(const std::string& full_name); private: Status SplitTableName() const; @@ -60,3 +65,13 @@ class PAIMON_EXPORT Identifier { }; } // namespace paimon + +namespace std { +template <> +struct hash { + size_t operator()(const paimon::Identifier& identifier) const { + return identifier.HashCode(); + } +}; + +} // namespace std diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index a47e8c00..b6a0e73f 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -27,6 +27,7 @@ set(PAIMON_COMMON_SRCS common/data/binary_string.cpp common/data/blob.cpp common/data/blob_descriptor.cpp + common/data/blob_view_struct.cpp common/data/blob_utils.cpp common/data/columnar/columnar_array.cpp common/data/columnar/columnar_map.cpp @@ -112,6 +113,7 @@ set(PAIMON_COMMON_SRCS common/reader/predicate_batch_reader.cpp common/reader/prefetch_file_batch_reader_impl.cpp common/reader/reader_utils.cpp + common/reader/blob_view_resolving_batch_reader.cpp common/reader/complete_row_kind_batch_reader.cpp common/reader/data_evolution_file_reader.cpp common/sst/block_handle.cpp @@ -334,6 +336,7 @@ set(PAIMON_CORE_SRCS core/table/system/system_table_schema.cpp core/tag/tag.cpp core/utils/branch_manager.cpp + core/utils/blob_view_lookup.cpp core/utils/consumer_manager.cpp core/utils/field_mapping.cpp core/utils/file_store_path_factory.cpp @@ -410,6 +413,7 @@ if(PAIMON_BUILD_TESTS) common/data/timestamp_test.cpp common/data/blob_test.cpp common/data/blob_descriptor_test.cpp + common/data/blob_view_struct_test.cpp common/data/blob_utils_test.cpp common/executor/default_executor_test.cpp common/format/column_stats_test.cpp @@ -472,6 +476,7 @@ if(PAIMON_BUILD_TESTS) common/reader/prefetch_file_batch_reader_impl_test.cpp common/reader/reader_utils_test.cpp common/reader/complete_row_kind_batch_reader_test.cpp + common/reader/blob_view_resolving_batch_reader_test.cpp common/reader/data_evolution_file_reader_test.cpp common/reader/data_evolution_array_test.cpp common/reader/data_evolution_row_test.cpp @@ -717,6 +722,7 @@ if(PAIMON_BUILD_TESTS) core/table/source/table_scan_test.cpp core/table/system/system_table_test.cpp core/tag/tag_test.cpp + core/utils/blob_view_lookup_test.cpp core/utils/branch_manager_test.cpp core/utils/consumer_manager_test.cpp core/utils/file_store_path_factory_cache_test.cpp diff --git a/src/paimon/common/catalog/catalog_context.h b/src/paimon/common/catalog/catalog_context.h new file mode 100644 index 00000000..fecc3e01 --- /dev/null +++ b/src/paimon/common/catalog/catalog_context.h @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once +#include +#include + +#include "paimon/fs/file_system.h" + +namespace paimon { +struct CatalogContext { + CatalogContext(const std::string& _root_path, + const std::map& _options, + const std::shared_ptr& _file_system) + : root_path(_root_path), options(_options), file_system(_file_system) {} + + std::string root_path; + std::map options; + std::shared_ptr file_system; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/blob_descriptor.cpp b/src/paimon/common/data/blob_descriptor.cpp index f5624c78..c262ebf1 100644 --- a/src/paimon/common/data/blob_descriptor.cpp +++ b/src/paimon/common/data/blob_descriptor.cpp @@ -53,7 +53,7 @@ Result> BlobDescriptor::Create(int8_t version, PAIMON_UNIQUE_PTR BlobDescriptor::Serialize(const std::shared_ptr& pool) const { MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); out.SetOrder(ByteOrder::PAIMON_LITTLE_ENDIAN); - out.WriteValue(version_); + out.WriteValue(kCurrentVersion); out.WriteValue(kMagic); out.WriteValue(static_cast(uri_.size())); diff --git a/src/paimon/common/data/blob_descriptor_test.cpp b/src/paimon/common/data/blob_descriptor_test.cpp index a5221bf9..1fbd9de0 100644 --- a/src/paimon/common/data/blob_descriptor_test.cpp +++ b/src/paimon/common/data/blob_descriptor_test.cpp @@ -59,6 +59,10 @@ TEST_F(BlobDescriptorTest, TestDeserializeCompatibilityForJavaWithVersion1) { ASSERT_EQ(descriptor->Uri(), "test_uri"); ASSERT_EQ(descriptor->Offset(), 1024); ASSERT_EQ(descriptor->Length(), 2048); + + PAIMON_UNIQUE_PTR cpp_serialized = descriptor->Serialize(pool_); + auto cpp_serialized_string = std::string(cpp_serialized->data(), cpp_serialized->size()); + ASSERT_NE(cpp_serialized_string, java_serialized); } TEST_F(BlobDescriptorTest, TestDeserializeCompatibilityForJavaWithVersion2) { diff --git a/src/paimon/common/data/blob_view_struct.cpp b/src/paimon/common/data/blob_view_struct.cpp new file mode 100644 index 00000000..e5bd2e9e --- /dev/null +++ b/src/paimon/common/data/blob_view_struct.cpp @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "paimon/common/data/blob_view_struct.h" + +#include + +#include "fmt/format.h" +#include "paimon/common/io/memory_segment_output_stream.h" +#include "paimon/common/memory/memory_segment_utils.h" +#include "paimon/common/utils/murmurhash_utils.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/io/byte_order.h" +#include "paimon/io/data_input_stream.h" +#include "paimon/memory/bytes.h" +#include "paimon/status.h" + +namespace paimon { +PAIMON_UNIQUE_PTR BlobViewStruct::Serialize(const std::shared_ptr& pool) const { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); + out.SetOrder(ByteOrder::PAIMON_LITTLE_ENDIAN); + + out.WriteValue(kCurrentVersion); + out.WriteValue(kMagic); + std::string identifier = identifier_.GetFullName(); + out.WriteValue(static_cast(identifier.size())); + auto uri_bytes = std::make_shared(identifier, pool.get()); + out.WriteBytes(uri_bytes); + out.WriteValue(field_id_); + out.WriteValue(row_id_); + return MemorySegmentUtils::CopyToBytes(out.Segments(), 0, out.CurrentSize(), pool.get()); +} + +Result> BlobViewStruct::Deserialize(const char* buffer, + uint64_t size) { + auto input_stream = std::make_shared(buffer, size); + DataInputStream in(std::move(input_stream)); + in.SetOrder(ByteOrder::PAIMON_LITTLE_ENDIAN); + + PAIMON_ASSIGN_OR_RAISE(int8_t version, in.ReadValue()); + if (version != kCurrentVersion) { + return Status::Invalid(fmt::format( + "Expecting BlobViewStruct version to be {}, but found {}.", kCurrentVersion, version)); + } + PAIMON_ASSIGN_OR_RAISE(int64_t magic, in.ReadValue()); + if (kMagic != magic) { + return Status::Invalid( + fmt::format("Invalid BlobViewStruct: missing magic header. Expected magic: {}, " + "but found {}", + kMagic, magic)); + } + PAIMON_ASSIGN_OR_RAISE(int32_t length, in.ReadValue()); + std::string identifier_str(length, '\0'); + PAIMON_RETURN_NOT_OK(in.Read(identifier_str.data(), identifier_str.size())); + PAIMON_ASSIGN_OR_RAISE(int32_t field_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int64_t row_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(Identifier identifier, Identifier::FromString(identifier_str)); + return std::make_unique(identifier, field_id, row_id); +} + +Result BlobViewStruct::IsBlobViewStruct(const char* buffer, uint64_t size) { + if (size < kMinViewLength) { + return false; + } + auto input_stream = std::make_shared(buffer, size); + DataInputStream in(std::move(input_stream)); + in.SetOrder(ByteOrder::PAIMON_LITTLE_ENDIAN); + + PAIMON_ASSIGN_OR_RAISE(int8_t version, in.ReadValue()); + if (version != kCurrentVersion) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(int64_t magic, in.ReadValue()); + return kMagic == magic; +} + +std::string BlobViewStruct::ToString() const { + return fmt::format("BlobViewStruct{{identifier={}, fieldId={}, rowId={}}}", + identifier_.GetFullName(), field_id_, row_id_); +} + +bool BlobViewStruct::operator==(const BlobViewStruct& other) const { + if (this == &other) { + return true; + } + return field_id_ == other.field_id_ && row_id_ == other.row_id_ && + identifier_ == other.identifier_; +} + +bool BlobViewStruct::operator!=(const BlobViewStruct& other) const { + return !(*this == other); +} + +int32_t BlobViewStruct::HashCode() const { + int32_t hash = + MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(&field_id_), + /*offset=*/0, sizeof(field_id_), identifier_.HashCode()); + return MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(&row_id_), + /*offset=*/0, sizeof(row_id_), hash); +} +} // namespace paimon diff --git a/src/paimon/common/data/blob_view_struct.h b/src/paimon/common/data/blob_view_struct.h new file mode 100644 index 00000000..971a80b3 --- /dev/null +++ b/src/paimon/common/data/blob_view_struct.h @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { +/// Serialized metadata for a BLOB view field. +/// A blob view only stores the coordinates needed to locate the original blob value in the +/// upstream table: identifier, field_id and row_id. The actual blob data is +/// resolved at read time by scanning the upstream table. +class BlobViewStruct { + public: + BlobViewStruct(const Identifier& identifier, int32_t field_id, int64_t row_id) + : identifier_(identifier), field_id_(field_id), row_id_(row_id) {} + + const Identifier& GetIdentifier() const { + return identifier_; + } + + int32_t FieldId() const { + return field_id_; + } + + int64_t RowId() const { + return row_id_; + } + + static Result> Deserialize(const char* buffer, uint64_t size); + static Result IsBlobViewStruct(const char* buffer, uint64_t size); + PAIMON_UNIQUE_PTR Serialize(const std::shared_ptr& pool) const; + std::string ToString() const; + int32_t HashCode() const; + + bool operator==(const BlobViewStruct& other) const; + bool operator!=(const BlobViewStruct& other) const; + + private: + static constexpr int64_t kMagic = 0x424C4F4256494557l; + static constexpr int8_t kCurrentVersion = 1; + /// one byte for version, eight bytes for magic number. + static constexpr uint64_t kMinViewLength = 9; + + Identifier identifier_; + int32_t field_id_; + int64_t row_id_; +}; + +/// Resolves a BlobViewStruct into the serialized BlobDescriptor bytes stored in the upstream +/// table. Returns nullptr when the referenced source-table cell is null. +using BlobViewResolver = std::function>(const BlobViewStruct&)>; + +} // namespace paimon + +namespace std { +template <> +struct hash { + size_t operator()(const paimon::BlobViewStruct& blob_view_struct) const { + return blob_view_struct.HashCode(); + } +}; + +} // namespace std diff --git a/src/paimon/common/data/blob_view_struct_test.cpp b/src/paimon/common/data/blob_view_struct_test.cpp new file mode 100644 index 00000000..783d621e --- /dev/null +++ b/src/paimon/common/data/blob_view_struct_test.cpp @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "paimon/common/data/blob_view_struct.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class BlobViewStructTest : public testing::Test { + public: + std::shared_ptr pool_ = GetDefaultPool(); + Identifier identifier_ = Identifier("test_db", "test_table"); + BlobViewStruct view_struct_ = BlobViewStruct(identifier_, /*field_id=*/7, /*row_id=*/1024); +}; + +TEST_F(BlobViewStructTest, TestConstructorAndGetters) { + ASSERT_EQ(view_struct_.GetIdentifier().GetDatabaseName(), "test_db"); + ASSERT_EQ(view_struct_.GetIdentifier().GetTableName(), "test_table"); + ASSERT_EQ(view_struct_.FieldId(), 7); + ASSERT_EQ(view_struct_.RowId(), 1024); +} + +TEST_F(BlobViewStructTest, TestSerializeDeserializeRoundTrip) { + auto serialized = view_struct_.Serialize(pool_); + ASSERT_NE(serialized, nullptr); + ASSERT_GT(serialized->size(), 0u); + + ASSERT_OK_AND_ASSIGN(auto restored, + BlobViewStruct::Deserialize(serialized->data(), serialized->size())); + ASSERT_EQ(restored->GetIdentifier().GetDatabaseName(), "test_db"); + ASSERT_EQ(restored->GetIdentifier().GetTableName(), "test_table"); + ASSERT_EQ(restored->FieldId(), 7); + ASSERT_EQ(restored->RowId(), 1024); +} + +TEST_F(BlobViewStructTest, TestDeserializeWithInvalidVersion) { + auto serialized = view_struct_.Serialize(pool_); + (*serialized)[0] = '\x02'; // invalid version (current is 1). + ASSERT_NOK_WITH_MSG(BlobViewStruct::Deserialize(serialized->data(), serialized->size()), + "Expecting BlobViewStruct version to be 1, but found 2"); +} + +TEST_F(BlobViewStructTest, TestDeserializeWithInvalidMagic) { + auto serialized = view_struct_.Serialize(pool_); + (*serialized)[1] = '\x00'; + ASSERT_NOK_WITH_MSG(BlobViewStruct::Deserialize(serialized->data(), serialized->size()), + "missing magic header"); +} + +TEST_F(BlobViewStructTest, TestToString) { + std::string debug_str = view_struct_.ToString(); + ASSERT_EQ(debug_str, "BlobViewStruct{identifier=test_db.test_table, fieldId=7, rowId=1024}"); +} + +TEST_F(BlobViewStructTest, TestEqual) { + { + // test equal itself + ASSERT_EQ(view_struct_, view_struct_); + } + { + // test equal + BlobViewStruct other_view_struct = + BlobViewStruct(identifier_, /*field_id=*/7, /*row_id=*/1024); + ASSERT_EQ(view_struct_, other_view_struct); + } + { + // test wrong identifier + Identifier wrong_identifier = Identifier("db", "table"); + BlobViewStruct wrong_view_struct = + BlobViewStruct(wrong_identifier, /*field_id=*/7, /*row_id=*/1024); + ASSERT_NE(view_struct_, wrong_view_struct); + } + { + // test wrong field_id + BlobViewStruct wrong_view_struct = + BlobViewStruct(identifier_, /*field_id=*/8, /*row_id=*/1024); + ASSERT_NE(view_struct_, wrong_view_struct); + } + { + // test wrong row_id + BlobViewStruct wrong_view_struct = + BlobViewStruct(identifier_, /*field_id=*/7, /*row_id=*/1000); + ASSERT_NE(view_struct_, wrong_view_struct); + } +} + +} // namespace paimon::test diff --git a/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp b/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp new file mode 100644 index 00000000..a2714a31 --- /dev/null +++ b/src/paimon/common/reader/blob_view_resolving_batch_reader.cpp @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/reader/blob_view_resolving_batch_reader.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/array/array_base.h" +#include "arrow/array/array_binary.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_binary.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/memory/bytes.h" +#include "paimon/status.h" + +namespace paimon { +BlobViewResolvingBatchReader::BlobViewResolvingBatchReader( + std::unique_ptr&& reader, std::vector read_blob_view_fields, + BlobViewResolver resolver, const std::shared_ptr& pool) + : pool_(pool), + arrow_pool_(GetArrowPool(pool)), + reader_(std::move(reader)), + read_blob_view_fields_(std::make_move_iterator(read_blob_view_fields.begin()), + std::make_move_iterator(read_blob_view_fields.end())), + resolver_(std::move(resolver)) {} + +Result BlobViewResolvingBatchReader::NextBatch() { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + return batch; + } + if (read_blob_view_fields_.empty()) { + return batch; + } + + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + if (struct_array == nullptr) { + return Status::Invalid( + "invalid batch, BlobViewResolvingBatchReader expects a StructArray batch."); + } + const auto struct_type = struct_array->struct_type(); + + arrow::ArrayVector new_fields = struct_array->fields(); + std::vector field_names; + field_names.reserve(struct_type->num_fields()); + + for (int32_t field_idx = 0; field_idx < struct_type->num_fields(); ++field_idx) { + const auto& field = struct_type->field(field_idx); + field_names.push_back(field->name()); + if (read_blob_view_fields_.find(field->name()) == read_blob_view_fields_.end()) { + continue; + } + const auto& column = struct_array->field(field_idx); + if (auto large_binary_array = std::dynamic_pointer_cast(column)) { + PAIMON_ASSIGN_OR_RAISE(new_fields[field_idx], ResolveBinaryColumn(large_binary_array)); + } else { + return Status::Invalid(fmt::format( + "BlobViewResolvingBatchReader expects blob-view column {} to be LargeBinaryArray.", + field->name())); + } + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr resolved_struct_array, + arrow::StructArray::Make(new_fields, field_names)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*resolved_struct_array, c_array.get(), c_schema.get())); + return batch; +} + +Result> BlobViewResolvingBatchReader::ResolveBinaryColumn( + const std::shared_ptr& blob_view_struct_array) { + arrow::LargeBinaryBuilder builder(arrow_pool_.get()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(blob_view_struct_array->length())); + for (int64_t row = 0; row < blob_view_struct_array->length(); ++row) { + if (blob_view_struct_array->IsNull(row)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + continue; + } + auto view = blob_view_struct_array->GetView(row); + PAIMON_ASSIGN_OR_RAISE(bool is_view_struct, + BlobViewStruct::IsBlobViewStruct(view.data(), view.size())); + if (!is_view_struct) { + return Status::Invalid( + "BlobViewResolvingBatchReader expects a serialized BlobViewStruct in the " + "blob-view column."); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr view_struct, + BlobViewStruct::Deserialize(view.data(), view.size())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr descriptor_bytes, resolver_(*view_struct)); + if (descriptor_bytes == nullptr) { + // null in source table + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + continue; + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append( + reinterpret_cast(descriptor_bytes->data()), descriptor_bytes->size())); + } + std::shared_ptr blob_descriptor_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&blob_descriptor_array)); + return blob_descriptor_array; +} + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_view_resolving_batch_reader.h b/src/paimon/common/reader/blob_view_resolving_batch_reader.h new file mode 100644 index 00000000..e3b14abb --- /dev/null +++ b/src/paimon/common/reader/blob_view_resolving_batch_reader.h @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/memory_pool.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/result.h" + +namespace paimon { +class BlobViewResolvingBatchReader : public BatchReader { + public: + BlobViewResolvingBatchReader(std::unique_ptr&& reader, + std::vector read_blob_view_fields, + BlobViewResolver resolver, + const std::shared_ptr& pool); + + Result NextBatch() override; + + void Close() override { + reader_->Close(); + } + + std::shared_ptr GetReaderMetrics() const override { + return reader_->GetReaderMetrics(); + } + + private: + Result> ResolveBinaryColumn( + const std::shared_ptr& blob_view_struct_array); + + private: + std::shared_ptr pool_; + std::unique_ptr arrow_pool_; + std::unique_ptr reader_; + std::set read_blob_view_fields_; + BlobViewResolver resolver_; +}; + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp b/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp new file mode 100644 index 00000000..4091718f --- /dev/null +++ b/src/paimon/common/reader/blob_view_resolving_batch_reader_test.cpp @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/reader/blob_view_resolving_batch_reader.h" + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/array/array_binary.h" +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_binary.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class BlobViewResolvingBatchReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + } + + void TearDown() override { + pool_.reset(); + } + + class InMemoryBatchReader : public BatchReader { + public: + explicit InMemoryBatchReader(const std::shared_ptr& struct_array) + : struct_array_(struct_array) { + if (!struct_array_) { + exhausted_ = true; + } + } + + Result NextBatch() override { + if (exhausted_) { + return MakeEofBatch(); + } + exhausted_ = true; + auto c_array = std::make_unique(); + auto c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*struct_array_, c_array.get(), c_schema.get())); + return std::make_pair(std::move(c_array), std::move(c_schema)); + } + + std::shared_ptr GetReaderMetrics() const override { + return std::make_shared(); + } + + void Close() override {} + + private: + std::shared_ptr struct_array_; + bool exhausted_ = false; + }; + + std::string MakeBlobViewStructBytes(const std::string& database, const std::string& table, + int32_t field_id, int64_t row_id) const { + Identifier identifier(database, table); + BlobViewStruct view_struct(identifier, field_id, row_id); + auto bytes = view_struct.Serialize(pool_); + return std::string(bytes->data(), bytes->size()); + } + + Result MakeBlobDescriptorBytes(const std::string& uri, int64_t offset, + int64_t length) const { + PAIMON_ASSIGN_OR_RAISE(auto descriptor, BlobDescriptor::Create(uri, offset, length)); + auto bytes = descriptor->Serialize(pool_); + return std::string(bytes->data(), bytes->size()); + } + + std::shared_ptr BuildStructArray(const std::vector& values, + const std::vector& valid) const { + arrow::LargeBinaryBuilder builder; + EXPECT_TRUE(builder.Reserve(static_cast(values.size())).ok()); + for (size_t i = 0; i < values.size(); ++i) { + if (!valid[i]) { + EXPECT_TRUE(builder.AppendNull().ok()); + } else { + EXPECT_TRUE(builder + .Append(reinterpret_cast(values[i].data()), + static_cast(values[i].size())) + .ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(builder.Finish(&array).ok()); + + arrow::FieldVector fields = {BlobUtils::ToArrowField("blob_col", /*nullable=*/true)}; + arrow::ArrayVector arrays = {array}; + auto result = arrow::StructArray::Make(arrays, fields).ValueOrDie(); + return result; + } + + private: + std::shared_ptr pool_; +}; + +TEST_F(BlobViewResolvingBatchReaderTest, TestEofBatch) { + auto inner_reader = std::make_unique(nullptr); + auto resolver = BlobViewResolver([](const BlobViewStruct&) -> Result> { + return std::shared_ptr(); + }); + BlobViewResolvingBatchReader reader(std::move(inner_reader), {"blob_col"}, std::move(resolver), + pool_); + ASSERT_OK_AND_ASSIGN(auto batch, reader.NextBatch()); + ASSERT_TRUE(BatchReader::IsEofBatch(batch)); +} + +TEST_F(BlobViewResolvingBatchReaderTest, TestEmptyReadBlobViewFields) { + std::string view_bytes = MakeBlobViewStructBytes("db", "table", /*field_id=*/1, /*row_id=*/7); + std::shared_ptr struct_array = BuildStructArray({view_bytes}, {true}); + + bool resolver_called = false; + auto resolver = BlobViewResolver( + [&resolver_called](const BlobViewStruct&) -> Result> { + resolver_called = true; + return std::shared_ptr(); + }); + + auto inner_reader = std::make_unique(struct_array); + BlobViewResolvingBatchReader reader(std::move(inner_reader), /*read_blob_view_fields=*/{}, + std::move(resolver), pool_); + ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(&reader)); + auto expected_array = std::make_shared(struct_array); + ASSERT_TRUE(expected_array->Equals(*result_array)); + ASSERT_FALSE(resolver_called); +} + +TEST_F(BlobViewResolvingBatchReaderTest, TestResolvesBlobViewColumn) { + auto row0_view = MakeBlobViewStructBytes("db", "tbl", /*field_id=*/3, /*row_id=*/100); + auto row1_view = MakeBlobViewStructBytes("db", "tbl", /*field_id=*/3, /*row_id=*/200); + std::shared_ptr src_struct = + BuildStructArray({row0_view, row1_view}, {true, true}); + + ASSERT_OK_AND_ASSIGN(auto expected_row0_descriptor, + MakeBlobDescriptorBytes("/path/a", /*offset=*/0, /*length=*/8)); + ASSERT_OK_AND_ASSIGN(auto expected_row1_descriptor, + MakeBlobDescriptorBytes("/path/b", /*offset=*/16, /*length=*/32)); + + auto resolver = + BlobViewResolver([&](const BlobViewStruct& view_struct) -> Result> { + if (view_struct.RowId() == 100) { + return std::make_shared(expected_row0_descriptor, pool_.get()); + } + if (view_struct.RowId() == 200) { + return std::make_shared(expected_row1_descriptor, pool_.get()); + } + return Status::Invalid("unexpected view struct"); + }); + + auto inner_reader = std::make_unique(src_struct); + BlobViewResolvingBatchReader reader(std::move(inner_reader), {"blob_col"}, std::move(resolver), + pool_); + ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(&reader)); + auto struct_array = std::dynamic_pointer_cast(result_array->chunk(0)); + + auto result_blob_column = + std::dynamic_pointer_cast(struct_array->field(0)); + ASSERT_FALSE(result_blob_column->IsNull(0)); + ASSERT_FALSE(result_blob_column->IsNull(1)); + ASSERT_EQ(result_blob_column->GetString(0), expected_row0_descriptor); + ASSERT_EQ(result_blob_column->GetString(1), expected_row1_descriptor); +} + +TEST_F(BlobViewResolvingBatchReaderTest, TestResolverError) { + auto view_bytes = MakeBlobViewStructBytes("db", "tbl", /*field_id=*/1, /*row_id=*/5); + std::shared_ptr src_struct = BuildStructArray({view_bytes}, {true}); + auto resolver = BlobViewResolver([](const BlobViewStruct&) -> Result> { + return Status::Invalid("cache miss"); + }); + auto inner_reader = std::make_unique(src_struct); + BlobViewResolvingBatchReader reader(std::move(inner_reader), {"blob_col"}, std::move(resolver), + pool_); + ASSERT_NOK_WITH_MSG(reader.NextBatch(), "cache miss"); +} + +} // namespace paimon::test diff --git a/src/paimon/core/catalog/identifier.cpp b/src/paimon/core/catalog/identifier.cpp index 3b403d30..32ba446a 100644 --- a/src/paimon/core/catalog/identifier.cpp +++ b/src/paimon/core/catalog/identifier.cpp @@ -23,6 +23,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/murmurhash_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/result.h" #include "paimon/status.h" @@ -40,8 +41,11 @@ Identifier::Identifier(const std::string& table) Identifier::Identifier(const std::string& database, const std::string& table) : database_(database), table_(table) {} -bool Identifier::operator==(const Identifier& other) { - return (database_ == other.database_ && table_ == other.table_); +bool Identifier::operator==(const Identifier& other) const { + if (this == &other) { + return true; + } + return database_ == other.database_ && table_ == other.table_; } const std::string& Identifier::GetDatabaseName() const { @@ -81,6 +85,35 @@ std::string Identifier::ToString() const { return fmt::format("Identifier{{database='{}', table='{}'}}", database_, table_); } +int32_t Identifier::HashCode() const { + int32_t hash = MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(database_.data()), + /*offset=*/0, database_.size()); + return MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(table_.data()), + /*offset=*/0, table_.size(), hash); +} + +std::string Identifier::GetFullName() const { + if (database_ == kUnknownDatabase) { + return table_; + } + return fmt::format("{}.{}", database_, table_); +} + +Result Identifier::FromString(const std::string& full_name) { + if (StringUtils::IsNullOrWhitespaceOnly(full_name)) { + return Status::Invalid("full name cannot be empty or whitespace only"); + } + // TODO(lisizhuo.lsz): deal with kUnknownDatabase to be done + const auto dot_pos = full_name.find('.'); + if (dot_pos == std::string::npos || dot_pos == 0 || dot_pos == full_name.size() - 1) { + return Status::Invalid( + fmt::format("cannot get splits from '{}' to get database and table", full_name)); + } + std::string database = full_name.substr(0, dot_pos); + std::string table = full_name.substr(dot_pos + 1); + return Identifier(database, table); +} + Status Identifier::SplitTableName() const { if (parsed_) { return Status::OK(); diff --git a/src/paimon/core/catalog/identifier_test.cpp b/src/paimon/core/catalog/identifier_test.cpp index 771c2f39..50afa6fc 100644 --- a/src/paimon/core/catalog/identifier_test.cpp +++ b/src/paimon/core/catalog/identifier_test.cpp @@ -113,6 +113,47 @@ TEST(IdentifierTest, ParseBranchSystemTable) { ASSERT_TRUE(is_system_table); } +TEST(IdentifierTest, TestGetFullName) { + { + // test unknown database + std::string table = "tbl$branch_dev$options"; + Identifier id(table); + ASSERT_EQ(table, id.GetFullName()); + } + { + // test normal database + std::string db = "database"; + std::string table = "tbl$branch_dev$options"; + Identifier id(db, table); + ASSERT_EQ("database.tbl$branch_dev$options", id.GetFullName()); + } +} + +TEST(IdentifierTest, TestFromString) { + { + // test invalid identifier string + ASSERT_NOK_WITH_MSG(Identifier::FromString(""), + "full name cannot be empty or whitespace only"); + ASSERT_NOK_WITH_MSG(Identifier::FromString(" "), + "full name cannot be empty or whitespace only"); + ASSERT_NOK_WITH_MSG(Identifier::FromString("abcd"), "cannot get splits from 'abcd'"); + ASSERT_NOK_WITH_MSG(Identifier::FromString(".abcd"), "cannot get splits from '.abcd'"); + ASSERT_NOK_WITH_MSG(Identifier::FromString("abcd."), "cannot get splits from 'abcd.'"); + } + { + // test normal database + ASSERT_OK_AND_ASSIGN(Identifier identifier, Identifier::FromString("ab.cd")); + Identifier expected("ab", "cd"); + ASSERT_EQ(identifier, expected); + } + { + // test normal database + ASSERT_OK_AND_ASSIGN(Identifier identifier, Identifier::FromString("ab.cd.ef")); + Identifier expected("ab", "cd.ef"); + ASSERT_EQ(identifier, expected); + } +} + TEST(IdentifierTest, InvalidSystemTableName) { Identifier invalid_middle("db", "tbl$bad$options"); ASSERT_NOK_WITH_MSG(invalid_middle.IsSystemTable(), diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 81c19c65..516ce826 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -19,18 +19,32 @@ #include "paimon/core/operation/data_evolution_split_read.h" #include +#include +#include +#include #include +#include "arrow/array/array_base.h" +#include "arrow/array/array_binary.h" +#include "arrow/array/array_nested.h" +#include "arrow/c/bridge.h" +#include "paimon/common/catalog/catalog_context.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/blob_view_struct.h" #include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h" #include "paimon/common/global_index/complete_index_score_batch_reader.h" +#include "paimon/common/reader/blob_view_resolving_batch_reader.h" #include "paimon/common/reader/complete_row_kind_batch_reader.h" #include "paimon/common/reader/concat_batch_reader.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/object_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/range_helper.h" #include "paimon/core/core_options.h" #include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/utils/blob_view_lookup.h" namespace paimon { Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptr& file) { if (!BlobUtils::IsBlobFile(file->file_name)) { @@ -115,28 +129,157 @@ bool DataEvolutionSplitRead::HasIndexScoreField(const std::shared_ptrGetFieldIndex(SpecialFields::IndexScore().Name()) != -1; } +std::vector DataEvolutionSplitRead::HasBlobViewField( + const CoreOptions& options, const std::shared_ptr& read_schema) { + std::vector read_blob_view_fields; + std::vector blob_view_fields = options.GetBlobViewFields(); + for (const auto& blob : blob_view_fields) { + if (read_schema->GetFieldByName(blob)) { + read_blob_view_fields.push_back(blob); + } + } + return read_blob_view_fields; +} + Result> DataEvolutionSplitRead::CreateReader( const std::shared_ptr& split) { if (auto indexed_split = std::dynamic_pointer_cast(split)) { PAIMON_RETURN_NOT_OK(indexed_split->Validate()); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr batch_reader, - InnerCreateReader(indexed_split->GetDataSplit(), indexed_split->RowRanges())); + const auto& data_split = indexed_split->GetDataSplit(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + InnerCreateReader(data_split, indexed_split->RowRanges())); if (HasIndexScoreField(raw_read_schema_)) { - return std::make_unique(std::move(batch_reader), - indexed_split->Scores(), pool_); + batch_reader = std::make_unique( + std::move(batch_reader), indexed_split->Scores(), pool_); } - return batch_reader; + return WrapWithBlobViewResolverIfNeeded(data_split, std::move(batch_reader)); } else if (auto data_split = std::dynamic_pointer_cast(split)) { if (HasIndexScoreField(raw_read_schema_)) { return Status::Invalid( "Invalid read schema, read _INDEX_SCORE while split cannot cast to IndexedSplit"); } - return InnerCreateReader(data_split, /*row_ranges=*/std::nullopt); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr inner_reader, + InnerCreateReader(data_split, /*row_ranges=*/std::nullopt)); + return WrapWithBlobViewResolverIfNeeded(data_split, std::move(inner_reader)); } return Status::Invalid("Invalid Split, cannot cast to IndexedSplit or DataSplit"); } +Result> DataEvolutionSplitRead::WrapWithBlobViewResolverIfNeeded( + const std::shared_ptr& data_split, + std::unique_ptr&& inner_reader) const { + std::vector read_blob_view_fields = HasBlobViewField(options_, raw_read_schema_); + if (read_blob_view_fields.empty()) { + return std::move(inner_reader); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr pre_reader, + CreateBlobViewReader(data_split, read_blob_view_fields)); + PAIMON_ASSIGN_OR_RAISE(std::unordered_set blob_view_structs, + ExtractBlobViewStructs(pre_reader.get())); + std::string warehouse_path = + PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(context_->GetPath())); + auto catalog_context = std::make_shared(warehouse_path, options_.ToMap(), + options_.GetFileSystem()); + PAIMON_ASSIGN_OR_RAISE( + BlobViewResolver resolver, + BlobViewLookup::CreateResolver(blob_view_structs, catalog_context, pool_)); + return std::make_unique( + std::move(inner_reader), std::move(read_blob_view_fields), std::move(resolver), pool_); +} + +Result> DataEvolutionSplitRead::CreateBlobViewReader( + const std::shared_ptr& data_split, + const std::vector& read_blob_view_fields) const { + auto split_impl = dynamic_cast(data_split.get()); + if (split_impl == nullptr) { + return Status::Invalid("unexpected error, split cast to impl failed"); + } + assert(raw_read_schema_->num_fields() > 0); + + arrow::FieldVector blob_view_arrow_fields; + blob_view_arrow_fields.reserve(read_blob_view_fields.size()); + for (const auto& field_name : read_blob_view_fields) { + auto field = raw_read_schema_->GetFieldByName(field_name); + if (field == nullptr) { + return Status::Invalid( + fmt::format("Blob view field {} not found in read schema.", field_name)); + } + blob_view_arrow_fields.push_back(std::move(field)); + } + auto blob_view_schema = arrow::schema(std::move(blob_view_arrow_fields)); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr data_file_path_factory, + path_factory_->CreateDataFilePathFactory(split_impl->Partition(), split_impl->Bucket())); + + // skip blob files: they only contain blob payloads, not the blob-view columns. + std::vector> data_files; + data_files.reserve(split_impl->DataFiles().size()); + for (const auto& file : split_impl->DataFiles()) { + if (!BlobUtils::IsBlobFile(file->file_name)) { + data_files.push_back(file); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::vector> raw_file_readers, + CreateRawFileReaders(split_impl->Partition(), data_files, blob_view_schema, + /*predicate=*/nullptr, /*dv_factory=*/nullptr, + /*row_ranges=*/std::nullopt, data_file_path_factory)); + + auto batch_readers = + ObjectUtils::MoveVector>(std::move(raw_file_readers)); + return std::make_unique(std::move(batch_readers), pool_); +} + +Result> DataEvolutionSplitRead::ExtractBlobViewStructs( + BatchReader* reader) { + if (reader == nullptr) { + return Status::Invalid("invalid reader in ExtractBlobViewStructs, reader is nullptr"); + } + std::unordered_set blob_view_structs; + while (true) { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + if (struct_array == nullptr) { + return Status::Invalid( + "invalid array in ExtractBlobViewStructs, batch array is not a StructArray."); + } + + for (int32_t field_idx = 0; field_idx < struct_array->num_fields(); ++field_idx) { + auto binary_array = + std::dynamic_pointer_cast(struct_array->field(field_idx)); + if (binary_array == nullptr) { + return Status::Invalid( + "invalid array in ExtractBlobViewStructs, blob view column is not a " + "LargeBinaryArray."); + } + for (int64_t row = 0; row < binary_array->length(); ++row) { + if (binary_array->IsNull(row)) { + continue; + } + std::string_view bytes = binary_array->GetView(row); + PAIMON_ASSIGN_OR_RAISE( + bool is_view, BlobViewStruct::IsBlobViewStruct(bytes.data(), bytes.size())); + if (!is_view) { + return Status::Invalid( + "blob-view-field requires blob field value to be a serialized " + "BlobViewStruct."); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr view_struct, + BlobViewStruct::Deserialize(bytes.data(), bytes.size())); + blob_view_structs.insert(*view_struct); + } + } + } + return blob_view_structs; +} + Result> DataEvolutionSplitRead::InnerCreateReader( const std::shared_ptr& data_split, const std::optional>& row_ranges) const { @@ -176,6 +319,7 @@ Result> DataEvolutionSplitRead::InnerCreateReader( ApplyPredicateFilterIfNeeded(std::move(concat_batch_reader), context_->GetPredicate())); return std::make_unique(std::move(batch_reader), pool_); } + Result> DataEvolutionSplitRead::ApplyIndexAndDvReaderIfNeeded( std::unique_ptr&& file_reader, const std::shared_ptr& file, const std::shared_ptr& data_schema, diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index 0129b007..16216216 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -23,8 +23,10 @@ #include #include #include +#include #include +#include "paimon/common/data/blob_view_struct.h" #include "paimon/common/reader/data_evolution_file_reader.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/operation/abstract_split_read.h" @@ -53,7 +55,8 @@ struct DeletionFile; /// otherwise, it must be present in the read path. /// /// Readers Overview: (ConcatBatchReader across -/// splits)->(CompleteIndexScoreBatchReader)->CompleteRowKindBatchReader->(PredicateBatchReader) +/// splits)->(BlobViewResolvingBatchReader)->(CompleteIndexScoreBatchReader)-> +/// CompleteRowKindBatchReader->(PredicateBatchReader) /// ->ConcatBatchReader across files->DataEvolutionFileReader->(ConcatBatchReader across blob files) /// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader) /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader @@ -143,6 +146,19 @@ class DataEvolutionSplitRead : public AbstractSplitRead { static bool HasIndexScoreField(const std::shared_ptr& read_schema); + static std::vector HasBlobViewField( + const CoreOptions& options, const std::shared_ptr& read_schema); + + static Result> ExtractBlobViewStructs(BatchReader* reader); + + Result> CreateBlobViewReader( + const std::shared_ptr& data_split, + const std::vector& read_blob_view_fields) const; + + Result> WrapWithBlobViewResolverIfNeeded( + const std::shared_ptr& data_split, + std::unique_ptr&& inner_reader) const; + private: Result> CreateUnionReader( const BinaryRow& partition, diff --git a/src/paimon/core/utils/blob_view_lookup.cpp b/src/paimon/core/utils/blob_view_lookup.cpp new file mode 100644 index 00000000..56979de6 --- /dev/null +++ b/src/paimon/core/utils/blob_view_lookup.cpp @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/blob_view_lookup.h" + +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/defs.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/memory/bytes.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/utils/special_field_ids.h" + +namespace paimon { +BlobViewLookup::TableReadPlan::TableReadPlan(const BlobViewStruct& view_struct) + : identifier_(view_struct.GetIdentifier()) { + references_by_field_id_.insert(view_struct.FieldId()); + row_ranges_.push_back(view_struct.RowId()); +} + +void BlobViewLookup::TableReadPlan::Add(const BlobViewStruct& view_struct) { + references_by_field_id_.insert(view_struct.FieldId()); + row_ranges_.push_back(view_struct.RowId()); +} + +std::vector BlobViewLookup::TableReadPlan::GetFieldIds() const { + return std::vector(references_by_field_id_.begin(), references_by_field_id_.end()); +} + +std::vector BlobViewLookup::TableReadPlan::GetSortedDistinctRanges() const { + if (row_ranges_.empty()) { + return {}; + } + std::vector sorted = row_ranges_; + std::sort(sorted.begin(), sorted.end()); + std::vector ranges; + int64_t range_start = sorted[0]; + int64_t range_end = range_start; + for (size_t i = 1; i < sorted.size(); ++i) { + const int64_t row_id = sorted[i]; + if (row_id == range_end) { + continue; + } + if (row_id != range_end + 1) { + ranges.emplace_back(range_start, range_end); + range_start = row_id; + } + range_end = row_id; + } + ranges.emplace_back(range_start, range_end); + return ranges; +} + +Result BlobViewLookup::CreateResolver( + const std::unordered_set& view_structs, + const std::shared_ptr& catalog_context, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(DescriptorMapping mapping, + PreloadDescriptors(view_structs, catalog_context, pool)); + return BlobViewResolver([cached = std::move(mapping)](const BlobViewStruct& view_struct) + -> Result> { + auto iter = cached.find(view_struct); + if (iter == cached.end()) { + return Status::Invalid(fmt::format("BlobViewStruct not found in preloaded cache: {}", + view_struct.ToString())); + } + return iter->second; + }); +} + +Result BlobViewLookup::PreloadDescriptors( + const std::unordered_set& view_structs, + const std::shared_ptr& catalog_context, + const std::shared_ptr& pool) { + std::unordered_map plan_by_identifier = + GroupByIdentifier(view_structs); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + Catalog::Create(catalog_context->root_path, catalog_context->options, + catalog_context->file_system)); + DescriptorMapping mapping; + for (const auto& [identifier, table_read_plan] : plan_by_identifier) { + std::string source_table_path = catalog->GetTableLocation(identifier); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + ScanContextBuilder scan_builder(source_table_path); + auto global_index_result = + BitmapGlobalIndexResult::FromRanges(table_read_plan.GetSortedDistinctRanges()); + scan_builder.SetGlobalIndexResult(global_index_result) + .WithMemoryPool(pool) + .WithFileSystem(catalog_context->file_system); + if (branch) { + scan_builder.AddOption(Options::BRANCH, branch.value()); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + + ReadContextBuilder read_builder(source_table_path); + std::vector field_ids = table_read_plan.GetFieldIds(); + field_ids.push_back(SpecialFieldIds::ROW_ID); + read_builder.SetReadFieldIds(field_ids) + .AddOption(Options::BLOB_AS_DESCRIPTOR, "true") + .EnablePrefetch(true) + .WithMemoryPool(pool) + .WithFileSystem(catalog_context->file_system); + if (branch) { + read_builder.WithBranch(branch.value()); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + table_read->CreateReader(plan->Splits())); + PAIMON_RETURN_NOT_OK( + ExtractBlobDescriptors(identifier, field_ids, pool, reader.get(), &mapping)); + } + return mapping; +} + +Status BlobViewLookup::ExtractBlobDescriptors(const Identifier& identifier, + const std::vector& field_ids, + const std::shared_ptr& pool, + BatchReader* reader, DescriptorMapping* mapping) { + if (reader == nullptr) { + return Status::Invalid("invalid reader in ExtractBlobDescriptors, reader is nullptr"); + } + while (true) { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + if (struct_array == nullptr) { + return Status::Invalid( + "invalid array in ExtractBlobDescriptors, batch array is not a StructArray."); + } + // skip the _VALUE_KIND column + if (static_cast(struct_array->num_fields()) - 1 != field_ids.size()) { + return Status::Invalid( + fmt::format("invalid array in ExtractBlobDescriptors, batch array fields(exclude " + "_VALUE_KIND) {} mismatch read field ids {}.", + struct_array->num_fields() - 1, field_ids.size())); + } + // get _VALUE_KIND + if (struct_array->struct_type()->field(0)->name() != SpecialFields::ValueKind().Name()) { + return Status::Invalid( + "invalid array in ExtractBlobDescriptors, expected _VALUE_KIND as the first " + "column"); + } + + // get _ROW_ID + if (struct_array->struct_type()->field(struct_array->num_fields() - 1)->name() != + SpecialFields::RowId().Name()) { + return Status::Invalid( + "invalid array in ExtractBlobDescriptors, expected _ROW_ID as the last column"); + } + auto row_id_array = struct_array->field(struct_array->num_fields() - 1); + auto typed_row_id_array = std::dynamic_pointer_cast(row_id_array); + if (!typed_row_id_array) { + return Status::Invalid( + fmt::format("invalid array does not contain {} field, or it cannot be casted to " + "Int64Array in ExtractBlobDescriptors.", + SpecialFields::RowId().Name())); + } + + // skip _VALUE_KIND + for (int32_t idx = 1; idx < struct_array->num_fields() - 1; ++idx) { + auto binary_array = + std::dynamic_pointer_cast(struct_array->field(idx)); + if (binary_array == nullptr) { + return Status::Invalid( + "invalid array in ExtractBlobDescriptors, column is not a LargeBinaryArray."); + } + for (int64_t row = 0; row < binary_array->length(); ++row) { + BlobViewStruct blob_view_struct(identifier, field_ids[idx - 1], + typed_row_id_array->Value(row)); + if (binary_array->IsNull(row)) { + // null in source table + (*mapping)[blob_view_struct] = nullptr; + continue; + } + std::string_view bytes = binary_array->GetView(row); + PAIMON_ASSIGN_OR_RAISE(bool is_descriptor, BlobDescriptor::IsBlobDescriptor( + bytes.data(), bytes.size())); + if (!is_descriptor) { + return Status::Invalid( + "requires blob field value to be a serialized BlobDescriptor in source " + "table."); + } + auto descriptor_bytes = std::make_shared(bytes.size(), pool.get()); + std::memcpy(descriptor_bytes->data(), bytes.data(), bytes.size()); + (*mapping)[blob_view_struct] = std::move(descriptor_bytes); + } + } + } + return Status::OK(); +} + +std::unordered_map BlobViewLookup::GroupByIdentifier( + const std::unordered_set& view_structs) { + std::unordered_map grouped; + for (const auto& view_struct : view_structs) { + auto identifier = view_struct.GetIdentifier(); + auto iter = grouped.find(identifier); + if (iter != grouped.end()) { + iter->second.Add(view_struct); + } else { + grouped.emplace(identifier, BlobViewLookup::TableReadPlan(view_struct)); + } + } + return grouped; +} + +} // namespace paimon diff --git a/src/paimon/core/utils/blob_view_lookup.h b/src/paimon/core/utils/blob_view_lookup.h new file mode 100644 index 00000000..81602945 --- /dev/null +++ b/src/paimon/core/utils/blob_view_lookup.h @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/common/catalog/catalog_context.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/memory/bytes.h" +#include "paimon/result.h" +#include "paimon/utils/range.h" + +namespace paimon { + +class BatchReader; + +/// Provide a function for converting {@link BlobViewStruct}s to {@link BlobDescriptor}s by scanning +/// the upstream tables in row-range chunks. +class BlobViewLookup { + public: + using DescriptorMapping = std::unordered_map>; + + BlobViewLookup() = delete; + ~BlobViewLookup() = delete; + + static Result CreateResolver( + const std::unordered_set& view_structs, + const std::shared_ptr& catalog_context, + const std::shared_ptr& pool); + + private: + class TableReadPlan { + public: + explicit TableReadPlan(const BlobViewStruct& view_struct); + + void Add(const BlobViewStruct& view_struct); + std::vector GetFieldIds() const; + std::vector GetSortedDistinctRanges() const; + + private: + Identifier identifier_; + std::set references_by_field_id_; + std::vector row_ranges_; + }; + + static Result PreloadDescriptors( + const std::unordered_set& view_structs, + const std::shared_ptr& catalog_context, + const std::shared_ptr& pool); + + static Status ExtractBlobDescriptors(const Identifier& identifier, + const std::vector& field_ids, + const std::shared_ptr& pool, + BatchReader* reader, DescriptorMapping* mapping); + + static std::unordered_map GroupByIdentifier( + const std::unordered_set& view_structs); +}; + +} // namespace paimon diff --git a/src/paimon/core/utils/blob_view_lookup_test.cpp b/src/paimon/core/utils/blob_view_lookup_test.cpp new file mode 100644 index 00000000..6dc2b89c --- /dev/null +++ b/src/paimon/core/utils/blob_view_lookup_test.cpp @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/blob_view_lookup.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/data/blob_view_struct.h" +#include "paimon/utils/range.h" + +namespace paimon::test { + +class BlobViewLookupTest : public testing::Test { + public: + static Identifier MakeIdentifier(const std::string& database, const std::string& table) { + return Identifier(database, table); + } + + static BlobViewStruct MakeView(const std::string& database, const std::string& table, + int32_t field_id, int64_t row_id) { + return BlobViewStruct(MakeIdentifier(database, table), field_id, row_id); + } +}; + +TEST_F(BlobViewLookupTest, TestConstruct) { + BlobViewStruct view = MakeView("db", "t", /*field_id=*/7, /*row_id=*/42); + BlobViewLookup::TableReadPlan plan(view); + + ASSERT_EQ(plan.identifier_, MakeIdentifier("db", "t")); + const auto& references = plan.references_by_field_id_; + ASSERT_EQ(references.size(), 1U); + auto iter = references.find(7); + ASSERT_NE(iter, references.end()); + + const auto& row_ranges = plan.row_ranges_; + ASSERT_EQ(row_ranges.size(), 1U); + ASSERT_EQ(row_ranges[0], 42); + + ASSERT_EQ(plan.GetFieldIds(), std::vector{7}); +} + +TEST_F(BlobViewLookupTest, TestAdd) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/7, /*row_id=*/10)); + + plan.Add(MakeView("db", "t", /*field_id=*/7, /*row_id=*/12)); + plan.Add(MakeView("db", "t", /*field_id=*/9, /*row_id=*/11)); + plan.Add(MakeView("db", "t", /*field_id=*/9, /*row_id=*/13)); + + const auto& references = plan.references_by_field_id_; + ASSERT_EQ(references.size(), 2U); + auto iter = references.find(7); + ASSERT_NE(iter, references.end()); + iter = references.find(9); + ASSERT_NE(iter, references.end()); + + ASSERT_EQ(plan.row_ranges_, (std::vector{10, 12, 11, 13})); +} + +TEST_F(BlobViewLookupTest, TestGetFieldIds) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/3, /*row_id=*/0)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/1)); + plan.Add(MakeView("db", "t", /*field_id=*/3, /*row_id=*/2)); + plan.Add(MakeView("db", "t", /*field_id=*/2, /*row_id=*/3)); + + ASSERT_EQ(plan.GetFieldIds(), (std::vector{1, 2, 3})); +} + +TEST_F(BlobViewLookupTest, TestGetSortedDistinctRangesMergesTwoAdjacentRowIds) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/1, /*row_id=*/5)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/6)); + + auto ranges = plan.GetSortedDistinctRanges(); + ASSERT_EQ(ranges.size(), 1U); + ASSERT_EQ(ranges[0], Range(5, 6)); +} + +TEST_F(BlobViewLookupTest, TestGetSortedDistinctRangesMergesContiguousAndGaps) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/1, /*row_id=*/5)); + // Out of order, with duplicates and a gap so we get two output ranges. + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/6)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/5)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/7)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/10)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/11)); + + auto ranges = plan.GetSortedDistinctRanges(); + ASSERT_EQ(ranges.size(), 2U); + ASSERT_EQ(ranges[0].from, 5); + ASSERT_EQ(ranges[0].to, 7); + ASSERT_EQ(ranges[1].from, 10); + ASSERT_EQ(ranges[1].to, 11); +} + +TEST_F(BlobViewLookupTest, TestGetSortedDistinctRangesWithNonContiguous) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/1, /*row_id=*/1)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/100)); + plan.Add(MakeView("db", "t", /*field_id=*/1, /*row_id=*/50)); + + const auto ranges = plan.GetSortedDistinctRanges(); + ASSERT_EQ(ranges.size(), 3U); + ASSERT_EQ(ranges[0].from, 1); + ASSERT_EQ(ranges[0].to, 1); + ASSERT_EQ(ranges[1].from, 50); + ASSERT_EQ(ranges[1].to, 50); + ASSERT_EQ(ranges[2].from, 100); + ASSERT_EQ(ranges[2].to, 100); +} + +TEST_F(BlobViewLookupTest, TestEmptyInputProducesEmptyOutput) { + auto grouped = BlobViewLookup::GroupByIdentifier({}); + ASSERT_TRUE(grouped.empty()); +} + +TEST_F(BlobViewLookupTest, TestSingleViewStructProducesSingleGroup) { + std::unordered_set views; + views.emplace(MakeView("db", "t", /*field_id=*/3, /*row_id=*/42)); + + auto grouped = BlobViewLookup::GroupByIdentifier(views); + ASSERT_EQ(grouped.size(), 1U); + + auto iter = grouped.find(MakeIdentifier("db", "t")); + ASSERT_NE(iter, grouped.end()); + const auto& plan = iter->second; + ASSERT_EQ(plan.GetFieldIds(), std::vector{3}); + ASSERT_EQ(plan.row_ranges_, std::vector{42}); +} + +TEST_F(BlobViewLookupTest, TestMultipleViewStructsOfSameTableAreMergedIntoOnePlan) { + std::unordered_set views; + views.emplace(MakeView("db", "t", /*field_id=*/3, /*row_id=*/1)); + views.emplace(MakeView("db", "t", /*field_id=*/3, /*row_id=*/2)); + views.emplace(MakeView("db", "t", /*field_id=*/4, /*row_id=*/1)); + + auto grouped = BlobViewLookup::GroupByIdentifier(views); + ASSERT_EQ(grouped.size(), 1U); + + const auto& plan = grouped.at(MakeIdentifier("db", "t")); + ASSERT_EQ(plan.GetFieldIds(), (std::vector{3, 4})); + + const auto& references = plan.references_by_field_id_; + ASSERT_EQ(references.size(), 2U); + auto iter = references.find(3); + ASSERT_NE(iter, references.end()); + iter = references.find(4); + ASSERT_NE(iter, references.end()); + + ASSERT_EQ(plan.row_ranges_.size(), 3U); +} + +TEST_F(BlobViewLookupTest, TestViewStructsOfDifferentTablesAreSplitIntoDistinctPlans) { + std::unordered_set views; + views.emplace(MakeView("db1", "t1", /*field_id=*/3, /*row_id=*/1)); + views.emplace(MakeView("db1", "t2", /*field_id=*/3, /*row_id=*/1)); + views.emplace(MakeView("db2", "t1", /*field_id=*/3, /*row_id=*/1)); + views.emplace(MakeView("db1", "t1", /*field_id=*/4, /*row_id=*/2)); + + auto grouped = BlobViewLookup::GroupByIdentifier(views); + ASSERT_EQ(grouped.size(), 3U); + + const auto& plan_db1_t1 = grouped.at(MakeIdentifier("db1", "t1")); + ASSERT_EQ(plan_db1_t1.GetFieldIds(), (std::vector{3, 4})); + ASSERT_EQ(plan_db1_t1.row_ranges_.size(), 2U); + + const auto& plan_db1_t2 = grouped.at(MakeIdentifier("db1", "t2")); + ASSERT_EQ(plan_db1_t2.GetFieldIds(), std::vector{3}); + ASSERT_EQ(plan_db1_t2.row_ranges_.size(), 1U); + + const auto& plan_db2_t1 = grouped.at(MakeIdentifier("db2", "t1")); + ASSERT_EQ(plan_db2_t1.GetFieldIds(), std::vector{3}); + ASSERT_EQ(plan_db2_t1.row_ranges_.size(), 1U); +} + +} // namespace paimon::test diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 30e439e6..1689e7cf 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -40,6 +40,7 @@ #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/data/blob_view_struct.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/path_util.h" @@ -2377,4 +2378,166 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) { "to be a BlobDescriptor or BlobViewStruct."); } +TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { + auto file_format = GetParam(); + if (file_format != "orc" && file_format != "parquet") { + return; + } + + const std::string upstream_db_name = "append_table_with_multi_blob"; + const std::string upstream_table_name = "append_table_with_multi_blob"; + std::string src_db_path = paimon::test::GetDataDir() + file_format + "/" + upstream_db_name + + ".db/" + upstream_table_name; + std::string dst_db_path = + PathUtil::JoinPath(dir_->Str(), upstream_db_name + ".db/" + upstream_table_name); + ASSERT_TRUE(TestUtil::CopyDirectory(src_db_path, dst_db_path)); + + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // src array + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + arrow::LargeBinaryBuilder view_builder; + for (int32_t i = 0; i < 8; ++i) { + if (i < 6) { + BlobViewStruct view_struct(upstream_identifier, /*field_id=*/6, + /*row_id=*/static_cast(i)); + auto serialized = view_struct.Serialize(GetDefaultPool()); + ASSERT_TRUE(view_builder + .Append(reinterpret_cast(serialized->data()), + serialized->size()) + .ok()); + } else { + ASSERT_TRUE(view_builder.AppendNull().ok()); + } + } + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + auto write_f0_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int32(), R"([100,101,102,103,104,105,106,107])") + .ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + // write & commit + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + std::string padding_b(2048, 'b'); + std::string padding_d(2048, 'd'); + std::string padding_e(2048, 'e'); + std::string padding_f(2048, 'f'); + + // scan & read + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 8); + ASSERT_OK_AND_ASSIGN(auto result_array, ConvertDescriptorToRawBlob(read_struct, {"view"})); + + // clang-format off + std::string expected_json = R"([ +[100, null], +[101, ")" + padding_b + R"("], +[102, null], +[103, ")" + padding_d + R"("], +[104, ")" + padding_e + R"("], +[105, ")" + padding_f + R"("], +[106, null], +[107, null] +])"; + // clang-format on + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + + ASSERT_TRUE(result_array->Equals(expected_with_rk)) + << "result_array:" << result_array->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); + + // Sub-case 1: scan with row_ranges. + { + ASSERT_OK_AND_ASSIGN(auto range_plan, ScanTable(table_path, /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 3), Range(5, 5)})); + ASSERT_OK_AND_ASSIGN(auto range_result, ReadTable(table_path, schema->field_names(), + range_plan, /*predicate=*/nullptr)); + ASSERT_TRUE(range_result.chunked_array); + auto range_concat = arrow::Concatenate(range_result.chunked_array->chunks()).ValueOrDie(); + auto range_struct = std::dynamic_pointer_cast(range_concat); + ASSERT_EQ(range_struct->length(), 4); + ASSERT_OK_AND_ASSIGN(auto range_resolved, + ConvertDescriptorToRawBlob(range_struct, {"view"})); + + // clang-format off + std::string range_json = R"([ +[101, ")" + padding_b + R"("], +[102, null], +[103, ")" + padding_d + R"("], +[105, ")" + padding_f + R"("] +])"; + // clang-format on + auto range_expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), range_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto range_expected_with_rk, + PrependRowKindColumn(range_expected_struct)); + ASSERT_TRUE(range_resolved->Equals(range_expected_with_rk)) + << "range_resolved:" << range_resolved->ToString() << std::endl + << "expected:" << range_expected_with_rk->ToString(); + } + + // Sub-case 2: scan with predicate (f0 > 102), data evolution split read will ignore format push + // down + { + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(102)); + ASSERT_OK_AND_ASSIGN(auto pred_plan, ScanTable(table_path, predicate, /*row_ranges=*/{})); + ASSERT_OK_AND_ASSIGN(auto pred_result, + ReadTable(table_path, schema->field_names(), pred_plan, predicate)); + ASSERT_TRUE(pred_result.chunked_array); + auto pred_concat = arrow::Concatenate(pred_result.chunked_array->chunks()).ValueOrDie(); + auto pred_struct = std::dynamic_pointer_cast(pred_concat); + ASSERT_EQ(pred_struct->length(), 8); + ASSERT_OK_AND_ASSIGN(auto pred_resolved, ConvertDescriptorToRawBlob(pred_struct, {"view"})); + + // clang-format off + std::string pred_json = R"([ +[100, null], +[101, ")" + padding_b + R"("], +[102, null], +[103, ")" + padding_d + R"("], +[104, ")" + padding_e + R"("], +[105, ")" + padding_f + R"("], +[106, null], +[107, null] +])"; + // clang-format on + auto pred_expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), pred_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto pred_expected_with_rk, + PrependRowKindColumn(pred_expected_struct)); + ASSERT_TRUE(pred_resolved->Equals(pred_expected_with_rk)) + << "pred_resolved:" << pred_resolved->ToString() << std::endl + << "expected:" << pred_expected_with_rk->ToString(); + } +} + } // namespace paimon::test From 08c0cc0ec234ad02b4901def8beb0ae88401f105 Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Thu, 4 Jun 2026 10:44:16 +0800 Subject: [PATCH 027/138] fix: fix unstable prefetch reader case --- .../common/reader/prefetch_file_batch_reader_impl_test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp index d2d6b5aa..165b9bf7 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp @@ -486,7 +486,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, SetReadRangesReturnErrorWhenPushDownFail data_array, data_type_, batch_size, /*read_ranges=*/{{0, 50}, {50, 100}}, /*need_prefetch=*/true, - /*set_read_ranges_statuses=*/{Status::OK(), Status::IOError("set read ranges failed")}); + /*set_read_ranges_statuses=*/ + {Status::IOError("set read ranges failed"), Status::IOError("set read ranges failed")}); ASSERT_OK_AND_ASSIGN( auto reader, From 0f099cca1456c3d7d1c5f39e24dad43032712468 Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:12:51 +0800 Subject: [PATCH 028/138] feat: remove support for FLOAT/DOUBLE type partition field From beaae3c629d4a08c9e0dfab1c91d46befb0d9833 Mon Sep 17 00:00:00 2001 From: Joey Date: Thu, 4 Jun 2026 16:19:07 +0800 Subject: [PATCH 029/138] fix: skip object store check when UseRESTCatalogCommit is enabled --- .../core/operation/file_store_commit.cpp | 3 ++- .../operation/file_store_commit_impl_test.cpp | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 8cc4ba9a..194b4e46 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -84,7 +84,8 @@ Result> FileStoreCommit::Create( assert(options.GetFileSystem()); assert(options.GetFileFormat()); PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); - if (is_object_store && opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { + if (is_object_store && !ctx->UseRESTCatalogCommit() && + opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { return Status::NotImplemented( "commit operation does not support object store file system for now"); } diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index be42387b..d609821a 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -1663,4 +1663,31 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { ASSERT_TRUE(commit_run_complete); } +TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { + // Verify: the object store check in FileStoreCommit::Create is skipped when + // UseRESTCatalogCommit is true. We can't use an actual oss:// path in unit + // tests (LocalFileSystem rejects the scheme), but we verify the condition + // by confirming that the "enable-object-store-commit-in-inte-test" flag is + // not needed when UseRESTCatalogCommit is enabled. End-to-end oss:// testing + // is covered by duckdb-paimon integration tests. + ASSERT_OK_AND_ASSIGN(bool is_oss, FileSystem::IsObjectStore("oss://bucket/path")); + ASSERT_TRUE(is_oss); + + // REST commit with local path should work without the object store flag + CommitContextBuilder builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN( + auto ctx, + builder.AddOption(Options::MANIFEST_FORMAT, "orc").UseRESTCatalogCommit(true).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(ctx))); + + auto msgs = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + 3); + ASSERT_GT(msgs.size(), 0); + ASSERT_OK(commit->Commit(msgs)); + ASSERT_OK_AND_ASSIGN(auto json, commit->GetLastCommitTableRequest()); + ASSERT_FALSE(json.empty()); +} + } // namespace paimon::test From 28e2b827ed2cd104efd0d9251161c856fbc97b31 Mon Sep 17 00:00:00 2001 From: fourier307 <8501328+fourier307@users.noreply.github.com> Date: Thu, 4 Jun 2026 18:54:09 +0800 Subject: [PATCH 030/138] feat: add Bucket() virtual interface to DataSplit base class --- include/paimon/table/source/data_split.h | 3 +++ src/paimon/core/table/source/data_split_impl.h | 2 +- src/paimon/core/table/source/fallback_data_split.h | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/include/paimon/table/source/data_split.h b/include/paimon/table/source/data_split.h index 865e785c..4de0d1cf 100644 --- a/include/paimon/table/source/data_split.h +++ b/include/paimon/table/source/data_split.h @@ -77,6 +77,9 @@ class PAIMON_EXPORT DataSplit : public Split { std::string ToString() const; }; + /// Get the bucket id of this data split. + virtual int32_t Bucket() const = 0; + /// Get the list of metadata for all data files in this split. /// @note This method will be removed in future versions and is only used for append tables. virtual std::vector GetFileList() const = 0; diff --git a/src/paimon/core/table/source/data_split_impl.h b/src/paimon/core/table/source/data_split_impl.h index 05076ab7..8dbe5377 100644 --- a/src/paimon/core/table/source/data_split_impl.h +++ b/src/paimon/core/table/source/data_split_impl.h @@ -54,7 +54,7 @@ class DataSplitImpl : public DataSplit { return partition_; } - int32_t Bucket() const { + int32_t Bucket() const override { return bucket_; } diff --git a/src/paimon/core/table/source/fallback_data_split.h b/src/paimon/core/table/source/fallback_data_split.h index c8d23987..897d5fad 100644 --- a/src/paimon/core/table/source/fallback_data_split.h +++ b/src/paimon/core/table/source/fallback_data_split.h @@ -30,6 +30,10 @@ class FallbackDataSplit : public DataSplit { FallbackDataSplit(const std::shared_ptr& split, bool is_fallback) : is_fallback_(is_fallback), split_(split) {} + int32_t Bucket() const override { + return split_->Bucket(); + } + std::vector GetFileList() const override { return split_->GetFileList(); } From 59853144ef40d6ae3fc614c6b1c9136894d67623 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Fri, 5 Jun 2026 10:09:39 +0800 Subject: [PATCH 031/138] fix(fs): normalize local filesystem paths and some refactoring * fix(fs): normalize local paths * fix(fs): simplify external path position update * fix(schema): treat rowkind as special field * refactor(fs): return LocalFile create as unique ptr From 91445ea2ca8bf2a1aa27b09d983427c46d31becd Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Fri, 5 Jun 2026 13:20:02 +0800 Subject: [PATCH 032/138] feat(benchmark): add append/pk table benchmark --- CMakeLists.txt | 47 +- benchmark/CMakeLists.txt | 78 +++ benchmark/benchmark_case_mor_read.cpp | 36 + benchmark/benchmark_case_pk_write.cpp | 30 + benchmark/benchmark_case_read.cpp | 36 + benchmark/benchmark_case_write.cpp | 30 + benchmark/benchmark_helpers.cpp | 94 +++ benchmark/benchmark_helpers.h | 62 ++ benchmark/benchmark_suite.cpp | 882 ++++++++++++++++++++++++ benchmark/benchmark_suite.h | 38 + benchmark/cli_option_parsing.h | 181 +++++ benchmark/cli_option_parsing_test.cpp | 180 +++++ benchmark/read_write_benchmark.cpp | 46 ++ cmake_modules/BuildUtils.cmake | 118 ++++ cmake_modules/DefineOptions.cmake | 10 + cmake_modules/FindbenchmarkAlt.cmake | 62 ++ cmake_modules/ThirdpartyToolchain.cmake | 62 ++ docs/source/examples/benchmark.rst | 88 +++ docs/source/examples/index.rst | 1 + src/paimon/testing/utils/CMakeLists.txt | 5 +- third_party/versions.txt | 5 + 21 files changed, 2084 insertions(+), 7 deletions(-) create mode 100644 benchmark/CMakeLists.txt create mode 100644 benchmark/benchmark_case_mor_read.cpp create mode 100644 benchmark/benchmark_case_pk_write.cpp create mode 100644 benchmark/benchmark_case_read.cpp create mode 100644 benchmark/benchmark_case_write.cpp create mode 100644 benchmark/benchmark_helpers.cpp create mode 100644 benchmark/benchmark_helpers.h create mode 100644 benchmark/benchmark_suite.cpp create mode 100644 benchmark/benchmark_suite.h create mode 100644 benchmark/cli_option_parsing.h create mode 100644 benchmark/cli_option_parsing_test.cpp create mode 100644 benchmark/read_write_benchmark.cpp create mode 100644 cmake_modules/FindbenchmarkAlt.cmake create mode 100644 docs/source/examples/benchmark.rst diff --git a/CMakeLists.txt b/CMakeLists.txt index 5cc1ca1d..64d6044c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(PAIMON_BUILD_STATIC "Build static library" ON) option(PAIMON_BUILD_SHARED "Build shared library" ON) option(PAIMON_BUILD_TESTS "Build tests" OFF) +option(PAIMON_BUILD_BENCHMARKS "Build benchmarks" OFF) option(PAIMON_USE_ASAN "Use Address Sanitizer" OFF) option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF) option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON) @@ -329,6 +330,29 @@ endif() set(ENV{PAIMON_TEST_DATA} "${CMAKE_SOURCE_DIR}/test/test_data") +if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS) + resolve_dependency(GTest) + include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) + + paimon_link_libraries_whole_archive(PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS + paimon_local_file_system_static) + paimon_link_libraries_no_as_needed(PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS + paimon_local_file_system_shared) + paimon_link_libraries_whole_archive(PAIMON_BLOB_FILE_FORMAT_STATIC_LINK_LIBS + paimon_blob_file_format_static) + paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS + paimon_parquet_file_format_static) + + if(PAIMON_ENABLE_ORC) + paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS + paimon_orc_file_format_static) + endif() + if(PAIMON_ENABLE_AVRO) + paimon_link_libraries_whole_archive(PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS + paimon_avro_file_format_static) + endif() +endif() + if(PAIMON_BUILD_TESTS) if(NOT PAIMON_ENABLE_ORC) message(FATAL_ERROR "PAIMON_ENABLE_ORC must be enabled if PAIMON_BUILD_TESTS is enable" @@ -340,7 +364,6 @@ if(PAIMON_BUILD_TESTS) endif() # Adding unit tests part of the "paimon" portion of the test suite add_custom_target(paimon-tests) - resolve_dependency(GTest) add_custom_target(unittest ctest @@ -350,7 +373,6 @@ if(PAIMON_BUILD_TESTS) --output-on-failure) add_dependencies(unittest paimon-tests) - include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) include_directories("${CMAKE_SOURCE_DIR}/test/") paimon_link_libraries_whole_archive( @@ -373,15 +395,11 @@ if(PAIMON_BUILD_TESTS) paimon_parquet_file_format_static) if(PAIMON_ENABLE_ORC) - paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS - paimon_orc_file_format_static) paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS paimon_orc_file_format_shared) list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) endif() if(PAIMON_ENABLE_AVRO) - paimon_link_libraries_whole_archive(PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS - paimon_avro_file_format_static) paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS paimon_avro_file_format_shared) list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS}) @@ -409,6 +427,19 @@ if(PAIMON_BUILD_TESTS) endif() endif() +if(PAIMON_BUILD_BENCHMARKS) + add_custom_target(paimon-benchmarks) + add_custom_target(benchmark + ctest + -j4 + -L + benchmark + --output-on-failure) + add_dependencies(benchmark paimon-benchmarks) + + set(PAIMON_BENCHMARK_LINK_TOOLCHAIN benchmark::benchmark) +endif() + paimon_print_dependency_resolution_summary() include(CMakePackageConfigHelpers) @@ -447,3 +478,7 @@ add_subdirectory(test/inte) install(EXPORT PaimonTargets NAMESPACE Paimon:: DESTINATION ${PAIMON_CMAKE_INSTALL_DIR}) + +if(PAIMON_BUILD_BENCHMARKS) + add_subdirectory(benchmark) +endif() diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 00000000..9b160fcd --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if(NOT PAIMON_BUILD_BENCHMARKS AND NOT PAIMON_BUILD_TESTS) + return() +endif() + +find_package(Threads REQUIRED) + +set(PAIMON_BENCHMARK_STATIC_LINK_LIBS + paimon_shared ${PAIMON_LOCAL_FILE_SYSTEM_SHARED_LINK_LIBS} + ${PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS} + ${PAIMON_BLOB_FILE_FORMAT_STATIC_LINK_LIBS}) + +if(PAIMON_ENABLE_ORC) + list(APPEND PAIMON_BENCHMARK_STATIC_LINK_LIBS + ${PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS}) +endif() + +if(PAIMON_ENABLE_AVRO) + list(APPEND PAIMON_BENCHMARK_STATIC_LINK_LIBS + ${PAIMON_AVRO_FILE_FORMAT_STATIC_LINK_LIBS}) +endif() + +set(PAIMON_BENCHMARK_PLATFORM_LINK_LIBS) +if(UNIX AND NOT APPLE) + find_library(PAIMON_BENCHMARK_RT_LIBRARY rt) + if(PAIMON_BENCHMARK_RT_LIBRARY) + list(APPEND PAIMON_BENCHMARK_PLATFORM_LINK_LIBS ${PAIMON_BENCHMARK_RT_LIBRARY}) + endif() +endif() + +if(PAIMON_BUILD_BENCHMARKS) + add_paimon_benchmark(read_write_benchmark + SOURCES + benchmark_helpers.cpp + benchmark_suite.cpp + benchmark_case_write.cpp + benchmark_case_read.cpp + benchmark_case_pk_write.cpp + benchmark_case_mor_read.cpp + read_write_benchmark.cpp + STATIC_LINK_LIBS + arrow + parquet + ${PAIMON_BENCHMARK_STATIC_LINK_LIBS} + test_utils_static + Threads::Threads + ${CMAKE_DL_LIBS} + ${PAIMON_BENCHMARK_PLATFORM_LINK_LIBS} + ${PAIMON_BENCHMARK_LINK_TOOLCHAIN} + EXTRA_INCLUDES + ${CMAKE_SOURCE_DIR}) +endif() + +if(PAIMON_BUILD_TESTS) + add_paimon_test(cli_option_parsing_test + SOURCES + cli_option_parsing_test.cpp + EXTRA_INCLUDES + ${CMAKE_SOURCE_DIR} + STATIC_LINK_LIBS + paimon_shared + ${GTEST_LINK_TOOLCHAIN}) +endif() diff --git a/benchmark/benchmark_case_mor_read.cpp b/benchmark/benchmark_case_mor_read.cpp new file mode 100644 index 00000000..b5d8ea62 --- /dev/null +++ b/benchmark/benchmark_case_mor_read.cpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +namespace { + +void BM_MOR_Read(::benchmark::State& state) { + paimon::benchmark::RunBMMorRead(state); +} + +} // namespace + +BENCHMARK(BM_MOR_Read) + ->ArgNames({"prefetch_parallel"}) + ->Unit(benchmark::kMillisecond) + ->UseRealTime() + ->Args({1}) + ->Args({2}) + ->Args({4}); diff --git a/benchmark/benchmark_case_pk_write.cpp b/benchmark/benchmark_case_pk_write.cpp new file mode 100644 index 00000000..546e7c7a --- /dev/null +++ b/benchmark/benchmark_case_pk_write.cpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +namespace { + +void BM_PK_Write(::benchmark::State& state) { + paimon::benchmark::RunBMPkWrite(state); +} + +} // namespace + +BENCHMARK(BM_PK_Write)->Unit(benchmark::kMillisecond)->UseRealTime(); diff --git a/benchmark/benchmark_case_read.cpp b/benchmark/benchmark_case_read.cpp new file mode 100644 index 00000000..583361d9 --- /dev/null +++ b/benchmark/benchmark_case_read.cpp @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +namespace { + +void BM_Read(::benchmark::State& state) { + paimon::benchmark::RunBMRead(state); +} + +} // namespace + +BENCHMARK(BM_Read) + ->ArgNames({"prefetch_parallel"}) + ->Unit(benchmark::kMillisecond) + ->UseRealTime() + ->Args({1}) + ->Args({2}) + ->Args({4}); diff --git a/benchmark/benchmark_case_write.cpp b/benchmark/benchmark_case_write.cpp new file mode 100644 index 00000000..5a60e660 --- /dev/null +++ b/benchmark/benchmark_case_write.cpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +namespace { + +void BM_Write(::benchmark::State& state) { + paimon::benchmark::RunBMWrite(state); +} + +} // namespace + +BENCHMARK(BM_Write)->Unit(benchmark::kMillisecond)->UseRealTime(); diff --git a/benchmark/benchmark_helpers.cpp b/benchmark/benchmark_helpers.cpp new file mode 100644 index 00000000..0b71d072 --- /dev/null +++ b/benchmark/benchmark_helpers.cpp @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_helpers.h" + +#include + +#include "benchmark/benchmark.h" +#include "fmt/format.h" + +namespace paimon::benchmark { + +bool BenchmarkHelpers::ValidateFileFormatOrSkip(::benchmark::State& state, + const std::string& file_format, bool is_supported, + SkipFn skip) { + if (!is_supported) { + skip(state, fmt::format("file format is not supported in this build: {}", file_format)); + return false; + } + return true; +} + +bool BenchmarkHelpers::ValidateSourcePresenceOrSkip(::benchmark::State& state, + const std::string& source_path, + const std::string& message, SkipFn skip) { + if (source_path.empty()) { + skip(state, message); + return false; + } + return true; +} + +bool BenchmarkHelpers::ValidateSourceSupportOrSkip(::benchmark::State& state, + const std::string& source_format, + bool is_supported, SkipFn skip) { + if (!is_supported) { + skip(state, + fmt::format("source data mode requires reader support in this build for format: {}", + source_format)); + return false; + } + return true; +} + +bool BenchmarkHelpers::ValidatePrefetchParallelOrSkip(::benchmark::State& state, + int32_t prefetch_parallel_num, SkipFn skip) { + if (prefetch_parallel_num <= 0) { + skip(state, "prefetch_parallel must be greater than 0"); + return false; + } + return true; +} + +Result BenchmarkHelpers::RunReadIterations(::benchmark::State& state, + const ReadOnceFn& read_once) { + int64_t rows_read = 0; + for (auto _ : state) { + PAIMON_ASSIGN_OR_RAISE(rows_read, read_once()); + } + return rows_read; +} + +Result BenchmarkHelpers::TryRunSourceTableReadMode(::benchmark::State& state, + const std::string& benchmark_name, + const std::string& source_table_path, + const ReadOnceFn& read_once) { + if (source_table_path.empty()) { + return false; + } + + std::cout << "[benchmark][" << benchmark_name << "] source_table_path=" << source_table_path + << std::endl; + PAIMON_ASSIGN_OR_RAISE(const int64_t rows_read, RunReadIterations(state, read_once)); + state.SetItemsProcessed(state.iterations() * rows_read); + return true; +} + +} // namespace paimon::benchmark diff --git a/benchmark/benchmark_helpers.h b/benchmark/benchmark_helpers.h new file mode 100644 index 00000000..f54f199d --- /dev/null +++ b/benchmark/benchmark_helpers.h @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" + +namespace benchmark { +class State; +} + +namespace paimon::benchmark { + +class BenchmarkHelpers { + public: + using ReadOnceFn = std::function()>; + using SkipFn = void (*)(::benchmark::State&, const std::string&); + + static bool ValidateFileFormatOrSkip(::benchmark::State& state, const std::string& file_format, + bool is_supported, SkipFn skip); + + static bool ValidateSourcePresenceOrSkip(::benchmark::State& state, + const std::string& source_path, + const std::string& message, SkipFn skip); + + static bool ValidateSourceSupportOrSkip(::benchmark::State& state, + const std::string& source_format, bool is_supported, + SkipFn skip); + + static bool ValidatePrefetchParallelOrSkip(::benchmark::State& state, + int32_t prefetch_parallel_num, SkipFn skip); + + static Result RunReadIterations(::benchmark::State& state, + const ReadOnceFn& read_once); + + static Result TryRunSourceTableReadMode(::benchmark::State& state, + const std::string& benchmark_name, + const std::string& source_table_path, + const ReadOnceFn& read_once); +}; + +} // namespace paimon::benchmark diff --git a/benchmark/benchmark_suite.cpp b/benchmark/benchmark_suite.cpp new file mode 100644 index 00000000..0a48a917 --- /dev/null +++ b/benchmark/benchmark_suite.cpp @@ -0,0 +1,882 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/benchmark_suite.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/io/api.h" +#include "benchmark/benchmark_helpers.h" +#include "benchmark/cli_option_parsing.h" +#include "paimon/api.h" +#include "paimon/catalog/catalog.h" +#include "paimon/testing/utils/testharness.h" + +#if __has_include("parquet/arrow/reader.h") +#include "parquet/arrow/reader.h" +#include "parquet/file_reader.h" +#define PAIMON_BENCHMARK_HAS_PARQUET_READER 1 +#else +#define PAIMON_BENCHMARK_HAS_PARQUET_READER 0 +#endif + +namespace paimon::benchmark { + +namespace { + +constexpr int64_t kSourceBatchMaxRows = 4096; +constexpr int32_t kRowToBatchThreadNumber = 3; + +struct BenchmarkCliOptions { + std::string source_data_file; + std::string source_table_path; + std::vector pk_columns; + std::vector> extra_options; +}; + +struct SourceDataSpec { + std::string format; + std::string path; +}; + +BenchmarkCliOptions& MutableBenchmarkCliOptions() { + static BenchmarkCliOptions options; + return options; +} + +const BenchmarkCliOptions& GetBenchmarkCliOptions() { + return MutableBenchmarkCliOptions(); +} + +Status ParsePaimonBenchmarkCliArgsImpl(int32_t* argc, char** argv) { + auto& options = MutableBenchmarkCliOptions(); + options = BenchmarkCliOptions{}; + const int32_t parsed_argc = *argc; + int32_t write_index = 1; + for (int32_t arg_index = 1; arg_index < parsed_argc; ++arg_index) { + const std::string arg(argv[arg_index]); + + PAIMON_ASSIGN_OR_RAISE(bool is_parsed, + paimon::benchmark::ParseStringOptionArg( + parsed_argc, argv, arg, "--paimon_source_data_file", &arg_index, + &options.source_data_file)); + if (is_parsed) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(is_parsed, paimon::benchmark::ParseStringOptionArg( + parsed_argc, argv, arg, "--paimon_source_table_path", + &arg_index, &options.source_table_path)); + if (is_parsed) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(is_parsed, paimon::benchmark::ParseCommaSeparatedOptionArg( + parsed_argc, argv, arg, "--paimon_pk_columns", + &arg_index, &options.pk_columns)); + if (is_parsed) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(is_parsed, paimon::benchmark::ParseDelimitedRepeatableOptionArg( + parsed_argc, argv, arg, "--paimon_option", &arg_index, + &options.extra_options)); + if (is_parsed) { + continue; + } + + argv[write_index++] = argv[arg_index]; + } + + *argc = write_index; + argv[write_index] = nullptr; + return Status::OK(); +} + +bool HasHelpFlagImpl(int32_t argc, char** argv) { + for (int32_t arg_index = 1; arg_index < argc; ++arg_index) { + const std::string arg(argv[arg_index]); + if (arg == "-h" || arg == "--help" || arg == "--help=true") { + return true; + } + } + return false; +} + +void PrintPaimonBenchmarkCliHelpImpl() { + std::cout + << "Paimon benchmark custom options:\n" + << " --paimon_source_data_file=\n" + << " Required. External source data file used to build benchmark data.\n" + << " Currently supports Parquet source files.\n" + << " Also supports: --paimon_source_data_file \n" + << " --paimon_source_table_path=\n" + << " Optional for BM_Read and BM_MOR_Read. If set, read directly from existing\n" + << " table path and skip source file loading and pre-write stage.\n" + << " Also supports: --paimon_source_table_path \n" + << " --paimon_pk_columns=\n" + << " Required by BM_PK_Write and BM_MOR_Read.\n" + << " Also supports: --paimon_pk_columns \n" + << " --paimon_option=:;:\n" + << " Optional and repeatable. Pass through table options as-is.\n" + << " Default table file format is parquet; use file.format: to override.\n" + << " Also supports: --paimon_option :;:\n" + << " Note: use quotes in shell, e.g. \"--paimon_option k1:v1;k2:v2\".\n" + << "\n" + << "Example:\n" + << " paimon-read-write-benchmark --paimon_source_data_file /path/data.parquet \\\n" + << " --paimon_pk_columns=id --paimon_option \"read.batch-size:8192\" \\\n" + << " --benchmark_filter=BM_Read\n" + << std::endl; +} + +Result> CreateBenchmarkWorkspace() { + auto workspace = paimon::test::UniqueTestDirectory::Create(); + if (workspace == nullptr) { + return Status::Invalid("failed to create benchmark workspace"); + } + return workspace; +} + +uint64_t NextTableId() { + static std::atomic id{0}; + return ++id; +} + +std::string RequirePath(const std::string& root_path, const std::string& db_name, + const std::string& table_name) { + return root_path + "/" + db_name + ".db/" + table_name; +} + +template +Result AddContext(paimon::Result&& result, const std::string& context) { + if (!result.ok()) { + const Status status = result.status(); + return status.WithMessage(context, ": ", status.message()); + } + return std::move(result).value(); +} + +Status AddContext(const paimon::Status& status, const std::string& context) { + if (!status.ok()) { + return status.WithMessage(context, ": ", status.message()); + } + return Status::OK(); +} + +void SkipWithMessage(::benchmark::State& state, const std::string& message) { + state.SkipWithError(message); +} + +std::string GetConfiguredFileFormat() { + std::string file_format = "parquet"; + for (const auto& kv : GetBenchmarkCliOptions().extra_options) { + if (kv.first == paimon::Options::FILE_FORMAT) { + file_format = kv.second; + } + } + return file_format; +} + +bool IsFileFormatSupported(const std::string& format) { + if (format == "parquet") { + return true; + } + if (format == "orc") { +#ifdef PAIMON_ENABLE_ORC + return true; +#else + return false; +#endif + } + return false; +} + +void ApplyExtraOptions(std::map* options) { + for (const auto& kv : GetBenchmarkCliOptions().extra_options) { + (*options)[kv.first] = kv.second; + } +} + +std::map BuildOptions(const std::string& file_format) { + std::map options = { + {paimon::Options::FILE_FORMAT, file_format}, + }; + ApplyExtraOptions(&options); + return options; +} + +std::map BuildPkOptions(const std::string& file_format) { + auto options = BuildOptions(file_format); + options[paimon::Options::BUCKET] = "1"; + options[paimon::Options::MERGE_ENGINE] = "deduplicate"; + return options; +} + +std::string GetSourceDataFilePath() { + return GetBenchmarkCliOptions().source_data_file; +} + +std::string GetSourceTablePath() { + return GetBenchmarkCliOptions().source_table_path; +} + +const std::vector& GetPkColumns() { + return GetBenchmarkCliOptions().pk_columns; +} + +SourceDataSpec GetSourceDataSpec() { + const std::string source_data_file_path = GetSourceDataFilePath(); + if (!source_data_file_path.empty()) { + return {"parquet", source_data_file_path}; + } + return {"", ""}; +} + +int64_t GetSourceBatchMaxRows() { + return kSourceBatchMaxRows; +} + +int32_t GetRowToBatchThreadNumber() { + return kRowToBatchThreadNumber; +} + +bool SupportsParquetSourceDataMode() { +#if PAIMON_BENCHMARK_HAS_PARQUET_READER + return true; +#else + return false; +#endif +} + +bool SupportsSourceDataMode(const std::string& source_format) { + if (source_format == "parquet") { + return SupportsParquetSourceDataMode(); + } + return false; +} + +struct SourceDataMetadata { + std::shared_ptr schema; + int64_t total_rows = 0; + std::string format; + std::string path; +}; + +#if PAIMON_BENCHMARK_HAS_PARQUET_READER +Result> OpenParquetSourceReader( + const std::string& path) { + auto input = arrow::io::ReadableFile::Open(path); + if (!input.ok()) { + return Status::Invalid("open Parquet source failed: ", path, ", ", + input.status().ToString()); + } + + std::unique_ptr parquet_reader; + const auto open_status = parquet::arrow::OpenFile( + input.ValueUnsafe(), arrow::default_memory_pool(), &parquet_reader); + if (!open_status.ok()) { + return Status::Invalid("create Parquet reader failed: ", open_status.ToString()); + } + parquet_reader->set_batch_size(GetSourceBatchMaxRows()); + return parquet_reader; +} +#endif + +Result LoadParquetSourceMetadata(const std::string& path) { +#if !PAIMON_BENCHMARK_HAS_PARQUET_READER + return Status::Invalid( + "Parquet source data mode requires parquet::arrow reader support in this build"); +#else + static SourceDataMetadata cache; + static std::mutex cache_mutex; + std::lock_guard lock(cache_mutex); + if (cache.path == path && cache.format == "parquet") { + return cache; + } + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr parquet_reader, + OpenParquetSourceReader(path)); + std::shared_ptr schema; + const auto schema_status = parquet_reader->GetSchema(&schema); + if (!schema_status.ok()) { + return Status::Invalid("read Parquet source schema failed: ", schema_status.ToString()); + } + + const int64_t total_rows = parquet_reader->parquet_reader()->metadata()->num_rows(); + if (total_rows <= 0) { + return Status::Invalid("Parquet source is empty: ", path); + } + + cache.schema = std::move(schema); + cache.total_rows = total_rows; + cache.format = "parquet"; + cache.path = path; + return cache; +#endif +} + +Result LoadSourceDataMetadata(const SourceDataSpec& source_spec) { + if (source_spec.format == "parquet") { + return LoadParquetSourceMetadata(source_spec.path); + } + return Status::Invalid("unknown source format: ", source_spec.format); +} + +std::shared_ptr BuildStructArrayFromRecordBatch( + const std::shared_ptr& batch) { + return std::make_shared(arrow::struct_(batch->schema()->fields()), + batch->num_rows(), batch->columns()); +} + +Result> MakeRecordBatch( + const std::shared_ptr& arr) { + ArrowArray c_array; + if (!arrow::ExportArray(*arr, &c_array).ok()) { + return Status::Invalid("failed to export arrow array"); + } + paimon::RecordBatchBuilder builder(&c_array); + return AddContext(builder.Finish(), "build paimon record batch"); +} + +Status EnsureTable(const std::string& root_path, const std::string& db_name, + const std::string& table_name, const std::map& options, + const std::shared_ptr& schema, + const std::vector& primary_keys = {}) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr catalog, + AddContext(paimon::Catalog::Create(root_path, options), "create catalog")); + PAIMON_RETURN_NOT_OK( + AddContext(catalog->CreateDatabase(db_name, options, true), "create database")); + + ArrowSchema c_schema; + if (!arrow::ExportSchema(*schema, &c_schema).ok()) { + return Status::Invalid("failed to export table schema"); + } + PAIMON_RETURN_NOT_OK( + AddContext(catalog->CreateTable(paimon::Identifier(db_name, table_name), &c_schema, + /*partition_keys=*/{}, primary_keys, options, + /*ignore_if_exists=*/false), + "create table")); + return Status::OK(); +} + +Status WriteSourceDataToWriter(paimon::FileStoreWrite* writer, const SourceDataSpec& source_spec) { + if (source_spec.format != "parquet") { + return Status::Invalid("unknown source format: ", source_spec.format); + } + +#if !PAIMON_BENCHMARK_HAS_PARQUET_READER + return Status::Invalid( + "Parquet source data mode requires parquet::arrow reader support in this build"); +#else + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr parquet_reader, + OpenParquetSourceReader(source_spec.path)); + std::unique_ptr batch_reader; + const auto reader_status = parquet_reader->GetRecordBatchReader(&batch_reader); + if (!reader_status.ok()) { + return Status::Invalid("create Parquet source batch reader failed: ", + reader_status.ToString()); + } + + int64_t written_rows = 0; + while (true) { + std::shared_ptr record_batch; + const auto read_status = batch_reader->ReadNext(&record_batch); + if (!read_status.ok()) { + return Status::Invalid("read Parquet source batch failed: ", read_status.ToString()); + } + if (record_batch == nullptr) { + break; + } + if (record_batch->num_rows() <= 0) { + continue; + } + + auto struct_array = BuildStructArrayFromRecordBatch(record_batch); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, + MakeRecordBatch(struct_array)); + PAIMON_RETURN_NOT_OK(AddContext(writer->Write(std::move(batch)), "write batch")); + written_rows += record_batch->num_rows(); + } + + if (written_rows <= 0) { + return Status::Invalid("source file has no non-empty data batches: ", source_spec.path); + } + return Status::OK(); +#endif +} + +Status WriteAndCommit(const std::string& table_path, + const std::map& options, + const SourceDataSpec& source_spec) { + paimon::WriteContextBuilder write_builder(table_path, "benchmark-writer"); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr write_ctx, + AddContext(write_builder.SetOptions(options).Finish(), "create write context")); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + AddContext(paimon::FileStoreWrite::Create(std::move(write_ctx)), + "create file store writer")); + + PAIMON_RETURN_NOT_OK(WriteSourceDataToWriter(writer.get(), source_spec)); + PAIMON_ASSIGN_OR_RAISE(std::vector> messages, + AddContext(writer->PrepareCommit(), "prepare commit")); + + paimon::CommitContextBuilder commit_builder(table_path, "benchmark-writer"); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr commit_ctx, + AddContext(commit_builder.SetOptions(options).Finish(), "create commit context")); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr committer, + AddContext(paimon::FileStoreCommit::Create(std::move(commit_ctx)), "create committer")); + PAIMON_RETURN_NOT_OK(AddContext(committer->Commit(messages), "commit write")); + return Status::OK(); +} + +struct SharedReadTableCache { + std::string key; + std::unique_ptr workspace; + std::string table_path; + int64_t total_rows = 0; +}; + +struct SharedMorReadTableCache { + std::string key; + std::unique_ptr workspace; + std::string table_path; + int64_t total_rows = 0; +}; + +std::string BuildReadTableCacheKey(const std::string& file_format, + const SourceDataSpec& source_spec) { + return file_format + "|" + source_spec.format + "|" + source_spec.path + "|" + + std::to_string(GetSourceBatchMaxRows()); +} + +std::string JoinColumns(const std::vector& columns) { + std::string joined; + for (size_t i = 0; i < columns.size(); ++i) { + if (i > 0) { + joined.append(","); + } + joined.append(columns[i]); + } + return joined; +} + +Result GetOrCreateSharedMorReadTable( + const std::string& file_format, const SourceDataSpec& source_spec) { + static SharedMorReadTableCache cache; + static std::mutex cache_mutex; + + const std::vector& pk_columns = GetPkColumns(); + const std::string cache_key = + BuildReadTableCacheKey(file_format, source_spec) + "|pk=" + JoinColumns(pk_columns); + std::lock_guard lock(cache_mutex); + if (cache.workspace != nullptr && cache.key == cache_key) { + std::cout << "[benchmark][mor-read] reuse_output_table_path=" << cache.table_path + << std::endl; + return &cache; + } + + auto options = BuildPkOptions(file_format); + PAIMON_ASSIGN_OR_RAISE(const SourceDataMetadata source_metadata, + LoadSourceDataMetadata(source_spec)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr workspace, + CreateBenchmarkWorkspace()); + const std::string db_name = "bench_db"; + const std::string table_name = "mor_read_shared_" + std::to_string(NextTableId()); + PAIMON_RETURN_NOT_OK(EnsureTable(workspace->Str(), db_name, table_name, options, + source_metadata.schema, + /*primary_keys=*/pk_columns)); + const std::string table_path = RequirePath(workspace->Str(), db_name, table_name); + std::cout << "[benchmark][mor-read] create_shared_output_table_path=" << table_path + << std::endl; + PAIMON_RETURN_NOT_OK(WriteAndCommit(table_path, options, source_spec)); + + cache.key = cache_key; + cache.workspace = std::move(workspace); + cache.table_path = table_path; + cache.total_rows = source_metadata.total_rows; + return &cache; +} + +Result GetOrCreateSharedReadTable(const std::string& file_format, + const SourceDataSpec& source_spec) { + static SharedReadTableCache cache; + static std::mutex cache_mutex; + + const std::string cache_key = BuildReadTableCacheKey(file_format, source_spec); + std::lock_guard lock(cache_mutex); + if (cache.workspace != nullptr && cache.key == cache_key) { + std::cout << "[benchmark][read] reuse_output_table_path=" << cache.table_path << std::endl; + return &cache; + } + + auto options = BuildOptions(file_format); + PAIMON_ASSIGN_OR_RAISE(const SourceDataMetadata source_metadata, + LoadSourceDataMetadata(source_spec)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr workspace, + CreateBenchmarkWorkspace()); + const std::string db_name = "bench_db"; + const std::string table_name = "read_shared_" + std::to_string(NextTableId()); + PAIMON_RETURN_NOT_OK( + EnsureTable(workspace->Str(), db_name, table_name, options, source_metadata.schema)); + const std::string table_path = RequirePath(workspace->Str(), db_name, table_name); + std::cout << "[benchmark][read] create_shared_output_table_path=" << table_path << std::endl; + PAIMON_RETURN_NOT_OK(WriteAndCommit(table_path, options, source_spec)); + + cache.key = cache_key; + cache.workspace = std::move(workspace); + cache.table_path = table_path; + cache.total_rows = source_metadata.total_rows; + return &cache; +} + +Result ReadRows(const std::string& table_path, + const std::map& options, + int32_t prefetch_parallel_num) { + paimon::ScanContextBuilder scan_builder(table_path); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_ctx, + AddContext(scan_builder.SetOptions(options).Finish(), "create scan context")); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scanner, + AddContext(paimon::TableScan::Create(std::move(scan_ctx)), "create scanner")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + AddContext(scanner->CreatePlan(), "create plan")); + + paimon::ReadContextBuilder read_builder(table_path); + constexpr int32_t kPrefetchBatchCount = 600; + read_builder.SetOptions(options) + .EnablePrefetch(true) + .SetPrefetchBatchCount(kPrefetchBatchCount) + .SetPrefetchMaxParallelNum(prefetch_parallel_num) + .EnableMultiThreadRowToBatch(GetRowToBatchThreadNumber() > 1) + .SetRowToBatchThreadNumber(GetRowToBatchThreadNumber()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_ctx, + AddContext(read_builder.Finish(), "create read context")); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr reader, + AddContext(paimon::TableRead::Create(std::move(read_ctx)), "create table reader")); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + AddContext(reader->CreateReader(plan->Splits()), "create batch reader")); + + int64_t total_rows = 0; + while (true) { + PAIMON_ASSIGN_OR_RAISE(paimon::BatchReader::ReadBatch batch, + AddContext(batch_reader->NextBatch(), "read next batch")); + if (paimon::BatchReader::IsEofBatch(batch)) { + break; + } + auto& [array, schema] = batch; + auto imported = arrow::ImportArray(array.get(), schema.get()); + if (!imported.ok()) { + return Status::Invalid("import c data array failed: ", imported.status().ToString()); + } + total_rows += imported.ValueUnsafe()->length(); + } + + return total_rows; +} + +struct PreparedSourceData { + std::shared_ptr schema; + int64_t total_rows = 0; +}; + +bool TryGetSourceSpec(::benchmark::State& state, SourceDataSpec* source_spec) { + (void)state; + *source_spec = GetSourceDataSpec(); + return true; +} + +bool TryPrepareSourceData(::benchmark::State& state, const SourceDataSpec& source_spec, + PreparedSourceData* prepared) { + auto source_metadata = LoadSourceDataMetadata(source_spec); + if (!source_metadata.ok()) { + SkipWithMessage(state, source_metadata.status().ToString()); + return false; + } + prepared->schema = source_metadata.value().schema; + prepared->total_rows = source_metadata.value().total_rows; + return true; +} + +} // namespace + +Status ParsePaimonBenchmarkCliArgs(int* argc, char** argv) { + auto parsed_argc = static_cast(*argc); + PAIMON_RETURN_NOT_OK(ParsePaimonBenchmarkCliArgsImpl(&parsed_argc, argv)); + *argc = static_cast(parsed_argc); + return Status::OK(); +} + +bool HasHelpFlag(int32_t argc, char** argv) { + return HasHelpFlagImpl(argc, argv); +} + +void PrintPaimonBenchmarkCliHelp() { + PrintPaimonBenchmarkCliHelpImpl(); +} + +void RunBMWrite(::benchmark::State& state) { + const std::string file_format = GetConfiguredFileFormat(); + SourceDataSpec source_spec; + if (!TryGetSourceSpec(state, &source_spec)) { + return; + } + if (!BenchmarkHelpers::ValidateSourcePresenceOrSkip( + state, source_spec.path, "--paimon_source_data_file is required", &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateSourceSupportOrSkip(state, source_spec.format, + SupportsSourceDataMode(source_spec.format), + &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateFileFormatOrSkip( + state, file_format, IsFileFormatSupported(file_format), &SkipWithMessage)) { + return; + } + + auto options = BuildOptions(file_format); + PreparedSourceData prepared; + if (!TryPrepareSourceData(state, source_spec, &prepared)) { + return; + } + auto workspace = CreateBenchmarkWorkspace(); + if (!workspace.ok()) { + SkipWithMessage(state, workspace.status().ToString()); + return; + } + + for (auto _ : state) { + const std::string db_name = "bench_db"; + const std::string table_name = "write_" + std::to_string(NextTableId()); + const Status ensure_status = + EnsureTable(workspace.value()->Str(), db_name, table_name, options, prepared.schema); + if (!ensure_status.ok()) { + SkipWithMessage(state, ensure_status.ToString()); + return; + } + const std::string table_path = RequirePath(workspace.value()->Str(), db_name, table_name); + std::cout << "[benchmark][write] output_table_path=" << table_path << std::endl; + const Status write_status = WriteAndCommit(table_path, options, source_spec); + if (!write_status.ok()) { + SkipWithMessage(state, write_status.ToString()); + return; + } + } + + state.SetItemsProcessed(state.iterations() * prepared.total_rows); +} + +void RunBMRead(::benchmark::State& state) { + const auto prefetch_parallel_num = static_cast(state.range(0)); + const std::string file_format = GetConfiguredFileFormat(); + const std::string source_table_path = GetSourceTablePath(); + SourceDataSpec source_spec; + if (!TryGetSourceSpec(state, &source_spec)) { + return; + } + if (!BenchmarkHelpers::ValidateFileFormatOrSkip( + state, file_format, IsFileFormatSupported(file_format), &SkipWithMessage)) { + return; + } + + if (!BenchmarkHelpers::ValidatePrefetchParallelOrSkip(state, prefetch_parallel_num, + &SkipWithMessage)) { + return; + } + + auto options = BuildOptions(file_format); + + auto source_table_read_result = BenchmarkHelpers::TryRunSourceTableReadMode( + state, "read", source_table_path, + [&]() { return ReadRows(source_table_path, options, prefetch_parallel_num); }); + if (!source_table_read_result.ok()) { + SkipWithMessage(state, source_table_read_result.status().ToString()); + return; + } + if (source_table_read_result.value()) { + return; + } + + if (!BenchmarkHelpers::ValidateSourcePresenceOrSkip( + state, source_spec.path, + "--paimon_source_data_file is required when --paimon_source_table_path is not set", + &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateSourceSupportOrSkip(state, source_spec.format, + SupportsSourceDataMode(source_spec.format), + &SkipWithMessage)) { + return; + } + + auto shared_table = GetOrCreateSharedReadTable(file_format, source_spec); + if (!shared_table.ok()) { + SkipWithMessage(state, shared_table.status().ToString()); + return; + } + + auto rows_read = BenchmarkHelpers::RunReadIterations(state, [&]() { + return ReadRows(shared_table.value()->table_path, options, prefetch_parallel_num); + }); + if (!rows_read.ok()) { + SkipWithMessage(state, rows_read.status().ToString()); + return; + } + + state.SetItemsProcessed(state.iterations() * rows_read.value()); +} + +void RunBMPkWrite(::benchmark::State& state) { + const std::string file_format = GetConfiguredFileFormat(); + SourceDataSpec source_spec; + if (!TryGetSourceSpec(state, &source_spec)) { + return; + } + if (!BenchmarkHelpers::ValidateSourcePresenceOrSkip( + state, source_spec.path, "--paimon_source_data_file is required", &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateSourceSupportOrSkip(state, source_spec.format, + SupportsSourceDataMode(source_spec.format), + &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateFileFormatOrSkip( + state, file_format, IsFileFormatSupported(file_format), &SkipWithMessage)) { + return; + } + const std::vector& pk_columns = GetPkColumns(); + if (pk_columns.empty()) { + SkipWithMessage(state, "--paimon_pk_columns is required for BM_PK_Write"); + return; + } + + auto options = BuildPkOptions(file_format); + PreparedSourceData prepared; + if (!TryPrepareSourceData(state, source_spec, &prepared)) { + return; + } + auto workspace = CreateBenchmarkWorkspace(); + if (!workspace.ok()) { + SkipWithMessage(state, workspace.status().ToString()); + return; + } + + for (auto _ : state) { + const std::string db_name = "bench_db"; + const std::string table_name = "pk_write_" + std::to_string(NextTableId()); + const Status ensure_status = + EnsureTable(workspace.value()->Str(), db_name, table_name, options, prepared.schema, + /*primary_keys=*/pk_columns); + if (!ensure_status.ok()) { + SkipWithMessage(state, ensure_status.ToString()); + return; + } + const std::string table_path = RequirePath(workspace.value()->Str(), db_name, table_name); + std::cout << "[benchmark][pk-write] output_table_path=" << table_path << std::endl; + const Status write_status = WriteAndCommit(table_path, options, source_spec); + if (!write_status.ok()) { + SkipWithMessage(state, write_status.ToString()); + return; + } + } + + state.SetItemsProcessed(state.iterations() * prepared.total_rows); +} + +void RunBMMorRead(::benchmark::State& state) { + const auto prefetch_parallel_num = static_cast(state.range(0)); + const std::string file_format = GetConfiguredFileFormat(); + const std::string source_table_path = GetSourceTablePath(); + SourceDataSpec source_spec; + if (!TryGetSourceSpec(state, &source_spec)) { + return; + } + if (!BenchmarkHelpers::ValidateFileFormatOrSkip( + state, file_format, IsFileFormatSupported(file_format), &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidatePrefetchParallelOrSkip(state, prefetch_parallel_num, + &SkipWithMessage)) { + return; + } + + const auto source_table_read_options = BuildOptions(file_format); + auto source_table_read_result = + BenchmarkHelpers::TryRunSourceTableReadMode(state, "mor-read", source_table_path, [&]() { + return ReadRows(source_table_path, source_table_read_options, prefetch_parallel_num); + }); + if (!source_table_read_result.ok()) { + SkipWithMessage(state, source_table_read_result.status().ToString()); + return; + } + if (source_table_read_result.value()) { + return; + } + + if (!BenchmarkHelpers::ValidateSourcePresenceOrSkip( + state, source_spec.path, + "--paimon_source_data_file is required when --paimon_source_table_path is not set", + &SkipWithMessage)) { + return; + } + if (!BenchmarkHelpers::ValidateSourceSupportOrSkip(state, source_spec.format, + SupportsSourceDataMode(source_spec.format), + &SkipWithMessage)) { + return; + } + if (GetPkColumns().empty()) { + SkipWithMessage(state, "--paimon_pk_columns is required for BM_MOR_Read"); + return; + } + + auto options = BuildPkOptions(file_format); + auto shared_table = GetOrCreateSharedMorReadTable(file_format, source_spec); + if (!shared_table.ok()) { + SkipWithMessage(state, shared_table.status().ToString()); + return; + } + + auto rows_read = BenchmarkHelpers::RunReadIterations(state, [&]() { + return ReadRows(shared_table.value()->table_path, options, prefetch_parallel_num); + }); + if (!rows_read.ok()) { + SkipWithMessage(state, rows_read.status().ToString()); + return; + } + state.SetItemsProcessed(state.iterations() * rows_read.value()); +} + +} // namespace paimon::benchmark diff --git a/benchmark/benchmark_suite.h b/benchmark/benchmark_suite.h new file mode 100644 index 00000000..69814cea --- /dev/null +++ b/benchmark/benchmark_suite.h @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "benchmark/benchmark.h" +#include "paimon/status.h" + +namespace paimon::benchmark { + +Status ParsePaimonBenchmarkCliArgs(int* argc, char** argv); +bool HasHelpFlag(int32_t argc, char** argv); +void PrintPaimonBenchmarkCliHelp(); + +void RunBMWrite(::benchmark::State& state); +void RunBMRead(::benchmark::State& state); +void RunBMPkWrite(::benchmark::State& state); +void RunBMMorRead(::benchmark::State& state); + +} // namespace paimon::benchmark diff --git a/benchmark/cli_option_parsing.h b/benchmark/cli_option_parsing.h new file mode 100644 index 00000000..70aea5c6 --- /dev/null +++ b/benchmark/cli_option_parsing.h @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon::benchmark { + +using ParsedOptions = std::vector>; + +inline bool ConsumeCliOption(const std::string& arg, const std::string& option_name, + std::string* value_out) { + const std::string prefix = option_name + "="; + if (arg.rfind(prefix, 0) != 0) { + return false; + } + *value_out = arg.substr(prefix.size()); + return true; +} + +inline std::string TrimAsciiWhitespace(const std::string& value) { + const auto first = value.find_first_not_of(" \t\n\r"); + if (first == std::string::npos) { + return ""; + } + const auto last = value.find_last_not_of(" \t\n\r"); + return value.substr(first, last - first + 1); +} + +inline Result> ParseCommaSeparatedColumns(const std::string& input, + const std::string& option_name) { + if (input.empty()) { + return Status::Invalid("missing value for ", option_name); + } + + std::vector columns; + size_t segment_start = 0; + for (size_t index = 0; index <= input.size(); ++index) { + if (index != input.size() && input[index] != ',') { + continue; + } + + const std::string column = + TrimAsciiWhitespace(input.substr(segment_start, index - segment_start)); + if (column.empty()) { + return Status::Invalid("invalid ", option_name, ": empty column name"); + } + columns.push_back(column); + segment_start = index + 1; + } + return columns; +} + +inline Result ParseDelimitedOptions(const std::string& input, + const std::string& option_name) { + if (input.empty()) { + return Status::Invalid("missing value for ", option_name); + } + + ParsedOptions parsed; + std::string token; + for (size_t index = 0; index <= input.size(); ++index) { + const bool at_end = (index == input.size()); + if (!at_end && input[index] != ';') { + token.push_back(input[index]); + continue; + } + + const std::string segment = TrimAsciiWhitespace(token); + if (segment.empty()) { + return Status::Invalid("invalid ", option_name, ": empty option segment"); + } + + const auto separator = segment.find(':'); + if (separator == std::string::npos) { + return Status::Invalid("invalid ", option_name, ": expected key:value"); + } + + const std::string key = TrimAsciiWhitespace(segment.substr(0, separator)); + const std::string value = TrimAsciiWhitespace(segment.substr(separator + 1)); + if (key.empty() || value.empty()) { + return Status::Invalid("invalid ", option_name, ": expected key:value"); + } + + parsed.emplace_back(key, value); + token.clear(); + } + return parsed; +} + +inline Result ParseStringOptionArg(int32_t argc, char** argv, const std::string& arg, + const std::string& option_name, int32_t* arg_index, + std::string* value_out) { + std::string parsed_value; + if (ConsumeCliOption(arg, option_name, &parsed_value)) { + *value_out = std::move(parsed_value); + return true; + } + + if (arg != option_name) { + return false; + } + + if (*arg_index + 1 >= argc) { + return Status::Invalid("missing value for ", option_name); + } + *value_out = argv[++(*arg_index)]; + return true; +} + +inline Result ParseCommaSeparatedOptionArg(int32_t argc, char** argv, const std::string& arg, + const std::string& option_name, int32_t* arg_index, + std::vector* columns_out) { + std::string parsed_value; + if (ConsumeCliOption(arg, option_name, &parsed_value)) { + PAIMON_ASSIGN_OR_RAISE(*columns_out, ParseCommaSeparatedColumns(parsed_value, option_name)); + return true; + } + + if (arg != option_name) { + return false; + } + + if (*arg_index + 1 >= argc) { + return Status::Invalid("missing value for ", option_name); + } + PAIMON_ASSIGN_OR_RAISE( + *columns_out, ParseCommaSeparatedColumns(std::string(argv[++(*arg_index)]), option_name)); + return true; +} + +inline Result ParseDelimitedRepeatableOptionArg( + int32_t argc, char** argv, const std::string& arg, const std::string& option_name, + int32_t* arg_index, std::vector>* options_out) { + std::string parsed_value; + if (ConsumeCliOption(arg, option_name, &parsed_value)) { + ParsedOptions parsed_options; + PAIMON_ASSIGN_OR_RAISE(parsed_options, ParseDelimitedOptions(parsed_value, option_name)); + options_out->insert(options_out->end(), parsed_options.begin(), parsed_options.end()); + return true; + } + + if (arg != option_name) { + return false; + } + + if (*arg_index + 1 >= argc) { + return Status::Invalid("missing value for ", option_name); + } + + const std::string option_arg = argv[++(*arg_index)]; + ParsedOptions parsed_options; + PAIMON_ASSIGN_OR_RAISE(parsed_options, ParseDelimitedOptions(option_arg, option_name)); + options_out->insert(options_out->end(), parsed_options.begin(), parsed_options.end()); + return true; +} + +} // namespace paimon::benchmark diff --git a/benchmark/cli_option_parsing_test.cpp b/benchmark/cli_option_parsing_test.cpp new file mode 100644 index 00000000..5d0c56ed --- /dev/null +++ b/benchmark/cli_option_parsing_test.cpp @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "benchmark/cli_option_parsing.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::testing { +namespace { + +struct ArgvHolder { + std::vector args; + std::vector argv; + + explicit ArgvHolder(std::vector in_args) : args(std::move(in_args)) { + argv.reserve(args.size()); + for (auto& arg : args) { + argv.push_back(arg.data()); + } + } + + int32_t argc() const { + return static_cast(argv.size()); + } +}; + +TEST(CliOptionParsingTest, ConsumeCliOptionWorks) { + std::string value; + ASSERT_TRUE(paimon::benchmark::ConsumeCliOption("--foo=bar", "--foo", &value)); + ASSERT_EQ(value, "bar"); + + value.clear(); + ASSERT_FALSE(paimon::benchmark::ConsumeCliOption("--foo", "--foo", &value)); +} + +TEST(CliOptionParsingTest, ParseCommaSeparatedColumnsWorks) { + ASSERT_OK_AND_ASSIGN(auto parsed, + paimon::benchmark::ParseCommaSeparatedColumns("id, name,age", "--cols")); + ASSERT_EQ(parsed.size(), 3U); + ASSERT_EQ(parsed[0], "id"); + ASSERT_EQ(parsed[1], "name"); + ASSERT_EQ(parsed[2], "age"); +} + +TEST(CliOptionParsingTest, ParseCommaSeparatedColumnsRejectsInvalidInput) { + ASSERT_NOK(paimon::benchmark::ParseCommaSeparatedColumns("", "--cols")); + ASSERT_NOK(paimon::benchmark::ParseCommaSeparatedColumns("id,", "--cols")); + ASSERT_NOK(paimon::benchmark::ParseCommaSeparatedColumns("id,,name", "--cols")); +} + +TEST(CliOptionParsingTest, ParseDelimitedOptionsWorks) { + ASSERT_OK_AND_ASSIGN( + auto parsed, paimon::benchmark::ParseDelimitedOptions("k1:v1;k2:v2", "--paimon_option")); + ASSERT_EQ(parsed.size(), 2U); + ASSERT_EQ(parsed[0], std::make_pair(std::string("k1"), std::string("v1"))); + ASSERT_EQ(parsed[1], std::make_pair(std::string("k2"), std::string("v2"))); +} + +TEST(CliOptionParsingTest, ParseDelimitedOptionsTrimsKeyAndValue) { + ASSERT_OK_AND_ASSIGN(auto parsed, paimon::benchmark::ParseDelimitedOptions(" k1 : v1 ; k2: v2 ", + "--paimon_option")); + ASSERT_EQ(parsed.size(), 2U); + ASSERT_EQ(parsed[0], std::make_pair(std::string("k1"), std::string("v1"))); + ASSERT_EQ(parsed[1], std::make_pair(std::string("k2"), std::string("v2"))); +} + +TEST(CliOptionParsingTest, ParseDelimitedOptionsRejectsInvalidInput) { + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions("", "--paimon_option")); + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions("k1:v1;", "--paimon_option")); + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions("k1:", "--paimon_option")); + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions(":v1", "--paimon_option")); + ASSERT_NOK(paimon::benchmark::ParseDelimitedOptions("k1: ", "--paimon_option")); +} + +TEST(CliOptionParsingTest, ParseStringOptionArgWorksForEqualsAndSeparatedForms) { + { + ArgvHolder argv_holder({"prog", "--foo=bar"}); + int32_t arg_index = 1; + std::string value; + ASSERT_OK_AND_ASSIGN(bool is_parsed, + paimon::benchmark::ParseStringOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--foo", &arg_index, &value)); + ASSERT_TRUE(is_parsed); + ASSERT_EQ(arg_index, 1); + ASSERT_EQ(value, "bar"); + } + + { + ArgvHolder argv_holder({"prog", "--foo", "bar"}); + int32_t arg_index = 1; + std::string value; + ASSERT_OK_AND_ASSIGN(bool is_parsed, + paimon::benchmark::ParseStringOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--foo", &arg_index, &value)); + ASSERT_TRUE(is_parsed); + ASSERT_EQ(arg_index, 2); + ASSERT_EQ(value, "bar"); + } +} + +TEST(CliOptionParsingTest, ParseStringOptionArgRejectsMissingValue) { + ArgvHolder argv_holder({"prog", "--foo"}); + int32_t arg_index = 1; + std::string value; + ASSERT_NOK(paimon::benchmark::ParseStringOptionArg(argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--foo", + &arg_index, &value)); +} + +TEST(CliOptionParsingTest, ParseStringOptionArgIgnoresOtherOptions) { + ArgvHolder argv_holder({"prog", "--bar=baz"}); + int32_t arg_index = 1; + std::string value; + ASSERT_OK_AND_ASSIGN(bool is_parsed, + paimon::benchmark::ParseStringOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--foo", &arg_index, &value)); + ASSERT_FALSE(is_parsed); + ASSERT_EQ(arg_index, 1); + ASSERT_TRUE(value.empty()); +} + +TEST(CliOptionParsingTest, ParseCommaSeparatedOptionArgAndDelimitedRepeatableOptionArgWorks) { + { + ArgvHolder argv_holder({"prog", "--cols", "id,name"}); + int32_t arg_index = 1; + std::vector columns; + ASSERT_OK_AND_ASSIGN(bool is_parsed, + paimon::benchmark::ParseCommaSeparatedOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--cols", &arg_index, &columns)); + ASSERT_TRUE(is_parsed); + ASSERT_EQ(arg_index, 2); + ASSERT_EQ(columns.size(), 2U); + ASSERT_EQ(columns[0], "id"); + ASSERT_EQ(columns[1], "name"); + } + + { + ArgvHolder argv_holder({"prog", "--paimon_option", "k1:v1;k2:v2"}); + int32_t arg_index = 1; + std::vector> options; + ASSERT_OK_AND_ASSIGN(bool is_parsed, paimon::benchmark::ParseDelimitedRepeatableOptionArg( + argv_holder.argc(), argv_holder.argv.data(), + argv_holder.args[arg_index], "--paimon_option", + &arg_index, &options)); + ASSERT_TRUE(is_parsed); + ASSERT_EQ(arg_index, 2); + ASSERT_EQ(options.size(), 2U); + ASSERT_EQ(options[0], std::make_pair(std::string("k1"), std::string("v1"))); + ASSERT_EQ(options[1], std::make_pair(std::string("k2"), std::string("v2"))); + } +} + +} // namespace +} // namespace paimon::testing diff --git a/benchmark/read_write_benchmark.cpp b/benchmark/read_write_benchmark.cpp new file mode 100644 index 00000000..5cd7ffb5 --- /dev/null +++ b/benchmark/read_write_benchmark.cpp @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "benchmark/benchmark.h" +#include "benchmark/benchmark_suite.h" + +int main(int argc, char** argv) { + if (paimon::benchmark::HasHelpFlag(static_cast(argc), argv)) { + paimon::benchmark::PrintPaimonBenchmarkCliHelp(); + return 0; + } + + const paimon::Status parse_status = paimon::benchmark::ParsePaimonBenchmarkCliArgs(&argc, argv); + if (!parse_status.ok()) { + std::cerr << "paimon-read-write-benchmark: " << parse_status.ToString() << std::endl; + std::cerr << "Try 'paimon-read-write-benchmark --help' for more information." << std::endl; + return 1; + } + + benchmark::Initialize(&argc, argv); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index cbc35748..1bc99e70 100644 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -407,3 +407,121 @@ function(add_paimon_test REL_TEST_NAME) ${PCH_ARGS} ${ARG_UNPARSED_ARGUMENTS}) endfunction() + +function(add_benchmark_case REL_BENCHMARK_NAME) + set(options ENABLED) + set(one_value_args) + set(multi_value_args + SOURCES + STATIC_LINK_LIBS + EXTRA_LINK_LIBS + EXTRA_INCLUDES + LABELS + PREFIX) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + if(NOT PAIMON_BUILD_BENCHMARKS AND NOT ARG_ENABLED) + return() + endif() + + get_filename_component(BENCHMARK_NAME ${REL_BENCHMARK_NAME} NAME_WE) + + if(ARG_PREFIX) + set(BENCHMARK_NAME "${ARG_PREFIX}-${BENCHMARK_NAME}") + endif() + + if(ARG_SOURCES) + set(SOURCES ${ARG_SOURCES}) + else() + set(SOURCES "${REL_BENCHMARK_NAME}.cpp") + endif() + + string(REPLACE "_" "-" BENCHMARK_NAME ${BENCHMARK_NAME}) + set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}") + message(STATUS ${BENCHMARK_NAME}) + add_executable(${BENCHMARK_NAME} ${SOURCES}) + + if(ARG_STATIC_LINK_LIBS) + target_link_libraries(${BENCHMARK_NAME} PRIVATE ${ARG_STATIC_LINK_LIBS}) + endif() + + if(ARG_EXTRA_LINK_LIBS) + target_link_libraries(${BENCHMARK_NAME} PRIVATE ${ARG_EXTRA_LINK_LIBS}) + endif() + + if(ARG_EXTRA_INCLUDES) + target_include_directories(${BENCHMARK_NAME} SYSTEM PUBLIC ${ARG_EXTRA_INCLUDES}) + endif() + + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + target_compile_options(${BENCHMARK_NAME} PRIVATE -Wno-global-constructors) + endif() + target_compile_options(${BENCHMARK_NAME} PRIVATE -fno-access-control) + + add_test(${BENCHMARK_NAME} + ${BUILD_SUPPORT_DIR}/run-test.sh + ${CMAKE_BINARY_DIR} + benchmark + ${BENCHMARK_PATH}) + + foreach(TARGET ${ARG_LABELS}) + add_dependencies(${TARGET} ${BENCHMARK_NAME}) + endforeach() + + set(LABELS) + list(APPEND LABELS "benchmark") + if(ARG_LABELS) + list(APPEND LABELS ${ARG_LABELS}) + endif() + + foreach(LABEL ${ARG_LABELS}) + set(LABEL_BENCHMARK_NAME "benchmark-${LABEL}") + if(NOT TARGET ${LABEL_BENCHMARK_NAME}) + add_custom_target(${LABEL_BENCHMARK_NAME} + ctest -L "${LABEL}" --output-on-failure + USES_TERMINAL) + endif() + add_dependencies(${LABEL_BENCHMARK_NAME} ${BENCHMARK_NAME}) + endforeach() + + set_property(TEST ${BENCHMARK_NAME} + APPEND + PROPERTY LABELS ${LABELS}) +endfunction() + +function(add_paimon_benchmark REL_BENCHMARK_NAME) + set(options) + set(one_value_args PREFIX) + set(multi_value_args LABELS) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + + if(ARG_PREFIX) + set(PREFIX ${ARG_PREFIX}) + else() + set(PREFIX "paimon") + endif() + + if(ARG_LABELS) + set(LABELS ${ARG_LABELS}) + else() + set(LABELS "paimon-benchmarks") + endif() + + add_benchmark_case(${REL_BENCHMARK_NAME} + PREFIX + ${PREFIX} + LABELS + ${LABELS} + ${ARG_UNPARSED_ARGUMENTS}) +endfunction() diff --git a/cmake_modules/DefineOptions.cmake b/cmake_modules/DefineOptions.cmake index 6b266bec..291ce1db 100644 --- a/cmake_modules/DefineOptions.cmake +++ b/cmake_modules/DefineOptions.cmake @@ -107,6 +107,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(PAIMON_BUILD_TESTS "Build the Paimon googletest unit tests" OFF) + define_option(PAIMON_BUILD_BENCHMARKS + "Build the Paimon Google Benchmark performance benchmarks" OFF) + if(PAIMON_BUILD_SHARED) set(PAIMON_TEST_LINKAGE_DEFAULT "shared") else() @@ -241,6 +244,13 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") AUTO BUNDLED SYSTEM) + + define_option_string(benchmark_SOURCE + "Dependency source for Google Benchmark" + "" + AUTO + BUNDLED + SYSTEM) endif() macro(validate_config) diff --git a/cmake_modules/FindbenchmarkAlt.cmake b/cmake_modules/FindbenchmarkAlt.cmake new file mode 100644 index 00000000..f3eeadf2 --- /dev/null +++ b/cmake_modules/FindbenchmarkAlt.cmake @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(_PAIMON_BENCHMARK_ROOTS ${benchmark_ROOT} ${BENCHMARK_ROOT} ${PAIMON_PACKAGE_PREFIX}) +list(REMOVE_ITEM _PAIMON_BENCHMARK_ROOTS "") +if(_PAIMON_BENCHMARK_ROOTS) + set(_PAIMON_BENCHMARK_FIND_ARGS HINTS ${_PAIMON_BENCHMARK_ROOTS} NO_DEFAULT_PATH) +endif() + +find_package(benchmark CONFIG QUIET ${_PAIMON_BENCHMARK_FIND_ARGS}) + +if(NOT TARGET benchmark::benchmark) + find_path(BENCHMARK_INCLUDE_DIR + NAMES benchmark/benchmark.h ${_PAIMON_BENCHMARK_FIND_ARGS} + PATH_SUFFIXES include) + find_library(BENCHMARK_LIBRARY + NAMES benchmark ${_PAIMON_BENCHMARK_FIND_ARGS} + PATH_SUFFIXES lib lib64) + find_library(BENCHMARK_MAIN_LIBRARY + NAMES benchmark_main ${_PAIMON_BENCHMARK_FIND_ARGS} + PATH_SUFFIXES lib lib64) + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(benchmarkAlt REQUIRED_VARS BENCHMARK_INCLUDE_DIR + BENCHMARK_LIBRARY) + + if(benchmarkAlt_FOUND) + if(NOT TARGET benchmark::benchmark) + add_library(benchmark::benchmark UNKNOWN IMPORTED) + set_target_properties(benchmark::benchmark + PROPERTIES IMPORTED_LOCATION "${BENCHMARK_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES + "${BENCHMARK_INCLUDE_DIR}") + endif() + + if(BENCHMARK_MAIN_LIBRARY AND NOT TARGET benchmark::benchmark_main) + add_library(benchmark::benchmark_main UNKNOWN IMPORTED) + set_target_properties(benchmark::benchmark_main + PROPERTIES IMPORTED_LOCATION "${BENCHMARK_MAIN_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES + "${BENCHMARK_INCLUDE_DIR}") + endif() + endif() +else() + set(benchmarkAlt_FOUND TRUE) +endif() + +unset(_PAIMON_BENCHMARK_ROOTS) +unset(_PAIMON_BENCHMARK_FIND_ARGS) diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index f305d140..1179f28c 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -245,6 +245,18 @@ else() endif() endif() +if(DEFINED ENV{PAIMON_BENCHMARK_URL}) + set(BENCHMARK_SOURCE_URL "$ENV{PAIMON_BENCHMARK_URL}") +else() + if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_BENCHMARK_PKG_NAME}") + set_urls(BENCHMARK_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_BENCHMARK_PKG_NAME}") + else() + set_urls(BENCHMARK_SOURCE_URL + "${THIRDPARTY_MIRROR_URL}https://github.com/google/benchmark/archive/refs/tags/v${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz" + ) + endif() +endif() + if(DEFINED ENV{PAIMON_TBB_URL}) set(TBB_SOURCE_URL "$ENV{PAIMON_TBB_URL}") else() @@ -557,6 +569,8 @@ function(paimon_get_dependency_compat_target DEPENDENCY_NAME OUT_VAR) set(_target libprotobuf) elseif("${DEPENDENCY_NAME}" STREQUAL "GTest") set(_target GTest::gtest) + elseif("${DEPENDENCY_NAME}" STREQUAL "benchmark") + set(_target benchmark::benchmark) elseif("${DEPENDENCY_NAME}" STREQUAL "RE2") set(_target re2::re2) elseif("${DEPENDENCY_NAME}" STREQUAL "Snappy") @@ -643,6 +657,8 @@ macro(paimon_build_dependency DEPENDENCY_NAME) build_avro() elseif("${DEPENDENCY_NAME}" STREQUAL "GTest") build_gtest() + elseif("${DEPENDENCY_NAME}" STREQUAL "benchmark") + build_benchmark() else() message(FATAL_ERROR "No bundled build rule for ${DEPENDENCY_NAME}") endif() @@ -1824,6 +1840,49 @@ macro(build_tbb) endmacro(build_tbb) +macro(build_benchmark) + message(STATUS "Building benchmark from source") + + set(BENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/benchmark_ep-install") + set(BENCHMARK_INCLUDE_DIR "${BENCHMARK_PREFIX}/include") + set(BENCHMARK_STATIC_LIB + "${BENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(BENCHMARK_MAIN_STATIC_LIB + "${BENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark_main${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + set(BENCHMARK_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${BENCHMARK_PREFIX}" + -DBENCHMARK_ENABLE_TESTING=OFF + -DBENCHMARK_ENABLE_GTEST_TESTS=OFF + -DBENCHMARK_DOWNLOAD_DEPENDENCIES=OFF) + + externalproject_add(benchmark_ep + URL ${BENCHMARK_SOURCE_URL} + URL_HASH "SHA256=${PAIMON_BENCHMARK_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${BENCHMARK_CMAKE_ARGS} + BUILD_BYPRODUCTS "${BENCHMARK_STATIC_LIB}" + "${BENCHMARK_MAIN_STATIC_LIB}") + + file(MAKE_DIRECTORY "${BENCHMARK_INCLUDE_DIR}") + + add_library(benchmark::benchmark STATIC IMPORTED) + set_target_properties(benchmark::benchmark + PROPERTIES IMPORTED_LOCATION "${BENCHMARK_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${BENCHMARK_INCLUDE_DIR}") + add_dependencies(benchmark::benchmark benchmark_ep) + + add_library(benchmark::benchmark_main STATIC IMPORTED) + set_target_properties(benchmark::benchmark_main + PROPERTIES IMPORTED_LOCATION "${BENCHMARK_MAIN_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES + "${BENCHMARK_INCLUDE_DIR}") + add_dependencies(benchmark::benchmark_main benchmark_ep) +endmacro() + macro(build_glog) message(STATUS "Building glog from source") set(GLOG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/glog_ep-install") @@ -1892,6 +1951,9 @@ if(PAIMON_ENABLE_ORC) resolve_dependency(Protobuf) resolve_dependency(ORC) endif() +if(PAIMON_BUILD_BENCHMARKS) + resolve_dependency(benchmark) +endif() if(PAIMON_ENABLE_JINDO) build_jindosdk_c() build_jindosdk_nextarch() diff --git a/docs/source/examples/benchmark.rst b/docs/source/examples/benchmark.rst new file mode 100644 index 00000000..ee17be75 --- /dev/null +++ b/docs/source/examples/benchmark.rst @@ -0,0 +1,88 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +================ +Benchmark Usage +================ + +Paimon C++ provides Google Benchmark based cases for append-table write/read and +primary-key table write/MOR read paths. Benchmarks are disabled by default. + +Build +===== + +Enable benchmarks when configuring CMake:: + + cmake -S . -B build -DPAIMON_BUILD_BENCHMARKS=ON + cmake --build build --target paimon-read-write-benchmark + +Run all benchmark cases through CTest:: + + cmake --build build --target benchmark + +Custom Options +============== + +``paimon-read-write-benchmark`` accepts Google Benchmark options plus the Paimon +specific options below: + +``--paimon_source_data_file=`` + Source data file used to build benchmark data. Currently Parquet source files + are supported. + +``--paimon_source_table_path=`` + Read directly from an existing table path for ``BM_Read`` and ``BM_MOR_Read``. + When set, the source loading and pre-write stage are skipped. + +``--paimon_pk_columns=`` + Primary key columns for ``BM_PK_Write`` and ``BM_MOR_Read``. These cases + explicitly use ``bucket=1`` because benchmark batches are written to bucket 0. + +``--paimon_option=:;:`` + Repeatable table options passed through to Paimon. The default table file + format is ``parquet``; use ``--paimon_option file.format:`` to + override it. For ``BM_PK_Write`` and ``BM_MOR_Read``, ``bucket`` is forced to + ``1``. + +Examples +======== + +Append table write:: + + paimon-read-write-benchmark \ + --paimon_source_data_file /path/data.parquet \ + --benchmark_filter=BM_Write + +Append table read with four prefetch workers:: + + paimon-read-write-benchmark \ + --paimon_source_data_file /path/data.parquet \ + --benchmark_filter=BM_Read/4 + +Primary-key table write:: + + paimon-read-write-benchmark \ + --paimon_source_data_file /path/data.parquet \ + --paimon_pk_columns=id \ + --benchmark_filter=BM_PK_Write + +MOR read from an existing table:: + + paimon-read-write-benchmark \ + --paimon_source_table_path /path/table \ + --paimon_pk_columns=id \ + --benchmark_filter=BM_MOR_Read/4 diff --git a/docs/source/examples/index.rst b/docs/source/examples/index.rst index c26b7497..b3114f21 100644 --- a/docs/source/examples/index.rst +++ b/docs/source/examples/index.rst @@ -23,3 +23,4 @@ Examples write_commit_scan_read clean + benchmark diff --git a/src/paimon/testing/utils/CMakeLists.txt b/src/paimon/testing/utils/CMakeLists.txt index e0b9c80b..b12af3c8 100644 --- a/src/paimon/testing/utils/CMakeLists.txt +++ b/src/paimon/testing/utils/CMakeLists.txt @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -if(PAIMON_BUILD_TESTS) +if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS) set(PAIMON_TEST_UTILS testharness.cpp data_generator.cpp) @@ -28,6 +28,9 @@ if(PAIMON_BUILD_TESTS) STATIC_LINK_LIBS paimon_static ${GTEST_LINK_TOOLCHAIN}) +endif() + +if(PAIMON_BUILD_TESTS) add_paimon_test(test_utils_test SOURCES diff --git a/third_party/versions.txt b/third_party/versions.txt index a49a128a..6b80b7a8 100644 --- a/third_party/versions.txt +++ b/third_party/versions.txt @@ -60,6 +60,10 @@ PAIMON_GTEST_BUILD_VERSION=1.11.0 PAIMON_GTEST_BUILD_SHA256_CHECKSUM=b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5 PAIMON_GTEST_PKG_NAME=gtest-${PAIMON_GTEST_BUILD_VERSION}.tar.gz +PAIMON_BENCHMARK_BUILD_VERSION=1.9.1 +PAIMON_BENCHMARK_BUILD_SHA256_CHECKSUM=32131c08ee31eeff2c8968d7e874f3cb648034377dfc32a4c377fa8796d84981 +PAIMON_BENCHMARK_PKG_NAME=benchmark-${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz + PAIMON_ARROW_BUILD_VERSION=17.0.0 PAIMON_ARROW_BUILD_SHA256_CHECKSUM=9d280d8042e7cf526f8c28d170d93bfab65e50f94569f6a790982a878d8d898d PAIMON_ARROW_PKG_NAME=apache-arrow-${PAIMON_ARROW_BUILD_VERSION}.tar.gz @@ -127,6 +131,7 @@ DEPENDENCIES=( "PAIMON_TBB_URL ${PAIMON_TBB_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/uxlfoundation/oneTBB/archive/refs/tags/${PAIMON_TBB_BUILD_VERSION}.tar.gz" "PAIMON_ORC_URL ${PAIMON_ORC_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/orc/archive/refs/tags/${PAIMON_ORC_BUILD_VERSION}.tar.gz" "PAIMON_GTEST_URL ${PAIMON_GTEST_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/googletest/archive/release-${PAIMON_GTEST_BUILD_VERSION}.tar.gz" + "PAIMON_BENCHMARK_URL ${PAIMON_BENCHMARK_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/benchmark/archive/refs/tags/v${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz" "PAIMON_ARROW_URL ${PAIMON_ARROW_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/arrow/releases/download/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}.tar.gz" "PAIMON_AVRO_URL ${PAIMON_AVRO_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/avro/archive/${PAIMON_AVRO_BUILD_VERSION}.tar.gz" "PAIMON_FMT_URL ${PAIMON_FMT_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/fmtlib/fmt/archive/refs/tags/${PAIMON_FMT_BUILD_VERSION}.tar.gz" From 02d10953d6df18702b7bf002ad9d09a87bf9aa29 Mon Sep 17 00:00:00 2001 From: Joey Date: Mon, 8 Jun 2026 10:34:54 +0800 Subject: [PATCH 033/138] Allow FileStoreCommit for PK tables with postpone bucket mode * Allow FileStoreCommit for PK tables with postpone bucket mode Postpone bucket mode (bucket=-2) writes data like an append table: all files go to bucket--2/ directory and the REST catalog server handles bucket redistribution during background compaction. The commit logic (manifest and snapshot generation) is identical to append tables, so there is no reason to block it. See: https://paimon.apache.org/docs/master/primary-key-table/data-distribution/#postpone-bucket Co-Authored-By: Claude Opus 4.6 (1M context) * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Address review feedback: use NumBuckets() and Options::BUCKET constant - Replace raw schema options map lookup with TableSchema::NumBuckets() for postpone bucket check - Use Options::BUCKET constant instead of hardcoded "bucket" in tests - Add IsNotImplemented() status kind assertion in rejection test Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../core/operation/file_store_commit.cpp | 8 ++- .../operation/file_store_commit_impl_test.cpp | 58 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 194b4e46..4310e201 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -34,6 +34,7 @@ #include "paimon/core/operation/file_store_commit_impl.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/bucket_mode.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -71,7 +72,12 @@ Result> FileStoreCommit::Create( const auto& schema = table_schema.value(); if (!schema->PrimaryKeys().empty() && ctx->GetOptions().find("enable-pk-commit-in-inte-test") == ctx->GetOptions().end()) { - return Status::NotImplemented("not support pk table commit yet"); + // Postpone bucket mode (bucket=-2) writes all data files to the bucket-postpone/ directory. + // A compaction job will later redistribute files into real buckets. The commit logic + // (manifest and snapshot generation) is the same as append tables, so we allow it. + if (schema->NumBuckets() != BucketModeDefine::POSTPONE_BUCKET) { + return Status::NotImplemented("not support pk table commit yet"); + } } auto opts = schema->Options(); for (const auto& [key, value] : ctx->GetOptions()) { diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index d609821a..81e2636d 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -1690,4 +1690,62 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_FALSE(json.empty()); } +// Verify that FileStoreCommit::Create succeeds for PK tables with postpone bucket mode (bucket=-2) +// without requiring the enable-pk-commit-in-inte-test workaround flag. +TEST_F(FileStoreCommitImplTest, TestPostponeBucketPKTableCommitAllowed) { + auto pk_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(pk_dir); + std::string pk_root = pk_dir->Str(); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(pk_root, {})); + ASSERT_OK(catalog->CreateDatabase("db", {}, false)); + + arrow::Schema pk_schema( + {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())}); + ::ArrowSchema arrow_schema; + ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok()); + std::map table_options = {{Options::BUCKET, "-2"}}; + ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl"), &arrow_schema, + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options, + /*ignore_if_exists=*/false)); + + std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); + + // Create FileStoreCommit WITHOUT the workaround flag — should succeed for postpone bucket + CommitContextBuilder builder(pk_table_path, "test_user"); + builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); + ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto committer, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_TRUE(committer != nullptr); +} + +// Verify that FileStoreCommit::Create still rejects PK tables with fixed bucket (bucket > 0) +// when the workaround flag is not set. +TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitRejected) { + auto pk_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(pk_dir); + std::string pk_root = pk_dir->Str(); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(pk_root, {})); + ASSERT_OK(catalog->CreateDatabase("db", {}, false)); + + arrow::Schema pk_schema( + {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())}); + ::ArrowSchema arrow_schema; + ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok()); + std::map table_options = {{Options::BUCKET, "4"}}; + ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl_fixed"), &arrow_schema, + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options, + /*ignore_if_exists=*/false)); + + std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl_fixed"); + + CommitContextBuilder builder(pk_table_path, "test_user"); + builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); + ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); + auto result = FileStoreCommit::Create(std::move(commit_context)); + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().IsNotImplemented()); + ASSERT_TRUE(result.status().ToString().find("not support pk table commit") != + std::string::npos); +} + } // namespace paimon::test From 992160cfebe8aa29370f0af25469d88862647810 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Mon, 8 Jun 2026 12:02:00 +0800 Subject: [PATCH 034/138] fix: Avoid signed overflow UB in BSI index and null-deref in bloom filter index --- .../bloomfilter/bloom_filter_file_index.cpp | 10 ++- .../bsi/bit_slice_index_bitmap_file_index.cpp | 71 ++++++++++++++----- ...bit_slice_index_bitmap_file_index_test.cpp | 46 ++++++++++++ 3 files changed, 107 insertions(+), 20 deletions(-) diff --git a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp index b0a40ff0..81bcf04d 100644 --- a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp +++ b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp @@ -80,9 +80,15 @@ BloomFilterFileIndexReader::BloomFilterFileIndexReader(const FastHash::HashFunct Result> BloomFilterFileIndexReader::VisitEqual( const Literal& literal) { + // This returns `Remain` to align with the current Java implementation in BF index, even though + // its predicate semantics are inconsistent here. In practice, equality tests in predicate + // evaluation always return false when the literal is null. See + // `null_false_leaf_binary_function.h`. + if (literal.IsNull()) { + return FileIndexResult::Remain(); + } int64_t hash = hash_function_(literal); - return literal.IsNull() || filter_.TestHash(hash) ? FileIndexResult::Remain() - : FileIndexResult::Skip(); + return filter_.TestHash(hash) ? FileIndexResult::Remain() : FileIndexResult::Skip(); } } // namespace paimon diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp index 49ab89ea..aab38648 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp @@ -19,7 +19,9 @@ #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h" #include +#include #include +#include #include "fmt/format.h" #include "paimon/common/file_index/bsi/bit_slice_index_roaring_bitmap.h" @@ -33,7 +35,17 @@ #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" #include "paimon/utils/roaring_bitmap32.h" +namespace { +// Safe absolute value for int64_t that avoids undefined behavior when value == INT64_MIN. +// This mirrors Java's Math.abs() wrapping semantics. +inline int64_t SafeAbs(int64_t value) { + if (value == INT64_MIN) { + return INT64_MIN; + } + return value < 0 ? -value : value; +} +} // namespace namespace paimon { class MemoryPool; @@ -155,10 +167,14 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis BitmapIndexResult::BitmapSupplier bitmap_supplier = [literal = literal, reader = shared_from_this()]() -> Result { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - if (value >= 0) { + if (value == INT64_MIN) { + // Everything is greater than INT64_MIN (writer cannot store it) + return RoaringBitmap32::Or(reader->positive_->IsNotNull(), + reader->negative_->IsNotNull()); + } else if (value >= 0) { return reader->positive_->GreaterThan(value); } else { - PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->negative_->LessThan(-value)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->negative_->LessThan(SafeAbs(value))); RoaringBitmap32 b2 = reader->positive_->IsNotNull(); b1 |= b2; return b1; @@ -172,10 +188,15 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis BitmapIndexResult::BitmapSupplier bitmap_supplier = [literal = literal, reader = shared_from_this()]() -> Result { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - if (value >= 0) { + if (value == INT64_MIN) { + // All non-null rows satisfy x >= INT64_MIN + return RoaringBitmap32::Or(reader->positive_->IsNotNull(), + reader->negative_->IsNotNull()); + } else if (value >= 0) { return reader->positive_->GreaterOrEqual(value); } else { - PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->negative_->LessOrEqual(-value)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, + reader->negative_->LessOrEqual(SafeAbs(value))); RoaringBitmap32 b2 = reader->positive_->IsNotNull(); b1 |= b2; return b1; @@ -189,8 +210,11 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis BitmapIndexResult::BitmapSupplier bitmap_supplier = [literal = literal, reader = shared_from_this()]() -> Result { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - if (value < 0) { - return reader->negative_->GreaterThan(-value); + if (value == INT64_MIN) { + // Nothing is less than INT64_MIN + return RoaringBitmap32(); + } else if (value < 0) { + return reader->negative_->GreaterThan(SafeAbs(value)); } else { PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->positive_->LessThan(value)); RoaringBitmap32 b2 = reader->negative_->IsNotNull(); @@ -205,8 +229,11 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis BitmapIndexResult::BitmapSupplier bitmap_supplier = [literal = literal, reader = shared_from_this()]() -> Result { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - if (value < 0) { - return reader->negative_->GreaterOrEqual(-value); + if (value == INT64_MIN) { + // Writer cannot store INT64_MIN, so no row can match + return RoaringBitmap32(); + } else if (value < 0) { + return reader->negative_->GreaterOrEqual(SafeAbs(value)); } else { PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 b1, reader->positive_->LessOrEqual(value)); RoaringBitmap32 b2 = reader->negative_->IsNotNull(); @@ -234,13 +261,17 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis result_bitmaps.reserve(literals.size()); for (const auto& literal : literals) { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - RoaringBitmap32 equal; - if (value < 0) { - PAIMON_ASSIGN_OR_RAISE(equal, reader->negative_->Equal(-value)); + if (value == INT64_MIN) { + // Writer cannot store INT64_MIN, so no row can match it + result_bitmaps.emplace_back(); + } else if (value < 0) { + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 equal, + reader->negative_->Equal(SafeAbs(value))); + result_bitmaps.emplace_back(std::move(equal)); } else { - PAIMON_ASSIGN_OR_RAISE(equal, reader->positive_->Equal(value)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 equal, reader->positive_->Equal(value)); + result_bitmaps.emplace_back(std::move(equal)); } - result_bitmaps.emplace_back(std::move(equal)); } return RoaringBitmap32::FastUnion(result_bitmaps); }; @@ -257,13 +288,17 @@ Result> BitSliceIndexBitmapFileIndexReader::Vis result_bitmaps.reserve(literals.size()); for (const auto& literal : literals) { PAIMON_ASSIGN_OR_RAISE(int64_t value, reader->value_mapper_(literal)); - RoaringBitmap32 equal; - if (value < 0) { - PAIMON_ASSIGN_OR_RAISE(equal, reader->negative_->Equal(-value)); + if (value == INT64_MIN) { + // Writer cannot store INT64_MIN, so no row can match it + result_bitmaps.emplace_back(); + } else if (value < 0) { + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 equal, + reader->negative_->Equal(SafeAbs(value))); + result_bitmaps.emplace_back(std::move(equal)); } else { - PAIMON_ASSIGN_OR_RAISE(equal, reader->positive_->Equal(value)); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap32 equal, reader->positive_->Equal(value)); + result_bitmaps.emplace_back(std::move(equal)); } - result_bitmaps.emplace_back(std::move(equal)); } auto in = RoaringBitmap32::FastUnion(result_bitmaps); ebm -= in; diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp index b678ba3f..128c1a69 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index_test.cpp @@ -18,6 +18,7 @@ #include "paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.h" +#include #include #include "gtest/gtest.h" @@ -387,4 +388,49 @@ TEST_F(BitSliceIndexBitmapIndexReaderTest, TestUnInvalidType) { "BitSliceIndexBitmapFileIndex only support TINYINT/SMALLINT/INT/BIGINT/DATE"); } +TEST_F(BitSliceIndexBitmapIndexReaderTest, TestReaderPredicatePruningWithInt64Min) { + // Reuse TestPrimitiveType's index bytes. + // data: null, 1, null, 2, -1 (non-null rows: {1, 3, 4}) + std::vector index_bytes = { + 1, 0, 0, 0, 5, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 2, 58, + 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 58, 48, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0, 0, + 0, 0, 1, 58, 48, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 4, 0}; + auto input_stream = + std::make_shared(index_bytes.data(), index_bytes.size()); + BitSliceIndexBitmapFileIndex file_index({}); + ASSERT_OK_AND_ASSIGN( + auto reader, + file_index.CreateReader(CreateArrowSchema(arrow::int64()).get(), + /*start=*/0, /*length=*/index_bytes.size(), input_stream, pool_)); + + // x > INT64_MIN: all non-null rows (writer cannot store INT64_MIN) + CheckResult(reader->VisitGreaterThan(Literal(INT64_MIN)).value(), {1, 3, 4}); + + // x >= INT64_MIN: all non-null rows + CheckResult(reader->VisitGreaterOrEqual(Literal(INT64_MIN)).value(), {1, 3, 4}); + + // x < INT64_MIN: empty + CheckResult(reader->VisitLessThan(Literal(INT64_MIN)).value(), {}); + + // x <= INT64_MIN: empty (no row has INT64_MIN) + CheckResult(reader->VisitLessOrEqual(Literal(INT64_MIN)).value(), {}); + + // x == INT64_MIN: empty + CheckResult(reader->VisitEqual(Literal(INT64_MIN)).value(), {}); + + // x != INT64_MIN: all non-null rows + CheckResult(reader->VisitNotEqual(Literal(INT64_MIN)).value(), {1, 3, 4}); + + // x IN (INT64_MIN, -1): only -1 matches → row 4 + CheckResult(reader->VisitIn({Literal(INT64_MIN), Literal(static_cast(-1))}).value(), + {4}); + + // x NOT IN (INT64_MIN, -1): all non-null except row 4 + CheckResult(reader->VisitNotIn({Literal(INT64_MIN), Literal(static_cast(-1))}).value(), + {1, 3}); +} + } // namespace paimon::test From 1a59f6d3e8bfc5a9f8bc3b27adf3eec3afaf332f Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Mon, 8 Jun 2026 14:47:27 +0800 Subject: [PATCH 035/138] fix: LZ4 block compressor/decompressor safety issues From 0fc492687f87c13648e4694ec6a2b46eb3714091 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Mon, 8 Jun 2026 18:14:42 +0800 Subject: [PATCH 036/138] fix(avro): format avro types in decoder errors --- .../format/avro/avro_direct_decoder.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/paimon/format/avro/avro_direct_decoder.cpp b/src/paimon/format/avro/avro_direct_decoder.cpp index 34d8d7db..dbc1252a 100644 --- a/src/paimon/format/avro/avro_direct_decoder.cpp +++ b/src/paimon/format/avro/avro_direct_decoder.cpp @@ -289,8 +289,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, case arrow::Type::DATE32: { if (logical_type.type() != ::avro::LogicalType::Type::DATE) { return Status::TypeError( - fmt::format("Unexpected avro type [{}] with arrow type [{}].", type, - arrow_type->ToString())); + fmt::format("Unexpected avro type [{}] with arrow type [{}].", + ::avro::toString(type), arrow_type->ToString())); } auto* builder = arrow::internal::checked_cast(array_builder); @@ -299,8 +299,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, } default: return Status::TypeError( - fmt::format("Unexpected avro type [{}] with arrow type [{}].", type, - arrow_type->ToString())); + fmt::format("Unexpected avro type [{}] with arrow type [{}].", + ::avro::toString(type), arrow_type->ToString())); } } @@ -332,8 +332,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, } default: return Status::TypeError( - fmt::format("Unexpected avro type [{}] with arrow type [{}].", type, - array_builder->type()->ToString())); + fmt::format("Unexpected avro type [{}] with arrow type [{}].", + ::avro::toString(type), array_builder->type()->ToString())); } } @@ -379,8 +379,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, } default: return Status::TypeError( - fmt::format("Unexpected avro type [{}] with arrow type [{}].", type, - array_builder->type()->ToString())); + fmt::format("Unexpected avro type [{}] with arrow type [{}].", + ::avro::toString(type), array_builder->type()->ToString())); } } @@ -398,7 +398,8 @@ Status DecodeAvroValueToBuilder(const ::avro::NodePtr& avro_node, return DecodeMapToBuilder(avro_node, decoder, array_builder, ctx); } default: - return Status::Invalid(fmt::format("Unsupported avro type: {}", type)); + return Status::Invalid( + fmt::format("Unsupported avro type: {}", ::avro::toString(type))); } } From 1afcf69addf1392cadbfc95ac8c47b043282d50f Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Tue, 9 Jun 2026 08:51:16 +0800 Subject: [PATCH 037/138] fix: override CachedInputStream::Advance to avoid real I/O for skipped pages --- cmake_modules/arrow.diff | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff index f61b61ca..ae775209 100644 --- a/cmake_modules/arrow.diff +++ b/cmake_modules/arrow.diff @@ -220,7 +220,7 @@ index 4d3acb491e..3906ff3c59 100644 --- a/cpp/src/parquet/file_reader.cc +++ b/cpp/src/parquet/file_reader.cc -@@ -207,6 +207,100 @@ +@@ -207,6 +207,117 @@ return {col_start, col_length}; } @@ -308,6 +308,23 @@ index 4d3acb491e..3906ff3c59 100644 + return std::shared_ptr<::arrow::Buffer>(std::move(buf)); + } + ++ // Override Advance to avoid real I/O for skipped pages. ++ // The default InputStream::Advance() calls Read() and discards the result, ++ // which would trigger source_->ReadAt() on cache miss — defeating page-level ++ // I/O skipping via data_page_filter. Since Advance() is only used to skip ++ // over data that will not be consumed, we can safely just move the position. ++ ::arrow::Status Advance(int64_t nbytes) override { ++ if (nbytes <= 0) { ++ return ::arrow::Status::OK(); ++ } ++ int64_t remaining = length_ - position_; ++ if (remaining <= 0) { ++ return ::arrow::Status::OK(); ++ } ++ position_ += std::min(nbytes, remaining); ++ return ::arrow::Status::OK(); ++ } ++ + private: + std::shared_ptr<::arrow::io::internal::ReadRangeCache> cache_; + std::shared_ptr source_; @@ -321,7 +338,7 @@ index 4d3acb491e..3906ff3c59 100644 // RowGroupReader::Contents implementation for the Parquet file specification class SerializedRowGroup : public RowGroupReader::Contents { public: -@@ -242,6 +336,11 @@ +@@ -242,6 +343,11 @@ // segments. PARQUET_ASSIGN_OR_THROW(auto buffer, cached_source_->Read(col_range)); stream = std::make_shared<::arrow::io::BufferReader>(buffer); @@ -333,7 +350,7 @@ index 4d3acb491e..3906ff3c59 100644 } else { stream = properties_.GetStream(source_, col_range.offset, col_range.length); } -@@ -417,6 +516,26 @@ +@@ -417,6 +523,26 @@ return cached_source_->WaitFor(ranges); } @@ -360,7 +377,7 @@ index 4d3acb491e..3906ff3c59 100644 // Metadata/footer parsing. Divided up to separate sync/async paths, and to use // exceptions for error handling (with the async path converting to Future/Status). -@@ -911,6 +1030,22 @@ +@@ -911,6 +1037,22 @@ return file->WhenBuffered(row_groups, column_indices); } @@ -410,3 +427,13 @@ diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.c message(FATAL_ERROR "libtool found appears to be the incompatible GNU libtool: ${LIBTOOL_MACOS}" ) endif() + +diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h +--- a/cpp/src/arrow/io/interfaces.h ++++ b/cpp/src/arrow/io/interfaces.h +@@ -211,7 +211,7 @@ + /// \brief Advance or skip stream indicated number of bytes + /// \param[in] nbytes the number to move forward + /// \return Status +- Status Advance(int64_t nbytes); ++ virtual Status Advance(int64_t nbytes); From d9e6d3e679f70cd71a3516a164922173c41a487b Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Tue, 9 Jun 2026 11:21:19 +0800 Subject: [PATCH 038/138] fix: replace arrow type DECIMAL to DECIMAL128 --- src/paimon/common/data/binary_row_writer.cpp | 2 +- src/paimon/common/data/internal_row.cpp | 2 +- .../common/data/serializer/binary_serializer_utils.cpp | 4 ++-- .../common/data/serializer/row_compacted_serializer.cpp | 8 ++++---- src/paimon/common/utils/fields_comparator.cpp | 2 +- src/paimon/core/bucket/bucket_id_calculator.cpp | 2 +- src/paimon/core/io/row_to_arrow_array_converter.h | 6 +++--- .../core/mergetree/compact/aggregate/field_max_agg.h | 2 +- .../core/mergetree/compact/aggregate/field_min_agg.h | 2 +- .../core/mergetree/compact/aggregate/field_sum_agg.cpp | 4 ++-- src/paimon/core/mergetree/in_memory_sort_buffer.cpp | 2 +- src/paimon/core/schema/schema_validation.cpp | 5 +++-- 12 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/paimon/common/data/binary_row_writer.cpp b/src/paimon/common/data/binary_row_writer.cpp index c9b5ce05..0b346e04 100644 --- a/src/paimon/common/data/binary_row_writer.cpp +++ b/src/paimon/common/data/binary_row_writer.cpp @@ -143,7 +143,7 @@ Result BinaryRowWriter::CreateFieldSetter( }; return field_setter; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(field_type.get()); assert(decimal_type); diff --git a/src/paimon/common/data/internal_row.cpp b/src/paimon/common/data/internal_row.cpp index 1744e15b..abc54e7c 100644 --- a/src/paimon/common/data/internal_row.cpp +++ b/src/paimon/common/data/internal_row.cpp @@ -113,7 +113,7 @@ Result InternalRow::CreateFieldGetter( }; break; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(field_type.get()); assert(decimal_type); diff --git a/src/paimon/common/data/serializer/binary_serializer_utils.cpp b/src/paimon/common/data/serializer/binary_serializer_utils.cpp index 4675e54e..544e0ca2 100644 --- a/src/paimon/common/data/serializer/binary_serializer_utils.cpp +++ b/src/paimon/common/data/serializer/binary_serializer_utils.cpp @@ -106,7 +106,7 @@ Status BinarySerializerUtils::WriteBinaryData(const std::shared_ptrSetNullAt(pos, type_id); return Status::OK(); - } else if (type_id != arrow::Type::type::DECIMAL && + } else if (type_id != arrow::Type::type::DECIMAL128 && type_id != arrow::Type::type::TIMESTAMP) { // if row writer, exclude decimal and timestamp when set null writer->SetNullAt(pos); @@ -170,7 +170,7 @@ Status BinarySerializerUtils::WriteBinaryData(const std::shared_ptr(type.get()); assert(decimal_type); auto precision = decimal_type->precision(); diff --git a/src/paimon/common/data/serializer/row_compacted_serializer.cpp b/src/paimon/common/data/serializer/row_compacted_serializer.cpp index 4bf247b3..491f3a65 100644 --- a/src/paimon/common/data/serializer/row_compacted_serializer.cpp +++ b/src/paimon/common/data/serializer/row_compacted_serializer.cpp @@ -97,7 +97,7 @@ Result RowCompactedSerializer::CompareField(const FieldInfo& field_info PAIMON_ASSIGN_OR_RAISE(Timestamp val2, reader2->ReadTimestamp(field_info.precision)); return val1 == val2 ? 0 : (val1 < val2 ? -1 : 1); } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { PAIMON_ASSIGN_OR_RAISE(Decimal val1, reader1->ReadDecimal(field_info.precision, field_info.scale)); PAIMON_ASSIGN_OR_RAISE(Decimal val2, @@ -127,7 +127,7 @@ Result RowCompactedSerializer::CreateSliceComparat arrow::internal::checked_pointer_cast(field_type); assert(timestamp_type); field_infos[i].precision = DateTimeUtils::GetPrecisionFromType(timestamp_type); - } else if (field_type->id() == arrow::Type::type::DECIMAL) { + } else if (field_type->id() == arrow::Type::type::DECIMAL128) { auto decimal_type = arrow::internal::checked_pointer_cast(field_type); assert(decimal_type); @@ -278,7 +278,7 @@ Result RowCompactedSerializer::CreateFieldR }; break; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(field_type.get()); assert(decimal_type); @@ -413,7 +413,7 @@ Result RowCompactedSerializer::CreateFieldW }; break; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(field_type.get()); assert(decimal_type); diff --git a/src/paimon/common/utils/fields_comparator.cpp b/src/paimon/common/utils/fields_comparator.cpp index 54a5cfd9..bb6f9f2f 100644 --- a/src/paimon/common/utils/fields_comparator.cpp +++ b/src/paimon/common/utils/fields_comparator.cpp @@ -166,7 +166,7 @@ Result FieldsComparator::CompareField( return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1); }); } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { auto* decimal_type = arrow::internal::checked_cast(input_type.get()); assert(decimal_type); diff --git a/src/paimon/core/bucket/bucket_id_calculator.cpp b/src/paimon/core/bucket/bucket_id_calculator.cpp index 6bede5ab..c3dcb79e 100644 --- a/src/paimon/core/bucket/bucket_id_calculator.cpp +++ b/src/paimon/core/bucket/bucket_id_calculator.cpp @@ -208,7 +208,7 @@ static Result WriteBucketRow(int32_t col_id, }; return writer_func; } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { const auto* decimal_type = arrow::internal::checked_cast(field->type().get()); assert(decimal_type); diff --git a/src/paimon/core/io/row_to_arrow_array_converter.h b/src/paimon/core/io/row_to_arrow_array_converter.h index 247a315b..43c97afd 100644 --- a/src/paimon/core/io/row_to_arrow_array_converter.h +++ b/src/paimon/core/io/row_to_arrow_array_converter.h @@ -131,7 +131,7 @@ Status RowToArrowArrayConverter::Reserve(arrow::ArrayBuilder* array_builde case arrow::Type::type::FLOAT: case arrow::Type::type::DOUBLE: case arrow::Type::type::TIMESTAMP: - case arrow::Type::type::DECIMAL: + case arrow::Type::type::DECIMAL128: break; case arrow::Type::type::STRING: { // reserve string data buffer @@ -203,7 +203,7 @@ Status RowToArrowArrayConverter::Accumulate(const arrow::Array* array, int case arrow::Type::type::FLOAT: case arrow::Type::type::DOUBLE: case arrow::Type::type::TIMESTAMP: - case arrow::Type::type::DECIMAL: + case arrow::Type::type::DECIMAL128: break; case arrow::Type::type::STRING: { auto string_array = arrow::internal::checked_cast(array); @@ -405,7 +405,7 @@ RowToArrowArrayConverter::AppendField(bool use_view, arrow::ArrayBuilder* DateTimeUtils::TimestampToInteger(timestamp, time_type)); }); } - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { PAIMON_ASSIGN_OR_RAISE(auto* field_builder, CastToTypedBuilder(array_builder)); auto decimal_type = diff --git a/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h index e4c04dcf..15d98fa5 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h +++ b/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h @@ -63,7 +63,7 @@ class FieldMaxAgg : public FieldAggregator { case arrow::Type::type::FLOAT: case arrow::Type::type::DOUBLE: case arrow::Type::type::TIMESTAMP: - case arrow::Type::type::DECIMAL: + case arrow::Type::type::DECIMAL128: case arrow::Type::type::STRING: case arrow::Type::type::BINARY: return FieldMaxFunc([](const VariantType& accumulator, diff --git a/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h index 69e10736..e57a5ea5 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h +++ b/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h @@ -63,7 +63,7 @@ class FieldMinAgg : public FieldAggregator { case arrow::Type::type::FLOAT: case arrow::Type::type::DOUBLE: case arrow::Type::type::TIMESTAMP: - case arrow::Type::type::DECIMAL: + case arrow::Type::type::DECIMAL128: case arrow::Type::type::STRING: case arrow::Type::type::BINARY: return FieldMinFunc([](const VariantType& accumulator, diff --git a/src/paimon/core/mergetree/compact/aggregate/field_sum_agg.cpp b/src/paimon/core/mergetree/compact/aggregate/field_sum_agg.cpp index 021f0202..a543e896 100644 --- a/src/paimon/core/mergetree/compact/aggregate/field_sum_agg.cpp +++ b/src/paimon/core/mergetree/compact/aggregate/field_sum_agg.cpp @@ -73,7 +73,7 @@ Result FieldSumAgg::CreateSumFunc( DataDefine::GetVariantValue(input_field); return sum; }); - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { return FieldSumFunc( [](const VariantType& accumulator, const VariantType& input_field) -> VariantType { auto v1 = DataDefine::GetVariantValue(accumulator); @@ -122,7 +122,7 @@ Result FieldSumAgg::CreateNegFunc( auto value = DataDefine::GetVariantValue(input_field); return (-value); }); - case arrow::Type::type::DECIMAL: { + case arrow::Type::type::DECIMAL128: { return FieldNegFunc([](const VariantType& input_field) -> VariantType { auto value = DataDefine::GetVariantValue(input_field); return Decimal(value.Precision(), value.Scale(), -value.Value()); diff --git a/src/paimon/core/mergetree/in_memory_sort_buffer.cpp b/src/paimon/core/mergetree/in_memory_sort_buffer.cpp index 66c6dce4..cf1174f8 100644 --- a/src/paimon/core/mergetree/in_memory_sort_buffer.cpp +++ b/src/paimon/core/mergetree/in_memory_sort_buffer.cpp @@ -142,7 +142,7 @@ Result InMemorySortBuffer::EstimateMemoryUse(const std::shared_ptrlength() * sizeof(double); case arrow::Type::type::TIMESTAMP: return null_bits_size_in_bytes + array->length() * sizeof(int64_t); - case arrow::Type::type::DECIMAL: + case arrow::Type::type::DECIMAL128: return null_bits_size_in_bytes + array->length() * sizeof(Decimal::int128_t); case arrow::Type::type::STRING: case arrow::Type::type::BINARY: { diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index eb0199a0..14907d69 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -51,8 +51,9 @@ namespace paimon { bool SchemaValidation::IsComplexType(const std::shared_ptr& field) { - return (field->type()->id() == arrow::Type::TIMESTAMP || - field->type()->id() == arrow::Type::DECIMAL || BlobUtils::IsBlobField(field)); + arrow::Type::type arrow_type_id = field->type()->id(); + return (arrow_type_id == arrow::Type::TIMESTAMP || arrow_type_id == arrow::Type::DECIMAL128 || + BlobUtils::IsBlobField(field)); } Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { From 3fec1aa89f8318b51a5207c74ac3330a910b3e26 Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Tue, 9 Jun 2026 14:50:44 +0800 Subject: [PATCH 039/138] fix: remove trailing semicolons from statement-like macro definitions --- include/paimon/result.h | 4 ++-- src/paimon/common/file_index/bitmap/bitmap_index_result.cpp | 4 ++-- src/paimon/common/utils/arrow/status_utils.h | 4 ++-- src/paimon/core/casting/cast_executor_test.cpp | 2 +- src/paimon/core/manifest/manifest_entry_serializer.cpp | 2 +- .../mergetree/compact/merge_tree_compact_rewriter_test.cpp | 6 +++--- src/paimon/core/operation/abstract_file_store_write.cpp | 2 +- src/paimon/core/operation/file_store_commit_impl_test.cpp | 5 +++-- src/paimon/format/orc/orc_file_batch_reader_test.cpp | 2 +- src/paimon/global_index/lumina/lumina_global_index.cpp | 2 +- src/paimon/global_index/lumina/lumina_utils.h | 4 ++-- src/paimon/testing/utils/testharness.h | 6 +++--- 12 files changed, 22 insertions(+), 21 deletions(-) diff --git a/include/paimon/result.h b/include/paimon/result.h index 68d2f4bc..bfba6ba5 100644 --- a/include/paimon/result.h +++ b/include/paimon/result.h @@ -273,11 +273,11 @@ inline Status GenericToStatus(Result&& res) { #define PAIMON_ASSIGN_OR_RAISE_IMPL(result_name, lhs, rexpr) \ auto&& result_name = (rexpr); \ PAIMON_RETURN_IF_(!(result_name).ok(), (result_name).status(), PAIMON_STRINGIFY(rexpr)); \ - lhs = std::move(result_name).value(); + lhs = std::move(result_name).value() #define PAIMON_ASSIGN_OR_RAISE_NAME(x, y) PAIMON_CONCAT(x, y) #define PAIMON_ASSIGN_OR_RAISE(lhs, rexpr) \ PAIMON_ASSIGN_OR_RAISE_IMPL(PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, \ - (rexpr)); + (rexpr)) } // namespace paimon diff --git a/src/paimon/common/file_index/bitmap/bitmap_index_result.cpp b/src/paimon/common/file_index/bitmap/bitmap_index_result.cpp index 1027ce52..809ceeae 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_index_result.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_index_result.cpp @@ -50,7 +50,7 @@ Result> BitmapIndexResult::And( typed_other]() -> Result { PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* bitmap, result->GetBitmap()); PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* other_bitmap, - typed_other->GetBitmap()) + typed_other->GetBitmap()); return RoaringBitmap32::And(*bitmap, *other_bitmap); }); } @@ -66,7 +66,7 @@ Result> BitmapIndexResult::Or( typed_other]() -> Result { PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* bitmap, result->GetBitmap()); PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap32* other_bitmap, - typed_other->GetBitmap()) + typed_other->GetBitmap()); return RoaringBitmap32::Or(*bitmap, *other_bitmap); }); } diff --git a/src/paimon/common/utils/arrow/status_utils.h b/src/paimon/common/utils/arrow/status_utils.h index 40ac94cb..a0b505f6 100644 --- a/src/paimon/common/utils/arrow/status_utils.h +++ b/src/paimon/common/utils/arrow/status_utils.h @@ -99,10 +99,10 @@ inline Status ToPaimonStatus(const arrow::Status& status) { auto&& result_name = (rexpr); \ PAIMON_RETURN_IF_(!(result_name).ok(), ToPaimonStatus((result_name).status()), \ PAIMON_STRINGIFY(rexpr)); \ - lhs = std::move(result_name).ValueUnsafe(); + lhs = std::move(result_name).ValueUnsafe() #define PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(lhs, rexpr) \ PAIMON_ASSIGN_OR_RAISE_IMPL_FROM_ARROW( \ - PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, (rexpr)); + PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, (rexpr)) } // namespace paimon diff --git a/src/paimon/core/casting/cast_executor_test.cpp b/src/paimon/core/casting/cast_executor_test.cpp index df9caff0..ed540ae4 100644 --- a/src/paimon/core/casting/cast_executor_test.cpp +++ b/src/paimon/core/casting/cast_executor_test.cpp @@ -2525,7 +2525,7 @@ TEST_F(CastExecutorTest, TestBooleanToDecimalCastExecutorCastLiteral) { } { ASSERT_OK_AND_ASSIGN(Literal valid_literal, - cast_executor->Cast(Literal(false), arrow::decimal128(3, 3))) + cast_executor->Cast(Literal(false), arrow::decimal128(3, 3))); ASSERT_EQ(valid_literal, Literal(Decimal(3, 3, 0))); } } diff --git a/src/paimon/core/manifest/manifest_entry_serializer.cpp b/src/paimon/core/manifest/manifest_entry_serializer.cpp index b8176d8f..053405b8 100644 --- a/src/paimon/core/manifest/manifest_entry_serializer.cpp +++ b/src/paimon/core/manifest/manifest_entry_serializer.cpp @@ -54,7 +54,7 @@ Result ManifestEntrySerializer::ConvertFrom(int32_t version, return Status::Invalid("ManifestEntry convert from row failed, with null DataFileMeta"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr meta, - data_file_meta_serializer_.FromRow(*file)) + data_file_meta_serializer_.FromRow(*file)); return ManifestEntry(file_kind, partition, bucket, total_buckets, meta); } diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp index d546035c..891b46c4 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp @@ -119,7 +119,7 @@ TEST_F(MergeTreeCompactRewriterTest, TestSimple) { // generate sorted runs and rewrite ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns(table_path, table_schema, /*bucket=*/1, - /*partition=*/{{"f1", "10"}})) + /*partition=*/{{"f1", "10"}})); ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite( /*output_level=*/5, /*drop_delete=*/true, runs)); // check compact result @@ -216,7 +216,7 @@ TEST_F(MergeTreeCompactRewriterTest, TestNotDropDelete) { // generate sorted runs and rewrite ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns(table_path, table_schema, /*bucket=*/1, - /*partition=*/{{"f1", "10"}})) + /*partition=*/{{"f1", "10"}})); ASSERT_OK_AND_ASSIGN(auto compact_result, rewriter->Rewrite( /*output_level=*/5, /*drop_delete=*/false, runs)); // check compact result @@ -287,7 +287,7 @@ TEST_F(MergeTreeCompactRewriterTest, TestIOException) { // generate sorted runs and rewrite ASSERT_OK_AND_ASSIGN(auto runs, GenerateSortedRuns(table_path, table_schema, /*bucket=*/1, - /*partition=*/{{"f1", "10"}})) + /*partition=*/{{"f1", "10"}})); // rewrite may trigger I/O exception ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); diff --git a/src/paimon/core/operation/abstract_file_store_write.cpp b/src/paimon/core/operation/abstract_file_store_write.cpp index 24a5363e..b81375e7 100644 --- a/src/paimon/core/operation/abstract_file_store_write.cpp +++ b/src/paimon/core/operation/abstract_file_store_write.cpp @@ -128,7 +128,7 @@ Status AbstractFileStoreWrite::Write(std::unique_ptr&& batch) { PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*data, batch->GetData())); PAIMON_ASSIGN_OR_RAISE(BinaryRow partition, - file_store_path_factory_->ToBinaryRow(batch->GetPartition())) + file_store_path_factory_->ToBinaryRow(batch->GetPartition())); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr writer, GetWriter(partition, batch->GetBucket())); assert(writer); diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index 81e2636d..34b44f33 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -255,8 +255,9 @@ class FileStoreCommitImplTest : public testing::Test { EXPECT_OK_AND_ASSIGN(std::unique_ptr in_stream, file_system->Open(path)); EXPECT_TRUE(in_stream); - EXPECT_OK_AND_ASSIGN([[maybe_unused]] int32_t length, - in_stream->Read(reinterpret_cast(buffer.data()), buffer.size())) + EXPECT_OK_AND_ASSIGN( + [[maybe_unused]] int32_t length, + in_stream->Read(reinterpret_cast(buffer.data()), buffer.size())); EXPECT_OK(in_stream->Close()); auto pool = GetDefaultPool(); diff --git a/src/paimon/format/orc/orc_file_batch_reader_test.cpp b/src/paimon/format/orc/orc_file_batch_reader_test.cpp index a9fe5db9..8fbb5161 100644 --- a/src/paimon/format/orc/orc_file_batch_reader_test.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader_test.cpp @@ -156,7 +156,7 @@ class OrcFileBatchReaderTest : public ::testing::Test, const std::optional& selection_bitmap, int32_t batch_size) const { EXPECT_OK_AND_ASSIGN( auto orc_batch_reader, - OrcFileBatchReader::Create(std::move(in_stream), pool_, options, batch_size)) + OrcFileBatchReader::Create(std::move(in_stream), pool_, options, batch_size)); EXPECT_TRUE(orc_batch_reader); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); diff --git a/src/paimon/global_index/lumina/lumina_global_index.cpp b/src/paimon/global_index/lumina/lumina_global_index.cpp index 2d13aa12..9d0fcdd1 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index.cpp @@ -323,7 +323,7 @@ Result> LuminaIndexWriter::Finish() { PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, file_manager_->NewFileName(LuminaDefines::kIdentifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, - file_manager_->NewOutputStream(index_file_name)) + file_manager_->NewOutputStream(index_file_name)); auto file_writer = std::make_unique(out); PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.Dump(std::move(file_writer), io_options_)); // prepare GlobalIndexIOMeta diff --git a/src/paimon/global_index/lumina/lumina_utils.h b/src/paimon/global_index/lumina/lumina_utils.h index 60f79785..37a81170 100644 --- a/src/paimon/global_index/lumina/lumina_utils.h +++ b/src/paimon/global_index/lumina/lumina_utils.h @@ -36,11 +36,11 @@ namespace paimon::lumina { auto&& result_name = (rexpr); \ PAIMON_RETURN_IF_(!(result_name).IsOk(), LuminaToPaimonStatus((result_name).GetStatus()), \ PAIMON_STRINGIFY(rexpr)); \ - lhs = std::move(result_name).TakeValue(); + lhs = std::move(result_name).TakeValue() #define PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA(lhs, rexpr) \ PAIMON_ASSIGN_OR_RAISE_IMPL_FROM_LUMINA( \ - PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, (rexpr)); + PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, (rexpr)) inline ::lumina::core::Status PaimonToLuminaStatus(const Status& status) { switch (status.code()) { diff --git a/src/paimon/testing/utils/testharness.h b/src/paimon/testing/utils/testharness.h index 5c92fa14..0886b517 100644 --- a/src/paimon/testing/utils/testharness.h +++ b/src/paimon/testing/utils/testharness.h @@ -93,15 +93,15 @@ ::testing::AssertionResult AssertStatus(const char* s_expr, const Status& s); #define ASSIGN_OR_HANDLE_ERROR_IMPL(handle_error, status_name, lhs, rexpr) \ auto&& status_name = (rexpr); \ handle_error(status_name.status()); \ - lhs = std::move(status_name).value(); + lhs = std::move(status_name).value() #define ASSERT_OK_AND_ASSIGN(lhs, rexpr) \ ASSIGN_OR_HANDLE_ERROR_IMPL( \ - ASSERT_OK, PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr); + ASSERT_OK, PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr) #define EXPECT_OK_AND_ASSIGN(lhs, rexpr) \ ASSIGN_OR_HANDLE_ERROR_IMPL( \ - EXPECT_OK, PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr); + EXPECT_OK, PAIMON_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr) #define EXPECT_OK(s) EXPECT_PRED_FORMAT1(paimon::test::AssertStatus, s) #define EXPECT_NOK(s) EXPECT_FALSE((s).ok()) From 09878639bb2e3011b028998f81d9fd29232e002e Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Tue, 9 Jun 2026 16:52:24 +0800 Subject: [PATCH 040/138] fix: change FileStorePathFactory::Create() to return shared_ptr From 7f3b6516d85c88f457a7d03d0b41440e7e2aabac Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:51:59 +0800 Subject: [PATCH 041/138] refactor(blob): refactor BlobFileContext to honor the actual schema when classifying BLOB fields --- .../core/operation/blob_file_context.cpp | 60 +++++++-------- src/paimon/core/operation/blob_file_context.h | 16 ---- .../core/operation/blob_file_context_test.cpp | 74 +++++++++++-------- 3 files changed, 67 insertions(+), 83 deletions(-) diff --git a/src/paimon/core/operation/blob_file_context.cpp b/src/paimon/core/operation/blob_file_context.cpp index de6cd857..270a6c4a 100644 --- a/src/paimon/core/operation/blob_file_context.cpp +++ b/src/paimon/core/operation/blob_file_context.cpp @@ -42,51 +42,61 @@ BlobFileContext::BlobFileContext(std::set descriptor_fields, std::unique_ptr BlobFileContext::Create( const std::shared_ptr& schema, const CoreOptions& options) { - // Check if there are any BLOB fields in the schema - bool has_blob = false; + // Collect the BLOB field names that are present in the given schema. The schema may + // only contain a subset of the table columns (e.g. a projected read/write schema), so all + // field categories below must be derived from this set rather than from the options + // alone, which describe the full table. + std::set schema_blob_fields; for (int i = 0; i < schema->num_fields(); ++i) { - if (BlobUtils::IsBlobField(schema->field(i))) { - has_blob = true; - break; + const auto& field = schema->field(i); + if (BlobUtils::IsBlobField(field)) { + schema_blob_fields.insert(field->name()); } } - if (!has_blob) { + if (schema_blob_fields.empty()) { return nullptr; } // Populate descriptor fields std::set descriptor_fields; for (const auto& name : options.GetBlobDescriptorFields()) { - descriptor_fields.insert(name); + if (schema_blob_fields.count(name) > 0) { + descriptor_fields.insert(name); + } } // Populate view fields std::set view_fields; for (const auto& name : options.GetBlobViewFields()) { - view_fields.insert(name); + if (schema_blob_fields.count(name) > 0) { + view_fields.insert(name); + } } // Populate inline fields from options (descriptor ∪ view) std::set inline_fields; for (const auto& name : options.GetBlobInlineFields()) { - inline_fields.insert(name); + if (schema_blob_fields.count(name) > 0) { + inline_fields.insert(name); + } } // Populate external storage fields std::set external_storage_fields; for (const auto& name : options.GetBlobExternalStorageFields()) { - external_storage_fields.insert(name); + if (schema_blob_fields.count(name) > 0) { + external_storage_fields.insert(name); + } } // Populate external storage path std::optional external_storage_path = options.GetBlobExternalStoragePath(); - // Determine blob_file_fields: BLOB fields that are NOT inline + // Determine blob_file_fields: schema BLOB fields that are NOT inline std::set blob_file_fields; - for (int i = 0; i < schema->num_fields(); ++i) { - const auto& field = schema->field(i); - if (BlobUtils::IsBlobField(field) && inline_fields.count(field->name()) == 0) { - blob_file_fields.insert(field->name()); + for (const auto& name : schema_blob_fields) { + if (inline_fields.count(name) == 0) { + blob_file_fields.insert(name); } } @@ -96,26 +106,6 @@ std::unique_ptr BlobFileContext::Create( std::move(blob_file_fields), std::move(external_storage_path))); } -bool BlobFileContext::IsInlineField(const std::string& field_name) const { - return inline_fields_.count(field_name) > 0; -} - -bool BlobFileContext::IsBlobFileField(const std::string& field_name) const { - return blob_file_fields_.count(field_name) > 0; -} - -bool BlobFileContext::IsDescriptorField(const std::string& field_name) const { - return descriptor_fields_.count(field_name) > 0; -} - -bool BlobFileContext::IsViewField(const std::string& field_name) const { - return view_fields_.count(field_name) > 0; -} - -bool BlobFileContext::IsExternalStorageField(const std::string& field_name) const { - return external_storage_fields_.count(field_name) > 0; -} - bool BlobFileContext::RequireBlobFileWriter() const { return !blob_file_fields_.empty(); } diff --git a/src/paimon/core/operation/blob_file_context.h b/src/paimon/core/operation/blob_file_context.h index 36f524ba..e3891d1f 100644 --- a/src/paimon/core/operation/blob_file_context.h +++ b/src/paimon/core/operation/blob_file_context.h @@ -49,22 +49,6 @@ class BlobFileContext { static std::unique_ptr Create(const std::shared_ptr& schema, const CoreOptions& options); - /// Returns true if the given field should be stored inline in the main data file - /// (either as descriptor bytes or view bytes). - bool IsInlineField(const std::string& field_name) const; - - /// Returns true if the given field should be written to a separate .blob file. - bool IsBlobFileField(const std::string& field_name) const; - - /// Returns true if the given field is a descriptor field. - bool IsDescriptorField(const std::string& field_name) const; - - /// Returns true if the given field is a view field. - bool IsViewField(const std::string& field_name) const; - - /// Returns true if the given field should be written to external storage. - bool IsExternalStorageField(const std::string& field_name) const; - /// Returns true if there are any BLOB fields that need a .blob file writer. bool RequireBlobFileWriter() const; diff --git a/src/paimon/core/operation/blob_file_context_test.cpp b/src/paimon/core/operation/blob_file_context_test.cpp index a5d9f75e..9565aaeb 100644 --- a/src/paimon/core/operation/blob_file_context_test.cpp +++ b/src/paimon/core/operation/blob_file_context_test.cpp @@ -94,18 +94,6 @@ TEST_F(BlobFileContextTest, MixedInlineAndBlobFile) { // blob file fields = non-inline blob fields ASSERT_EQ(context->GetBlobFileFields(), std::set({"video", "audio"})); - // Query methods - ASSERT_TRUE(context->IsInlineField("image")); - ASSERT_TRUE(context->IsDescriptorField("image")); - ASSERT_FALSE(context->IsViewField("image")); - ASSERT_FALSE(context->IsBlobFileField("image")); - - ASSERT_FALSE(context->IsInlineField("video")); - ASSERT_TRUE(context->IsBlobFileField("video")); - - ASSERT_FALSE(context->IsInlineField("audio")); - ASSERT_TRUE(context->IsBlobFileField("audio")); - // Requires blob file writer for video and audio ASSERT_TRUE(context->RequireBlobFileWriter()); ASSERT_FALSE(context->RequireExternalStorageWriter()); @@ -129,9 +117,6 @@ TEST_F(BlobFileContextTest, ExternalStorageFields) { ASSERT_EQ(context->GetExternalStoragePath(), "oss://bucket/blob/"); ASSERT_TRUE(context->GetBlobFileFields().empty()); - ASSERT_TRUE(context->IsExternalStorageField("image")); - ASSERT_FALSE(context->IsExternalStorageField("video")); - ASSERT_FALSE(context->RequireBlobFileWriter()); ASSERT_TRUE(context->RequireExternalStorageWriter()); } @@ -151,10 +136,6 @@ TEST_F(BlobFileContextTest, ViewFields) { ASSERT_EQ(context->GetInlineFields(), std::set({"ref_image"})); ASSERT_EQ(context->GetBlobFileFields(), std::set({"raw_blob"})); - ASSERT_TRUE(context->IsInlineField("ref_image")); - ASSERT_TRUE(context->IsViewField("ref_image")); - ASSERT_FALSE(context->IsDescriptorField("ref_image")); - ASSERT_TRUE(context->RequireBlobFileWriter()); ASSERT_FALSE(context->RequireExternalStorageWriter()); } @@ -179,24 +160,53 @@ TEST_F(BlobFileContextTest, DescriptorAndViewTogether) { ASSERT_EQ(context->GetExternalStoragePath(), "/tmp/ext/"); ASSERT_EQ(context->GetBlobFileFields(), std::set({"normal_blob"})); - ASSERT_TRUE(context->IsDescriptorField("desc_blob")); - ASSERT_TRUE(context->IsExternalStorageField("desc_blob")); - ASSERT_TRUE(context->IsInlineField("desc_blob")); - ASSERT_FALSE(context->IsBlobFileField("desc_blob")); + ASSERT_TRUE(context->RequireBlobFileWriter()); + ASSERT_TRUE(context->RequireExternalStorageWriter()); +} - ASSERT_TRUE(context->IsViewField("view_blob")); - ASSERT_TRUE(context->IsInlineField("view_blob")); - ASSERT_FALSE(context->IsDescriptorField("view_blob")); +TEST_F(BlobFileContextTest, PartialSchemaIgnoresAbsentFields) { + // Schema only carries "image"; "video" and "audio" are not part of this write schema. + auto schema = MakeSchema({"id"}, {"image"}); + std::map opts_map = { + {Options::BLOB_DESCRIPTOR_FIELD, "image,audio"}, + {Options::BLOB_VIEW_FIELD, "video"}, + {Options::BLOB_EXTERNAL_STORAGE_FIELD, "image,video"}, + {Options::BLOB_EXTERNAL_STORAGE_PATH, "oss://bucket/blob/"}, + }; + ASSERT_OK_AND_ASSIGN(auto options, CoreOptions::FromMap(opts_map)); + auto context = BlobFileContext::Create(schema, options); + ASSERT_TRUE(context); - ASSERT_FALSE(context->IsInlineField("normal_blob")); - ASSERT_TRUE(context->IsBlobFileField("normal_blob")); + // Only "image" survives filtering; "audio" / "video" are not in the schema. + ASSERT_EQ(context->GetDescriptorFields(), std::set({"image"})); + ASSERT_TRUE(context->GetViewFields().empty()); + ASSERT_EQ(context->GetInlineFields(), std::set({"image"})); + ASSERT_EQ(context->GetExternalStorageFields(), std::set({"image"})); - // Non-existent field - ASSERT_FALSE(context->IsInlineField("not_exist")); - ASSERT_FALSE(context->IsBlobFileField("not_exist")); + // No non-inline blob field remains in the schema. + ASSERT_TRUE(context->GetBlobFileFields().empty()); - ASSERT_TRUE(context->RequireBlobFileWriter()); + ASSERT_FALSE(context->RequireBlobFileWriter()); ASSERT_TRUE(context->RequireExternalStorageWriter()); } +TEST_F(BlobFileContextTest, PartialSchemaWithOnlyBlobFileField) { + auto schema = MakeSchema({"id"}, {"audio"}); + std::map opts_map = { + {Options::BLOB_DESCRIPTOR_FIELD, "image"}, {Options::BLOB_VIEW_FIELD, "video"}, + // "audio" is not configured as inline -> goes to .blob file + }; + ASSERT_OK_AND_ASSIGN(auto options, CoreOptions::FromMap(opts_map)); + auto context = BlobFileContext::Create(schema, options); + ASSERT_TRUE(context); + + ASSERT_TRUE(context->GetDescriptorFields().empty()); + ASSERT_TRUE(context->GetViewFields().empty()); + ASSERT_TRUE(context->GetInlineFields().empty()); + ASSERT_EQ(context->GetBlobFileFields(), std::set({"audio"})); + + ASSERT_TRUE(context->RequireBlobFileWriter()); + ASSERT_FALSE(context->RequireExternalStorageWriter()); +} + } // namespace paimon From 552080669b52289c2ad5fac4f7a9d466f32efc8a Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Wed, 10 Jun 2026 09:30:34 +0800 Subject: [PATCH 042/138] fix: address ubsan findings --- build_support/lsan-suppressions.txt | 7 +++ build_support/ubsan-suppressions.txt | 4 ++ cmake_modules/BuildUtils.cmake | 14 +++-- cmake_modules/san-config.cmake | 5 +- .../common/data/columnar/columnar_utils.h | 4 +- src/paimon/common/data/decimal.cpp | 10 +++- src/paimon/common/data/decimal_test.cpp | 15 ++++- .../global_index/global_index_utils_test.cpp | 4 +- .../io/memory_segment_output_stream.cpp | 4 +- src/paimon/common/memory/memory_segment.h | 17 ++++-- src/paimon/common/predicate/literal.cpp | 8 ++- src/paimon/common/utils/decimal_utils.cpp | 57 +++++++++++++------ .../common/utils/field_type_utils_test.cpp | 7 +-- .../core/bucket/hive_bucket_function.cpp | 24 ++++---- src/paimon/core/bucket/hive_bucket_function.h | 2 +- .../core/bucket/hive_bucket_function_test.cpp | 6 +- src/paimon/core/bucket/hive_hasher.h | 24 ++++---- src/paimon/format/orc/orc_adapter.cpp | 21 +++++-- 18 files changed, 159 insertions(+), 74 deletions(-) diff --git a/build_support/lsan-suppressions.txt b/build_support/lsan-suppressions.txt index 927afb39..e9f3bb0c 100644 --- a/build_support/lsan-suppressions.txt +++ b/build_support/lsan-suppressions.txt @@ -17,3 +17,10 @@ # False positive from atexit() registration in libc leak:*__new_exitfn* + +# Lance's Rust/Tokio runtime can leave worker/TLS allocations alive at process +# exit. Suppress these third-party runtime shutdown leftovers without hiding +# all leaks from liblance_lib_rc.so. +leak:tokio::runtime::blocking::pool::spawn_blocking +leak:tokio::runtime::scheduler::multi_thread::worker::create +leak:std::thread::Builder::spawn_unchecked diff --git a/build_support/ubsan-suppressions.txt b/build_support/ubsan-suppressions.txt index 13a83393..d3151a8a 100644 --- a/build_support/ubsan-suppressions.txt +++ b/build_support/ubsan-suppressions.txt @@ -14,3 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +# Arrow's ISO8601 string-to-timestamp parser intentionally reaches the int64 +# nanosecond boundary for values such as 1677-09-21 00:12:43.145224192. +signed-integer-overflow:std::chrono::__duration_cast_impl diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index 1bc99e70..efca80b7 100644 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -92,6 +92,8 @@ function(add_paimon_lib LIB_NAME) # Generate a single "objlib" from all C++ modules and link # that "objlib" into each library kind, to avoid compiling twice add_library(${LIB_NAME}_objlib OBJECT ${ARG_SOURCES}) + target_link_libraries(${LIB_NAME}_objlib + PRIVATE "$") if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") target_compile_options(${LIB_NAME}_objlib PRIVATE -Wno-global-constructors) endif() @@ -180,12 +182,12 @@ function(add_paimon_lib LIB_NAME) PUBLIC "$") if(NOT APPLE) - target_link_options(${LIB_NAME}_shared - PRIVATE - -Wl,--exclude-libs,ALL - -Wl,-Bsymbolic - -Wl,-z,defs - -Wl,--gc-sections) + set(SHARED_LINK_OPTIONS -Wl,--exclude-libs,ALL -Wl,-Bsymbolic + -Wl,--gc-sections) + if(NOT PAIMON_USE_ASAN AND NOT PAIMON_USE_UBSAN) + list(APPEND SHARED_LINK_OPTIONS -Wl,-z,defs) + endif() + target_link_options(${LIB_NAME}_shared PRIVATE ${SHARED_LINK_OPTIONS}) endif() install(TARGETS ${LIB_NAME}_shared ${INSTALL_IS_OPTIONAL} diff --git a/cmake_modules/san-config.cmake b/cmake_modules/san-config.cmake index bb078c72..7b985e5b 100644 --- a/cmake_modules/san-config.cmake +++ b/cmake_modules/san-config.cmake @@ -29,8 +29,9 @@ endif() if(PAIMON_USE_UBSAN) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - target_compile_options(paimon_sanitizer_flags INTERFACE -fsanitize=undefined - -fno-omit-frame-pointer) + target_compile_options(paimon_sanitizer_flags + INTERFACE -fsanitize=undefined -fno-sanitize=vptr + -fno-omit-frame-pointer) target_link_options(paimon_sanitizer_flags INTERFACE -fsanitize=undefined) message(STATUS "Undefined Behavior Sanitizer enabled") else() diff --git a/src/paimon/common/data/columnar/columnar_utils.h b/src/paimon/common/data/columnar/columnar_utils.h index ca9f58ef..c1270ed5 100644 --- a/src/paimon/common/data/columnar/columnar_utils.h +++ b/src/paimon/common/data/columnar/columnar_utils.h @@ -109,7 +109,9 @@ class ColumnarUtils { MemoryPool* pool) { auto view = GetView(array, pos); std::shared_ptr bytes = Bytes::AllocateBytes(view.size(), pool); - memcpy(bytes->data(), view.data(), view.size()); + if (!view.empty()) { + memcpy(bytes->data(), view.data(), view.size()); + } return bytes; } }; diff --git a/src/paimon/common/data/decimal.cpp b/src/paimon/common/data/decimal.cpp index 0ce5b220..20bf4664 100644 --- a/src/paimon/common/data/decimal.cpp +++ b/src/paimon/common/data/decimal.cpp @@ -110,10 +110,14 @@ Decimal Decimal::FromUnscaledBytes(int32_t precision, int32_t scale, Bytes* byte int32_t Decimal::clz_u128(uint128_t u) { uint64_t hi = u >> 64; + if (hi != 0) { + return __builtin_clzll(hi); + } uint64_t lo = u; - int32_t retval[3] = {__builtin_clzll(hi), __builtin_clzll(lo) + 64, 128}; - int32_t idx = !hi + ((!lo) & (!hi)); - return retval[idx]; + if (lo != 0) { + return __builtin_clzll(lo) + 64; + } + return 128; } int32_t Decimal::count_leading_zero_bytes(uint128_t u) { diff --git a/src/paimon/common/data/decimal_test.cpp b/src/paimon/common/data/decimal_test.cpp index 038b3d3b..a37a6bef 100644 --- a/src/paimon/common/data/decimal_test.cpp +++ b/src/paimon/common/data/decimal_test.cpp @@ -133,7 +133,12 @@ TEST(DecimalTest, TestCompatibleWithJava) { } TEST(DecimalTest, TestCompareTo) { - auto CheckResult = [](const Decimal& decimal1, const Decimal& decimal2) { + auto min_int128 = DecimalUtils::StrToInt128("-170141183460469231731687303715884105728").value(); + auto can_negate = [min_int128](const Decimal& decimal) { + return decimal.Value() != min_int128; + }; + + auto CheckResult = [can_negate](const Decimal& decimal1, const Decimal& decimal2) { ASSERT_FALSE(decimal1 < decimal1); ASSERT_FALSE(decimal1 > decimal1); ASSERT_EQ(decimal1, decimal1); @@ -146,6 +151,9 @@ TEST(DecimalTest, TestCompareTo) { ASSERT_EQ(decimal3.CompareTo(decimal1), 0); ASSERT_EQ(decimal3, decimal1); + if (!can_negate(decimal1) || !can_negate(decimal2)) { + return; + } Decimal negative_decimal1(decimal1.Precision(), decimal1.Scale(), -decimal1.Value()); Decimal negative_decimal2(decimal2.Precision(), decimal2.Scale(), -decimal2.Value()); ASSERT_EQ(negative_decimal1.CompareTo(negative_decimal2), 1); @@ -157,10 +165,13 @@ TEST(DecimalTest, TestCompareTo) { ASSERT_EQ(negative_decimal3, negative_decimal1); }; - auto CheckEqual = [](const Decimal& decimal1, const Decimal& decimal2) { + auto CheckEqual = [can_negate](const Decimal& decimal1, const Decimal& decimal2) { ASSERT_EQ(decimal1.CompareTo(decimal2), 0); ASSERT_EQ(decimal2.CompareTo(decimal1), 0); + if (!can_negate(decimal1) || !can_negate(decimal2)) { + return; + } Decimal negative_decimal1(decimal1.Precision(), decimal1.Scale(), -decimal1.Value()); Decimal negative_decimal2(decimal2.Precision(), decimal2.Scale(), -decimal2.Value()); ASSERT_EQ(negative_decimal1.CompareTo(negative_decimal2), 0); diff --git a/src/paimon/common/global_index/global_index_utils_test.cpp b/src/paimon/common/global_index/global_index_utils_test.cpp index c31aebd0..7354bced 100644 --- a/src/paimon/common/global_index/global_index_utils_test.cpp +++ b/src/paimon/common/global_index/global_index_utils_test.cpp @@ -34,7 +34,9 @@ class GlobalIndexUtilsTest : public ::testing::Test { /// Helper to create a valid ArrowArray with the given number of int32 elements. static ArrowArray CreateInt32Array(const std::vector& values) { arrow::Int32Builder builder; - EXPECT_TRUE(builder.AppendValues(values).ok()); + if (!values.empty()) { + EXPECT_TRUE(builder.AppendValues(values).ok()); + } std::shared_ptr array; EXPECT_TRUE(builder.Finish(&array).ok()); ArrowArray c_array; diff --git a/src/paimon/common/io/memory_segment_output_stream.cpp b/src/paimon/common/io/memory_segment_output_stream.cpp index 498b5abc..5355f72b 100644 --- a/src/paimon/common/io/memory_segment_output_stream.cpp +++ b/src/paimon/common/io/memory_segment_output_stream.cpp @@ -55,7 +55,9 @@ void MemorySegmentOutputStream::WriteString(const std::string& str) { void MemorySegmentOutputStream::Write(const char* data, uint32_t size) { auto bytes = std::make_shared(size, pool_.get()); - memcpy(bytes->data(), data, size); + if (size != 0) { + memcpy(bytes->data(), data, size); + } auto segment = MemorySegment::Wrap(bytes); Write(segment, 0, segment.Size()); } diff --git a/src/paimon/common/memory/memory_segment.h b/src/paimon/common/memory/memory_segment.h index a98e5d8b..7fc2a111 100644 --- a/src/paimon/common/memory/memory_segment.h +++ b/src/paimon/common/memory/memory_segment.h @@ -112,14 +112,18 @@ class PAIMON_EXPORT MemorySegment { inline void Get(int32_t index, T* dst, int32_t offset, int32_t length) const { assert(static_cast(dst->size()) >= (offset + length)); assert(size_ >= (index + length)); - std::memcpy(const_cast(dst->data()) + offset, data_ + index, length); + if (length != 0) { + std::memcpy(const_cast(dst->data()) + offset, data_ + index, length); + } } template inline void Put(int32_t index, const T& src, int32_t offset, int32_t length) { assert(static_cast(src.size()) >= (offset + length)); assert(size_ >= (index + length)); - std::memcpy(MutableData() + index, src.data() + offset, length); + if (length != 0) { + std::memcpy(MutableData() + index, src.data() + offset, length); + } } template @@ -150,13 +154,16 @@ class PAIMON_EXPORT MemorySegment { assert(offset >= 0); assert(target_offset >= 0); assert(num_bytes >= 0); - - std::memcpy(target->MutableData() + target_offset, data_ + offset, num_bytes); + if (num_bytes != 0) { + std::memcpy(target->MutableData() + target_offset, data_ + offset, num_bytes); + } } void CopyToUnsafe(int32_t offset, void* target, int32_t target_offset, int32_t num_bytes) const { - std::memcpy(static_cast(target) + target_offset, data_ + offset, num_bytes); + if (num_bytes != 0) { + std::memcpy(static_cast(target) + target_offset, data_ + offset, num_bytes); + } } int32_t Compare(const MemorySegment& seg2, int32_t offset1, int32_t offset2, int32_t len) const; diff --git a/src/paimon/common/predicate/literal.cpp b/src/paimon/common/predicate/literal.cpp index 0157a38b..3b2bcc0e 100644 --- a/src/paimon/common/predicate/literal.cpp +++ b/src/paimon/common/predicate/literal.cpp @@ -158,7 +158,9 @@ Literal::Literal(FieldType binary_type, const char* str, size_t size, bool own_d impl_->own_data_ = own_data; if (own_data) { impl_->value_.Buffer = new char[size]; - memcpy(impl_->value_.Buffer, str, size); + if (size > 0) { + memcpy(impl_->value_.Buffer, str, size); + } impl_->hash_code_ = impl_->CalculateHashCode(); } else { impl_->value_.Buffer = const_cast(str); @@ -221,7 +223,9 @@ Literal& Literal::operator=(const Literal& other) { impl_->type_ == FieldType::BLOB) && impl_->own_data_) { impl_->value_.Buffer = new char[other.impl_->size_]; - memcpy(impl_->value_.Buffer, other.impl_->value_.Buffer, other.impl_->size_); + if (other.impl_->size_ > 0) { + memcpy(impl_->value_.Buffer, other.impl_->value_.Buffer, other.impl_->size_); + } } else { impl_->value_ = other.impl_->value_; } diff --git a/src/paimon/common/utils/decimal_utils.cpp b/src/paimon/common/utils/decimal_utils.cpp index ff4aa1bf..2cee2561 100644 --- a/src/paimon/common/utils/decimal_utils.cpp +++ b/src/paimon/common/utils/decimal_utils.cpp @@ -20,6 +20,7 @@ #include "paimon/common/utils/decimal_utils.h" #include +#include #include #include @@ -74,27 +75,51 @@ std::optional DecimalUtils::RescaleDecimalWithOverflowCheck( Result DecimalUtils::StrToInt128(const std::string& str) { try { - Decimal::int128_t ret = 0; size_t length = str.length(); - if (length > 0) { - bool is_negative = str[0] == '-'; - size_t posn = is_negative ? 1 : 0; - while (posn < length) { - size_t group = std::min(18ul, length - posn); - int64_t chunk = std::stoll(str.substr(posn, group)); - int64_t multiple = 1; - for (size_t i = 0; i < group; ++i) { - multiple *= 10; + if (length == 0) { + return Status::Invalid("invalid string: [], cannot convert to int128"); + } + bool is_negative = str[0] == '-'; + size_t posn = is_negative ? 1 : 0; + if (posn == length) { + return Status::Invalid( + fmt::format("invalid string: [{}], cannot convert to int128", str)); + } + + Decimal::uint128_t magnitude = 0; + Decimal::uint128_t max_magnitude = (static_cast(1) << 127) - 1; + if (is_negative) { + max_magnitude += 1; + } + while (posn < length) { + size_t group = std::min(18ul, length - posn); + for (size_t i = 0; i < group; ++i) { + if (!std::isdigit(static_cast(str[posn + i]))) { + return Status::Invalid( + fmt::format("invalid string: [{}], cannot convert to int128", str)); } - ret *= multiple; - ret += chunk; - posn += group; } - if (is_negative) { - ret = -ret; + uint64_t chunk = std::stoull(str.substr(posn, group)); + uint64_t multiple = 1; + for (size_t i = 0; i < group; ++i) { + multiple *= 10; + } + if (magnitude > (max_magnitude - chunk) / multiple) { + return Status::Invalid( + fmt::format("invalid string: [{}], cannot convert to int128", str)); + } + magnitude = magnitude * multiple + chunk; + posn += group; + } + if (is_negative) { + if (magnitude == max_magnitude) { + auto max_value = + static_cast((static_cast(1) << 127) - 1); + return -max_value - 1; } + return -static_cast(magnitude); } - return ret; + return static_cast(magnitude); } catch (...) { return Status::Invalid(fmt::format("invalid string: [{}], cannot convert to int128", str)); } diff --git a/src/paimon/common/utils/field_type_utils_test.cpp b/src/paimon/common/utils/field_type_utils_test.cpp index 7c2b237a..50f60237 100644 --- a/src/paimon/common/utils/field_type_utils_test.cpp +++ b/src/paimon/common/utils/field_type_utils_test.cpp @@ -102,8 +102,7 @@ TEST(FieldTypeUtilsTest, ConvertToFieldType) { ASSERT_EQ(result, FieldType::STRUCT); // Test unsupported Arrow type - ASSERT_NOK(FieldTypeUtils::ConvertToFieldType( - static_cast(999))); // Invalid Arrow type + ASSERT_NOK(FieldTypeUtils::ConvertToFieldType(arrow::Type::type::UINT8)); } // Test case: FieldTypeToString @@ -127,8 +126,8 @@ TEST(FieldTypeUtilsTest, FieldTypeToString) { ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::STRUCT), "STRUCT"); // Test UNKNOWN type - auto unknown_type = static_cast(999); - ASSERT_EQ(FieldTypeUtils::FieldTypeToString(unknown_type), "UNKNOWN, type id:999"); + auto unknown_type = static_cast(128); + ASSERT_EQ(FieldTypeUtils::FieldTypeToString(unknown_type), "UNKNOWN, type id:128"); } } // namespace paimon::test diff --git a/src/paimon/core/bucket/hive_bucket_function.cpp b/src/paimon/core/bucket/hive_bucket_function.cpp index 72f0f006..c78c7947 100644 --- a/src/paimon/core/bucket/hive_bucket_function.cpp +++ b/src/paimon/core/bucket/hive_bucket_function.cpp @@ -71,15 +71,15 @@ Result> HiveBucketFunction::Create( } int32_t HiveBucketFunction::Bucket(const BinaryRow& row, int32_t num_buckets) const { - static constexpr int32_t SEED = 0; - int32_t hash = SEED; + static constexpr uint32_t SEED = 0; + uint32_t hash = SEED; for (int32_t i = 0; i < row.GetFieldCount(); i++) { - hash = (31 * hash) + ComputeHash(row, i); + hash = 31U * hash + ComputeHash(row, i); } - return Mod(hash & std::numeric_limits::max(), num_buckets); + return Mod(static_cast(hash & 0x7FFFFFFF), num_buckets); } -int32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_index) const { +uint32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_index) const { if (row.IsNullAt(field_index)) { return 0; } @@ -89,17 +89,17 @@ int32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_inde case FieldType::BOOLEAN: return HiveHasher::HashInt(row.GetBoolean(field_index) ? 1 : 0); case FieldType::TINYINT: - return HiveHasher::HashInt(static_cast(row.GetByte(field_index))); + return HiveHasher::HashInt(static_cast(row.GetByte(field_index))); case FieldType::SMALLINT: - return HiveHasher::HashInt(static_cast(row.GetShort(field_index))); + return HiveHasher::HashInt(static_cast(row.GetShort(field_index))); case FieldType::INT: case FieldType::DATE: - return HiveHasher::HashInt(row.GetInt(field_index)); + return HiveHasher::HashInt(static_cast(row.GetInt(field_index))); case FieldType::BIGINT: - return HiveHasher::HashLong(row.GetLong(field_index)); + return HiveHasher::HashLong(static_cast(row.GetLong(field_index))); case FieldType::FLOAT: { float float_value = row.GetFloat(field_index); - int32_t bits; + uint32_t bits; if (float_value == -0.0f) { bits = 0; } else { @@ -109,9 +109,9 @@ int32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_inde } case FieldType::DOUBLE: { double double_value = row.GetDouble(field_index); - int64_t bits; + uint64_t bits; if (double_value == -0.0) { - bits = 0L; + bits = 0; } else { std::memcpy(&bits, &double_value, sizeof(bits)); } diff --git a/src/paimon/core/bucket/hive_bucket_function.h b/src/paimon/core/bucket/hive_bucket_function.h index 82f0c40d..7b742634 100644 --- a/src/paimon/core/bucket/hive_bucket_function.h +++ b/src/paimon/core/bucket/hive_bucket_function.h @@ -55,7 +55,7 @@ class HiveBucketFunction : public BucketFunction { explicit HiveBucketFunction(const std::vector& field_infos); /// Compute the Hive hash for a single field value. - int32_t ComputeHash(const BinaryRow& row, int32_t field_index) const; + uint32_t ComputeHash(const BinaryRow& row, int32_t field_index) const; /// Mod operation that always returns non-negative result. static int32_t Mod(int32_t value, int32_t divisor); diff --git a/src/paimon/core/bucket/hive_bucket_function_test.cpp b/src/paimon/core/bucket/hive_bucket_function_test.cpp index 73a94c5a..b6c948c5 100644 --- a/src/paimon/core/bucket/hive_bucket_function_test.cpp +++ b/src/paimon/core/bucket/hive_bucket_function_test.cpp @@ -119,12 +119,12 @@ TEST_F(HiveBucketFunctionTest, TestHiveBucketFunction) { // Verify individual hash components: // HiveHasher.hashBytes("hello") = 99162322 - ASSERT_EQ(99162322, HiveHasher::HashBytes("hello", 5)); + ASSERT_EQ(99162322U, HiveHasher::HashBytes("hello", 5)); // HiveHasher.hashBytes({1,2,3}) = 1026 - ASSERT_EQ(1026, HiveHasher::HashBytes("\x01\x02\x03", 3)); + ASSERT_EQ(1026U, HiveHasher::HashBytes("\x01\x02\x03", 3)); // BigDecimal("12.34").hashCode() = 1234 * 31 + 2 = 38256 // (After normalizing "12.3400" -> "12.34", unscaled=1234, scale=2) - ASSERT_EQ(38256, HiveHasher::HashDecimal(Decimal::FromUnscaledLong(123400, 10, 4))); + ASSERT_EQ(38256U, HiveHasher::HashDecimal(Decimal::FromUnscaledLong(123400, 10, 4))); // expectedHash = 31*(31*(31*7 + 99162322) + 1026) + 38256 = 805989529 (with int32 overflow) // bucket = (805989529 & INT32_MAX) % 8 = 1 diff --git a/src/paimon/core/bucket/hive_hasher.h b/src/paimon/core/bucket/hive_hasher.h index d86be406..fad87a1a 100644 --- a/src/paimon/core/bucket/hive_hasher.h +++ b/src/paimon/core/bucket/hive_hasher.h @@ -30,21 +30,21 @@ namespace paimon { /// hash implementation, ensuring consistent bucket assignment between Paimon C++ and Java. class HiveHasher { public: - /// Hash an int value (identity function, same as Hive). - static int32_t HashInt(int32_t input) { + /// Hash an int value (identity function, same as Hive's 32-bit int hash). + static uint32_t HashInt(uint32_t input) { return input; } /// Hash a long value (same as Java's Long.hashCode). - static int32_t HashLong(int64_t input) { - return static_cast(input ^ (static_cast(input) >> 32)); + static uint32_t HashLong(uint64_t input) { + return static_cast(input ^ (input >> 32)); } /// Hash a byte array. - static int32_t HashBytes(const char* bytes, int32_t length) { - int32_t result = 0; + static uint32_t HashBytes(const char* bytes, int32_t length) { + uint32_t result = 0; for (int32_t i = 0; i < length; i++) { - result = (result * 31) + static_cast(bytes[i]); + result = result * 31U + static_cast(static_cast(bytes[i])); } return result; } @@ -60,7 +60,7 @@ class HiveHasher { /// /// @param decimal The decimal value to normalize. /// @return The hash code of the normalized decimal, computed as Java BigDecimal.hashCode(). - static int32_t HashDecimal(const Decimal& decimal) { + static uint32_t HashDecimal(const Decimal& decimal) { // Java BigDecimal.hashCode() = unscaledValue.intValue() * 31 + scale // For compact decimals (precision <= 18), we can use the long value directly. // For non-compact decimals, we need to handle the 128-bit value. @@ -92,7 +92,9 @@ class HiveHasher { } // Count integer digits - auto abs_value = value < 0 ? -value : value; + auto abs_value = + value < 0 ? static_cast(0) - static_cast(value) + : static_cast(value); int32_t total_digits = 0; auto temp = abs_value; while (temp > 0) { @@ -148,8 +150,8 @@ class HiveHasher { // Compute Java BigDecimal.hashCode(): // hashCode = intValue(unscaledValue) * 31 + scale // intValue() returns the low 32 bits of the value - auto int_value = static_cast(static_cast(value)); - return int_value * 31 + scale; + auto int_value = static_cast(value); + return int_value * 31U + static_cast(scale); } private: diff --git a/src/paimon/format/orc/orc_adapter.cpp b/src/paimon/format/orc/orc_adapter.cpp index b4f28fb6..4e58da86 100644 --- a/src/paimon/format/orc/orc_adapter.cpp +++ b/src/paimon/format/orc/orc_adapter.cpp @@ -639,19 +639,26 @@ Result> MakeOrcBackedTimestampBuilder( : nullptr; const int64_t* seconds = typed_batch->data.data(); const int64_t* nanos = typed_batch->nanoseconds.data(); + const bool has_nulls = typed_batch->hasNulls; + const auto* not_null = typed_batch->notNull.data(); + auto is_null = [has_nulls, not_null](int64_t index) { return has_nulls && !not_null[index]; }; auto timestamp_type = arrow::internal::checked_pointer_cast(type); assert(timestamp_type); int32_t precision = DateTimeUtils::GetPrecisionFromType(timestamp_type); // TODO(lisizhuo.lsz): check nano overflow in arrow if (precision == Timestamp::MIN_PRECISION) { auto transform_iter = arrow::internal::MakeLazyRange( - [seconds](int64_t index) { return seconds[index]; }, typed_batch->numElements); + [seconds, is_null](int64_t index) { return is_null(index) ? 0 : seconds[index]; }, + typed_batch->numElements); PAIMON_RETURN_NOT_OK_FROM_ARROW( builder->AppendValues(transform_iter.begin(), transform_iter.end(), valid_bytes)); return builder; } else if (precision == Timestamp::MILLIS_PRECISION) { auto transform_iter = arrow::internal::MakeLazyRange( - [seconds, nanos](int64_t index) { + [seconds, nanos, is_null](int64_t index) { + if (is_null(index)) { + return int64_t{0}; + } return seconds[index] * DateTimeUtils::CONVERSION_FACTORS[DateTimeUtils::TimeType::MILLISECOND] + nanos[index] / @@ -663,7 +670,10 @@ Result> MakeOrcBackedTimestampBuilder( return builder; } else if (precision == Timestamp::DEFAULT_PRECISION) { auto transform_iter = arrow::internal::MakeLazyRange( - [seconds, nanos](int64_t index) { + [seconds, nanos, is_null](int64_t index) { + if (is_null(index)) { + return int64_t{0}; + } return seconds[index] * DateTimeUtils::CONVERSION_FACTORS[DateTimeUtils::TimeType::MICROSECOND] + nanos[index] / @@ -675,7 +685,10 @@ Result> MakeOrcBackedTimestampBuilder( return builder; } else if (precision == Timestamp::MAX_PRECISION) { auto transform_iter = arrow::internal::MakeLazyRange( - [seconds, nanos](int64_t index) { + [seconds, nanos, is_null](int64_t index) { + if (is_null(index)) { + return int64_t{0}; + } return seconds[index] * DateTimeUtils::CONVERSION_FACTORS[DateTimeUtils::TimeType::NANOSECOND] + nanos[index]; From 06a9f020a4bc865b138e5020432ff07d3e2fa7e7 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Wed, 10 Jun 2026 10:23:55 +0800 Subject: [PATCH 043/138] fix(build): make package config relocatable --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 64d6044c..30c2474d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -482,3 +482,7 @@ install(EXPORT PaimonTargets if(PAIMON_BUILD_BENCHMARKS) add_subdirectory(benchmark) endif() + +install(EXPORT PaimonTargets + NAMESPACE Paimon:: + DESTINATION ${PAIMON_CMAKE_INSTALL_DIR}) From 0e2108bf61cf85ab22f9b5c7ad143b9bed7bc190 Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Wed, 10 Jun 2026 11:51:32 +0800 Subject: [PATCH 044/138] fix(parquet): fix incorrect chunk_end calculation in ComputePageRanges when dictionary page is present --- .../page_filtered_row_group_reader.cpp | 10 +- .../page_filtered_row_group_reader_test.cpp | 150 +++++++++++++++++- 2 files changed, 154 insertions(+), 6 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index d44c11b4..2e9d9b36 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -319,18 +319,20 @@ std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRange for (int32_t col_idx : column_indices) { auto col_chunk = rg_metadata->ColumnChunk(col_idx); int64_t data_page_offset = col_chunk->data_page_offset(); - int64_t total_compressed_size = col_chunk->total_compressed_size(); - int64_t chunk_end = data_page_offset + total_compressed_size; - + int64_t data_page_compressed_size = col_chunk->total_compressed_size(); // Dictionary page: always include if present if (col_chunk->has_dictionary_page()) { int64_t dict_offset = col_chunk->dictionary_page_offset(); int64_t dict_size = data_page_offset - dict_offset; if (dict_size > 0) { + // if dictionary exists, the data page size should be reduced by the dictionary + data_page_compressed_size -= dict_size; ranges.push_back({dict_offset, dict_size}); } } + int64_t chunk_end = data_page_offset + data_page_compressed_size; + // Try to get OffsetIndex for page-level ranges std::shared_ptr<::parquet::OffsetIndex> offset_index; if (rg_page_index_reader) { @@ -339,7 +341,7 @@ std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRange if (!offset_index) { // No OffsetIndex: fall back to entire column chunk - ranges.push_back({data_page_offset, total_compressed_size}); + ranges.push_back({data_page_offset, data_page_compressed_size}); continue; } diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 87fe7349..11cb0ddb 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -77,7 +77,8 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { /// @param max_row_group_length Controls row group size void WriteTestFile(const std::string& file_name, const std::shared_ptr& struct_array, - int32_t write_batch_size, int64_t max_row_group_length) { + int32_t write_batch_size, int64_t max_row_group_length, + bool enable_dictionary = false) { auto data_type = struct_array->struct_type(); auto data_schema = arrow::schema(data_type->fields()); auto data_arrow_array = std::make_unique(); @@ -87,7 +88,11 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { ::parquet::WriterProperties::Builder builder; builder.write_batch_size(write_batch_size); builder.max_row_group_length(max_row_group_length); - builder.disable_dictionary(); // Ensure page index min/max are meaningful + if (enable_dictionary) { + builder.enable_dictionary(); + } else { + builder.disable_dictionary(); // Ensure page index min/max are meaningful + } builder.enable_write_page_index(); // Enable page index for page-level filtering // Set data page size to 1 byte to force a new page after every write_batch_size rows. // The writer flushes a page when accumulated data exceeds data_pagesize, so setting @@ -722,4 +727,145 @@ TEST_F(PageFilteredRowGroupReaderTest, EndToEndPageLevelPreBuffer) { ASSERT_EQ(10, offset); } +/// Test: ComputePageRanges with dictionary encoding produces correct chunk_end. +/// +/// When dictionary encoding is enabled, the column chunk layout is: +/// [Dictionary Page] [Data Page 0] [Data Page 1] ... [Data Page N] +/// And total_compressed_size covers the entire chunk starting from dictionary_page_offset. +/// +/// The bug: chunk_end = data_page_offset + total_compressed_size is wrong because +/// total_compressed_size already includes the dictionary page size. The correct +/// chunk_end should be dictionary_page_offset + total_compressed_size. +/// +/// This test verifies that: +/// 1. No range exceeds the true chunk boundary (overshoot regression). +/// 2. At least one non-dictionary data-page range is present (not truncated). +/// 3. The maximum range_end equals true_chunk_end when requesting all rows. +/// 4. End-to-end reads with page-level filtering return correct query results. +TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding) { + std::string file_name = dir_->Str() + "/compute_ranges_dict.parquet"; + + // Use low-cardinality data to ensure dictionary encoding is actually used. + // 100 rows with values cycling through 0..9 → dictionary will have 10 entries. + arrow::Int32Builder val_builder; + ASSERT_TRUE(val_builder.Reserve(100).ok()); + for (int32_t i = 0; i < 100; ++i) { + val_builder.UnsafeAppend(i % 10); + } + auto val_array = val_builder.Finish().ValueOrDie(); + auto field = arrow::field("val", arrow::int32()); + auto struct_array = arrow::StructArray::Make({val_array}, {field}).ValueOrDie(); + + // Write with dictionary encoding enabled and 1 row per page. + // Each page has min==max==val for that row, enabling precise page-level skipping. + WriteTestFile(file_name, struct_array, /*write_batch_size=*/1, + /*max_row_group_length=*/100, /*enable_dictionary=*/true); + + // Open the file and verify metadata confirms dictionary page presence + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); + ASSERT_TRUE(parquet_reader); + + auto file_metadata = parquet_reader->metadata(); + auto rg_metadata = file_metadata->RowGroup(0); + auto col_chunk = rg_metadata->ColumnChunk(0); + + // Precondition: dictionary page must exist for this test to be meaningful + ASSERT_TRUE(col_chunk->has_dictionary_page()); + + int64_t dict_offset = col_chunk->dictionary_page_offset(); + int64_t data_page_offset = col_chunk->data_page_offset(); + int64_t total_compressed_size = col_chunk->total_compressed_size(); + + // The true chunk end is dict_offset + total_compressed_size + int64_t true_chunk_end = dict_offset + total_compressed_size; + // The buggy chunk end would be data_page_offset + total_compressed_size + int64_t buggy_chunk_end = data_page_offset + total_compressed_size; + + // Sanity: dict page is before data pages, so buggy end > true end + ASSERT_LT(dict_offset, data_page_offset); + ASSERT_GT(buggy_chunk_end, true_chunk_end); + // Now call ComputePageRanges with all rows matching + RowRanges row_ranges; + row_ranges.Add(RowRanges::Range(0, 99)); + + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + parquet_reader.get(), /*row_group_index=*/0, row_ranges, /*column_indices=*/{0}); + + ASSERT_FALSE(ranges.empty()); + + // --- Check 1: No range should extend beyond the true chunk end --- + // With the bug, the last data page's range would use chunk_end = data_page_offset + + // total_compressed_size, which overshoots by the dictionary page size. + for (auto& range : ranges) { + int64_t range_end = range.offset + range.length; + ASSERT_LE(range_end, true_chunk_end); + } + + // --- Check 2: At least one non-dictionary data-page range is present --- + // Guards against truncation: if only the dictionary range is returned, the test + // would still pass the overshoot check but miss that data pages are lost. + int data_page_range_count = 0; + for (const auto& range : ranges) { + if (range.offset >= data_page_offset) { + ++data_page_range_count; + } + } + ASSERT_GE(data_page_range_count, 1); + + // --- Check 3: Maximum range_end equals true_chunk_end when requesting all rows --- + int64_t max_range_end = 0; + for (const auto& range : ranges) { + int64_t range_end = range.offset + range.length; + max_range_end = std::max(max_range_end, range_end); + } + ASSERT_EQ(max_range_end, true_chunk_end); + + // --- Check 4: No range exceeds file size --- + for (const auto& range : ranges) { + ASSERT_LE(range.offset + range.length, static_cast(length)); + } + + // --- End-to-end check 1: read all rows (no predicate filtering) --- + // Verifies that reading a dictionary-encoded file with page index enabled + // returns all 100 rows with correct values. + auto read_schema = arrow::schema({field}); + auto predicate_all = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(0)); + std::shared_ptr result_all; + ReadWithPredicateImpl(file_name, read_schema, predicate_all, &result_all); + ASSERT_TRUE(result_all); + ASSERT_EQ(100, result_all->length()); + + // --- End-to-end check 2: full range query with page level skipping --- + // Build expected array: val = i % 10 for i in [0, 100), wrapped in a struct. + // Concatenate all chunks and compare with Equals + auto actual_struct_arr = arrow::Concatenate(result_all->chunks()).ValueOrDie(); + ASSERT_TRUE(actual_struct_arr->Equals(struct_array)); + + // --- End-to-end check 3: partial-row query with page-level skipping --- + // Predicate val >= 7 skips pages where val < 7, keeping only val in {7,8,9}. + // Out of 100 rows, 30 rows satisfy val >= 7 (3 per cycle × 10 cycles). + auto predicate_partial = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(7)); + std::shared_ptr result_partial; + ReadWithPredicateImpl(file_name, read_schema, predicate_partial, &result_partial); + ASSERT_TRUE(result_partial); + + // Build expected StructArray and compare with Equals + arrow::Int32Builder expected_builder; + ASSERT_TRUE(expected_builder.Reserve(30).ok()); + for (int32_t i = 0; i < 100; ++i) { + if (i % 10 >= 7) { + expected_builder.UnsafeAppend(i % 10); + } + } + auto expected_array = expected_builder.Finish().ValueOrDie(); + auto expected_struct = arrow::StructArray::Make({expected_array}, {field}).ValueOrDie(); + auto partial_concat = arrow::Concatenate(result_partial->chunks()).ValueOrDie(); + ASSERT_TRUE(partial_concat->Equals(expected_struct)); +} + } // namespace paimon::parquet::test From 92ff6cb1e307bd7f7af212f266144d56be7739dd Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:19:15 +0800 Subject: [PATCH 045/138] feat(fs): support int64 file IO sizes --- include/paimon/fs/file_system.h | 12 +- include/paimon/io/buffered_input_stream.h | 22 ++-- include/paimon/io/byte_array_input_stream.h | 12 +- include/paimon/io/data_input_stream.h | 8 +- include/paimon/memory/bytes.h | 2 +- src/paimon/common/data/blob.cpp | 16 ++- src/paimon/common/data/blob_test.cpp | 4 +- src/paimon/common/data/decimal_test.cpp | 2 +- .../bitmap/bitmap_file_index_test.cpp | 2 +- .../bloomfilter/bloom_filter_file_index.cpp | 2 +- .../bsi/bit_slice_index_bitmap_file_index.cpp | 2 +- .../rangebitmap/bit_slice_index_bitmap.cpp | 1 + .../rangebitmap/range_bitmap_io_test.cpp | 2 +- src/paimon/common/fs/file_system.cpp | 14 ++- src/paimon/common/fs/file_system_test.cpp | 51 ++++---- .../bitmap/bitmap_global_index_test.cpp | 1 - .../btree/btree_compatibility_test.cpp | 5 +- .../btree_global_index_integration_test.cpp | 2 +- .../btree/lazy_filtered_btree_reader_test.cpp | 2 +- .../range_bitmap_global_index_test.cpp | 1 - .../wrap/file_index_writer_wrapper.h | 22 +--- .../common/io/buffered_input_stream.cpp | 41 ++++--- .../common/io/byte_array_input_stream.cpp | 40 +++--- src/paimon/common/io/cache_input_stream.h | 25 +++- .../common/io/cache_input_stream_test.cpp | 10 +- src/paimon/common/io/data_input_stream.cpp | 29 ++--- src/paimon/common/io/data_output_stream.cpp | 10 +- src/paimon/common/io/data_output_stream.h | 6 +- src/paimon/common/io/offset_input_stream.cpp | 68 ++++++----- src/paimon/common/io/offset_input_stream.h | 13 +- .../common/io/offset_input_stream_test.cpp | 45 +------ src/paimon/common/memory/bytes.cpp | 3 +- .../common/memory/memory_segment_utils.cpp | 7 +- .../common/memory/memory_segment_utils.h | 4 +- .../prefetch_file_batch_reader_impl_test.cpp | 7 +- src/paimon/common/sst/bloom_filter_handle.h | 7 +- src/paimon/common/sst/sst_file_reader.cpp | 2 +- src/paimon/common/sst/sst_file_utils.h | 2 - src/paimon/common/sst/sst_file_writer.cpp | 1 - .../arrow/arrow_input_stream_adapter.cpp | 38 +++--- .../utils/arrow/arrow_input_stream_adapter.h | 4 +- .../arrow/arrow_output_stream_adapter.cpp | 13 +- .../utils/arrow/arrow_stream_adapter_test.cpp | 2 +- src/paimon/common/utils/math.h | 21 ++++ src/paimon/common/utils/math_test.cpp | 10 ++ src/paimon/common/utils/read_ahead_cache.cpp | 14 +-- src/paimon/common/utils/stream_utils.h | 18 +-- src/paimon/common/utils/stream_utils_test.cpp | 4 +- .../bitmap_deletion_vector.cpp | 8 +- .../deletionvectors/deletion_file_writer.cpp | 10 +- .../deletionvectors/deletion_file_writer.h | 2 - .../deletion_file_writer_test.cpp | 2 +- src/paimon/core/index/index_file.h | 4 +- .../manifest/manifest_committable_test.cpp | 2 +- .../lookup/remote_lookup_file_manager.cpp | 27 +++-- .../remote_lookup_file_manager_test.cpp | 8 +- src/paimon/core/mergetree/spill_reader.cpp | 2 +- src/paimon/core/mergetree/spill_writer.cpp | 2 +- .../core/mergetree/write_buffer_test.cpp | 2 +- .../operation/file_store_commit_impl_test.cpp | 2 +- .../format/avro/avro_file_batch_reader.cpp | 1 + .../format/avro/avro_input_stream_impl.cpp | 15 ++- .../format/avro/avro_output_stream_impl.cpp | 10 +- .../format/blob/blob_file_batch_reader.cpp | 4 +- src/paimon/format/blob/blob_format_writer.cpp | 20 +-- src/paimon/format/blob/blob_format_writer.h | 4 +- .../format/blob/blob_format_writer_test.cpp | 6 +- .../format/orc/orc_input_stream_impl.cpp | 21 +++- .../format/orc/orc_output_stream_impl.cpp | 18 ++- .../parquet/column_index_filter_test.cpp | 2 +- .../parquet/file_reader_wrapper_test.cpp | 2 +- .../page_filtered_row_group_reader_test.cpp | 12 +- .../format/parquet/parquet_reader_builder.h | 2 +- .../parquet/parquet_stats_extractor.cpp | 2 +- .../parquet/predicate_pushdown_test.cpp | 2 +- src/paimon/fs/jindo/jindo_file_status.h | 2 +- src/paimon/fs/jindo/jindo_file_system.cpp | 32 +++-- src/paimon/fs/jindo/jindo_file_system.h | 10 +- .../fs/jindo/jindo_file_system_test.cpp | 6 +- src/paimon/fs/local/local_file.cpp | 48 ++++---- src/paimon/fs/local/local_file.h | 8 +- src/paimon/fs/local/local_file_status.h | 6 +- src/paimon/fs/local/local_file_system.cpp | 22 ++-- src/paimon/fs/local/local_file_system.h | 10 +- src/paimon/fs/local/local_file_test.cpp | 18 +-- .../lucene/lucene_global_index_writer.cpp | 7 +- src/paimon/global_index/lucene/lucene_input.h | 4 +- .../lumina/lumina_file_io_test.cpp | 89 ++++++-------- .../global_index/lumina/lumina_file_reader.h | 114 +++++++----------- .../global_index/lumina/lumina_file_writer.h | 35 +++--- src/paimon/testing/mock/mock_file_system.h | 12 +- .../testing/mock/mock_format_writer.cpp | 4 +- src/paimon/testing/utils/test_helper.h | 4 +- test/inte/blob_table_inte_test.cpp | 7 +- test/inte/read_inte_test.cpp | 10 +- test/inte/write_and_read_inte_test.cpp | 2 +- 96 files changed, 643 insertions(+), 628 deletions(-) diff --git a/include/paimon/fs/file_system.h b/include/paimon/fs/file_system.h index f2388dc3..0a89a1e6 100644 --- a/include/paimon/fs/file_system.h +++ b/include/paimon/fs/file_system.h @@ -75,7 +75,7 @@ class PAIMON_EXPORT InputStream : public Stream { /// @return Result containing the actual number of bytes read on success, or an error status on /// failure. /// @note The stream position advances by the number of bytes actually read. - virtual Result Read(char* buffer, uint32_t size) = 0; + virtual Result Read(char* buffer, int64_t size) = 0; /// Read data from given position in the stream. /// @@ -85,7 +85,7 @@ class PAIMON_EXPORT InputStream : public Stream { /// @param[out] buffer The buffer to store the read content. /// @param size The number of bytes to read. /// @param offset The position in the stream to read from. - virtual Result Read(char* buffer, uint32_t size, uint64_t offset) = 0; + virtual Result Read(char* buffer, int64_t size, int64_t offset) = 0; /// Asynchronously read data from the input stream. /// @@ -100,7 +100,7 @@ class PAIMON_EXPORT InputStream : public Stream { /// @param callback The callback function to be invoked upon completion of the read operation. /// The callback will receive a Status object indicating the success or failure /// of the read operation. - virtual void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + virtual void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) = 0; /// Get an identifier that uniquely identify the underlying content. @@ -109,7 +109,7 @@ class PAIMON_EXPORT InputStream : public Stream { virtual Result GetUri() const = 0; /// Get the total length of the file in bytes. - virtual Result Length() const = 0; + virtual Result Length() const = 0; }; /// Abstract class for output stream operations. @@ -123,7 +123,7 @@ class PAIMON_EXPORT OutputStream : public Stream { /// @return Result containing the actual number of bytes written on success, or an error status /// on failure. /// @note The stream position advances by the number of bytes actually written. - virtual Result Write(const char* buffer, uint32_t size) = 0; + virtual Result Write(const char* buffer, int64_t size) = 0; /// Flush pending data to the disk. virtual Status Flush() = 0; @@ -162,7 +162,7 @@ class PAIMON_EXPORT FileStatus { /// Get the size of the file in bytes. /// @note For directories, this method is undefined behavior. - virtual uint64_t GetLen() const = 0; + virtual int64_t GetLen() const = 0; /// Check if this entry represents a directory. virtual bool IsDir() const = 0; diff --git a/include/paimon/io/buffered_input_stream.h b/include/paimon/io/buffered_input_stream.h index 2af6b731..6b172cb9 100644 --- a/include/paimon/io/buffered_input_stream.h +++ b/include/paimon/io/buffered_input_stream.h @@ -45,7 +45,7 @@ class PAIMON_EXPORT BufferedInputStream : public InputStream { /// @param in The underlying input stream to wrap. /// @param buffer_size Size of the internal buffer in bytes. /// @param pool Memory pool for buffer allocation. - BufferedInputStream(const std::shared_ptr& in, int32_t buffer_size, + BufferedInputStream(const std::shared_ptr& in, int64_t buffer_size, MemoryPool* pool); ~BufferedInputStream() noexcept override; @@ -54,20 +54,20 @@ class PAIMON_EXPORT BufferedInputStream : public InputStream { Result GetPos() const override; - Result Read(char* buffer, uint32_t size) override; + Result Read(char* buffer, int64_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; - Result Length() const override; + Result Length() const override; Status Close() override; Result GetUri() const override; - static constexpr int32_t DEFAULT_BUFFER_SIZE = 8192; + static constexpr int64_t DEFAULT_BUFFER_SIZE = 8192; private: /// Fill the internal buffer from the underlying stream. @@ -75,15 +75,15 @@ class PAIMON_EXPORT BufferedInputStream : public InputStream { /// Internal read implementation. /// @pre size > 0 - Result InnerRead(char* buffer, int32_t size); + Result InnerRead(char* buffer, int64_t size); /// Validate that the expected number of bytes were read. - Status AssertReadLength(int32_t read_length, int32_t actual_read_length) const; + Status AssertReadLength(int64_t read_length, int64_t actual_read_length) const; private: - int32_t buffer_size_; - int32_t pos_ = 0; - int32_t count_ = 0; + int64_t buffer_size_; + int64_t pos_ = 0; + int64_t count_ = 0; std::unique_ptr buffer_; std::shared_ptr in_; }; diff --git a/include/paimon/io/byte_array_input_stream.h b/include/paimon/io/byte_array_input_stream.h index e66c5e8a..ab2d2f19 100644 --- a/include/paimon/io/byte_array_input_stream.h +++ b/include/paimon/io/byte_array_input_stream.h @@ -32,7 +32,7 @@ namespace paimon { /// Input stream for memory buffer, inherits from `InputStream`. class PAIMON_EXPORT ByteArrayInputStream : public InputStream { public: - ByteArrayInputStream(const char* buffer, uint64_t length); + ByteArrayInputStream(const char* buffer, int64_t length); ~ByteArrayInputStream() override = default; /// @return The raw data pointer of current pos. @@ -44,14 +44,14 @@ class PAIMON_EXPORT ByteArrayInputStream : public InputStream { return position_; } - Result Read(char* buffer, uint32_t size) override; + Result Read(char* buffer, int64_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; - Result Length() const override { + Result Length() const override { return length_; } @@ -61,7 +61,7 @@ class PAIMON_EXPORT ByteArrayInputStream : public InputStream { private: const char* buffer_; - const uint64_t length_; + const int64_t length_; int64_t position_; }; } // namespace paimon diff --git a/include/paimon/io/data_input_stream.h b/include/paimon/io/data_input_stream.h index 8dc15a2c..d954003e 100644 --- a/include/paimon/io/data_input_stream.h +++ b/include/paimon/io/data_input_stream.h @@ -58,7 +58,7 @@ class PAIMON_EXPORT DataInputStream { /// Read raw data of specified size from the stream. /// @param data Buffer to store the read data. /// @param size Number of bytes to read. - Status Read(char* data, uint32_t size) const; + Status Read(char* data, int64_t size) const; /// Read string from the stream. /// @note First read length (int16), then read string bytes. @@ -68,7 +68,7 @@ class PAIMON_EXPORT DataInputStream { Result GetPos() const; /// Get the total length of the underlying input stream. - Result Length() const; + Result Length() const; /// Set the byte order for endianness conversion. /// @param order The byte order to use `PAIMON_BIG_ENDIAN` or `PAIMON_LITTLE_ENDIAN`. @@ -80,11 +80,11 @@ class PAIMON_EXPORT DataInputStream { /// Validate that the expected number of bytes were read. /// @param read_length Expected number of bytes to read. /// @param actual_read_length Actual number of bytes read. - Status AssertReadLength(int32_t read_length, int32_t actual_read_length) const; + Status AssertReadLength(int64_t read_length, int64_t actual_read_length) const; /// Check if there are enough bytes available to read. /// @param need_length Number of bytes needed. - Status AssertBoundary(int32_t need_length) const; + Status AssertBoundary(int64_t need_length) const; /// Determine if byte swapping is needed based on current byte order and system endianness. /// @return `true` if byte swapping is required, `false` otherwise. diff --git a/include/paimon/memory/bytes.h b/include/paimon/memory/bytes.h index b6f5280f..5c7fb716 100644 --- a/include/paimon/memory/bytes.h +++ b/include/paimon/memory/bytes.h @@ -81,7 +81,7 @@ class PAIMON_EXPORT Bytes { /// @param length Number of bytes to allocate. /// @param pool Memory pool to use for allocation. /// @return Unique pointer to the newly allocated Bytes object. - static PAIMON_UNIQUE_PTR AllocateBytes(int32_t length, MemoryPool* pool); + static PAIMON_UNIQUE_PTR AllocateBytes(size_t length, MemoryPool* pool); /// Allocate a new Bytes object from string data. /// diff --git a/src/paimon/common/data/blob.cpp b/src/paimon/common/data/blob.cpp index da47a920..0c706aaa 100644 --- a/src/paimon/common/data/blob.cpp +++ b/src/paimon/common/data/blob.cpp @@ -97,8 +97,20 @@ Result> Blob::NewInputStream( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file, fs->Open(impl_->GetDescriptor()->Uri())); - return OffsetInputStream::Create(std::move(file), impl_->GetDescriptor()->Length(), - impl_->GetDescriptor()->Offset()); + int64_t blob_length = impl_->GetDescriptor()->Length(); + int64_t blob_offset = impl_->GetDescriptor()->Offset(); + + PAIMON_ASSIGN_OR_RAISE(int64_t total_length, file->Length()); + if (PAIMON_UNLIKELY(blob_offset > total_length)) { + return Status::Invalid( + fmt::format("offset {} exceed total length {}", blob_offset, total_length)); + } + if (blob_length == -1) { + // blob_length == -1 means it's dynamic length, should read to the end + blob_length = total_length - blob_offset; + } + + return OffsetInputStream::Create(std::move(file), blob_length, blob_offset, total_length); } Result> Blob::ToData(const std::shared_ptr& fs, diff --git a/src/paimon/common/data/blob_test.cpp b/src/paimon/common/data/blob_test.cpp index dbb2401e..aa20c6e5 100644 --- a/src/paimon/common/data/blob_test.cpp +++ b/src/paimon/common/data/blob_test.cpp @@ -119,7 +119,7 @@ TEST_F(BlobTest, TestNewInputStreamWithOffsetAndLength) { ASSERT_OK_AND_ASSIGN(auto input_stream, blob->NewInputStream(file_system_)); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(uint64_t length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, input_stream->Length()); ASSERT_EQ(6, length); // Test reading with offset and length applied @@ -136,7 +136,7 @@ TEST_F(BlobTest, TestNewInputStreamWithDynamicLength) { ASSERT_OK_AND_ASSIGN(auto input_stream, blob->NewInputStream(file_system_)); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(uint64_t length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, input_stream->Length()); ASSERT_EQ(12, length); // Test reading from offset to end (should read "cdefghijklmn") diff --git a/src/paimon/common/data/decimal_test.cpp b/src/paimon/common/data/decimal_test.cpp index a37a6bef..f8931b10 100644 --- a/src/paimon/common/data/decimal_test.cpp +++ b/src/paimon/common/data/decimal_test.cpp @@ -100,7 +100,7 @@ TEST(DecimalTest, TestCompatibleWithJava) { auto pool = GetDefaultPool(); auto file_system = std::make_unique(); auto file_name = paimon::test::GetDataDir() + "/decimal_bytes.data"; - uint64_t file_length = file_system->GetFileStatus(file_name).value()->GetLen(); + int64_t file_length = file_system->GetFileStatus(file_name).value()->GetLen(); ASSERT_GT(file_length, 0); ASSERT_OK_AND_ASSIGN(auto input_stream, file_system->Open(file_name)); auto data_bytes = Bytes::AllocateBytes(file_length, pool.get()); diff --git a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp index cbe92a8b..eccbb1f3 100644 --- a/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp +++ b/src/paimon/common/file_index/bitmap/bitmap_file_index_test.cpp @@ -625,7 +625,7 @@ TEST_F(BitmapIndexTest, TestHighCardinalityForCompatibility) { auto file_system = std::make_unique(); ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(index_file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, input_stream->Length()); BitmapFileIndex file_index({}); ASSERT_OK_AND_ASSIGN(auto reader, file_index.CreateReader(CreateArrowSchema(type).get(), diff --git a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp index 81bcf04d..ca33c696 100644 --- a/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp +++ b/src/paimon/common/file_index/bloomfilter/bloom_filter_file_index.cpp @@ -47,7 +47,7 @@ Result> BloomFilterFileIndex::CreateReader( PAIMON_RETURN_NOT_OK(input_stream->Seek(start, SeekOrigin::FS_SEEK_SET)); auto bytes = std::make_shared(length, pool.get()); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, input_stream->Read(bytes->data(), bytes->size())); if (static_cast(actual_read_len) != bytes->size()) { return Status::Invalid( diff --git a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp index aab38648..aca978bd 100644 --- a/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp +++ b/src/paimon/common/file_index/bsi/bit_slice_index_bitmap_file_index.cpp @@ -67,7 +67,7 @@ Result> BitSliceIndexBitmapFileIndex::CreateRea PAIMON_RETURN_NOT_OK(input_stream->Seek(start, SeekOrigin::FS_SEEK_SET)); auto bytes = std::make_unique(length, pool.get()); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, input_stream->Read(bytes->data(), bytes->size())); if (static_cast(actual_read_len) != bytes->size()) { return Status::Invalid( diff --git a/src/paimon/common/file_index/rangebitmap/bit_slice_index_bitmap.cpp b/src/paimon/common/file_index/rangebitmap/bit_slice_index_bitmap.cpp index 18f71076..5743bebc 100644 --- a/src/paimon/common/file_index/rangebitmap/bit_slice_index_bitmap.cpp +++ b/src/paimon/common/file_index/rangebitmap/bit_slice_index_bitmap.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "paimon/common/io/memory_segment_output_stream.h" diff --git a/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp b/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp index 51da76a7..e2bb867e 100644 --- a/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp +++ b/src/paimon/common/file_index/rangebitmap/range_bitmap_io_test.cpp @@ -105,7 +105,7 @@ TEST_F(RangeBitmapIoTest, TestSimple) { ASSERT_OK_AND_ASSIGN(std::shared_ptr out, fs_->Create(file_path, /*overwrite=*/false)); ASSERT_OK_AND_ASSIGN( - int32_t write_len, + int64_t write_len, out->Write(reinterpret_cast(serialized_bytes->data()), serialized_bytes->size())); ASSERT_EQ(write_len, serialized_bytes->size()); ASSERT_OK(out->Flush()); diff --git a/src/paimon/common/fs/file_system.cpp b/src/paimon/common/fs/file_system.cpp index 23895fc1..a5ae3ce8 100644 --- a/src/paimon/common/fs/file_system.cpp +++ b/src/paimon/common/fs/file_system.cpp @@ -22,6 +22,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" @@ -50,10 +51,11 @@ Status FileSystem::ReadFile(const std::string& path, std::string* content) { Status s = in->Close(); (void)s; }); - PAIMON_ASSIGN_OR_RAISE(uint64_t length, in->Length()); - content->resize(length); - PAIMON_ASSIGN_OR_RAISE(int32_t read_length, in->Read(content->data(), length)); - if (read_length != static_cast(length)) { + PAIMON_ASSIGN_OR_RAISE(int64_t length, in->Length()); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "read length")); + content->resize(static_cast(length)); + PAIMON_ASSIGN_OR_RAISE(int64_t read_length, in->Read(content->data(), length)); + if (read_length != length) { return Status::IOError(fmt::format("path {}, expect read len {}, actual read len {}", path, length, read_length)); } @@ -71,8 +73,8 @@ Status FileSystem::WriteFile(const std::string& path, const std::string& content Status s = out->Close(); (void)s; }); - int32_t length = content.size(); - PAIMON_ASSIGN_OR_RAISE(int32_t write_length, out->Write(content.data(), length)); + auto length = static_cast(content.size()); + PAIMON_ASSIGN_OR_RAISE(int64_t write_length, out->Write(content.data(), length)); if (write_length != length) { return Status::IOError(fmt::format("path {}, expect write len {}, actual write len {}", path, length, write_length)); diff --git a/src/paimon/common/fs/file_system_test.cpp b/src/paimon/common/fs/file_system_test.cpp index b7188c4b..d6e14d6d 100644 --- a/src/paimon/common/fs/file_system_test.cpp +++ b/src/paimon/common/fs/file_system_test.cpp @@ -94,7 +94,7 @@ class FileSystemTest : public ::testing::Test, public ::testing::WithParamInterf fs_->Create(file, /*overwrite=*/true)); std::string input = "paimon"; char chars[8] = {1, 2, 3, 4, 5, 6, 7, 8}; - ASSERT_OK_AND_ASSIGN(int32_t size, out->Write(chars, input.size())); + ASSERT_OK_AND_ASSIGN(int64_t size, out->Write(chars, input.size())); ASSERT_EQ(size, input.size()); ASSERT_OK(out->Flush()); ASSERT_OK(out->Close()); @@ -210,7 +210,7 @@ TEST_P(FileSystemTest, TestCreate) { ASSERT_OK_AND_ASSIGN(std::unique_ptr out, fs_->Create(path, /*overwrite=*/true)); ASSERT_TRUE(out); std::string input = "paimon"; - ASSERT_OK_AND_ASSIGN(int32_t size, out->Write(input.data(), input.size())); + ASSERT_OK_AND_ASSIGN(int64_t size, out->Write(input.data(), input.size())); ASSERT_EQ(size, input.size()); ASSERT_OK(out->Close()); @@ -225,7 +225,7 @@ TEST_P(FileSystemTest, TestCreateRelativeFileInCurrentDirectory) { std::string path = "relative_file_" + RandomName(); ASSERT_OK_AND_ASSIGN(auto out, fs_->Create(path, /*overwrite=*/true)); std::string content = "content"; - ASSERT_OK_AND_ASSIGN(int32_t write_len, out->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK_AND_ASSIGN(std::string uri, out->GetUri()); ASSERT_FALSE(uri.empty()); @@ -245,7 +245,7 @@ TEST_P(FileSystemTest, TestSimpleWriteAndRead) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK(out_stream->Flush()); @@ -263,7 +263,7 @@ TEST_P(FileSystemTest, TestSimpleWriteAndRead) { // read from cur pos std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -275,7 +275,7 @@ TEST_P(FileSystemTest, TestSimpleWriteAndRead) { ASSERT_OK_AND_ASSIGN(uri, in_stream->GetUri()); ASSERT_EQ(uri, file_path); - ASSERT_OK_AND_ASSIGN(uint64_t file_len, in_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t file_len, in_stream->Length()); ASSERT_EQ(file_len, content.size()); ASSERT_OK_AND_ASSIGN(pos, in_stream->GetPos()); @@ -290,7 +290,7 @@ TEST_P(FileSystemTest, TestWriteMultipleTimes) { // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); for (const auto& str : content_vec) { - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(str.data(), str.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(str.data(), str.size())); ASSERT_EQ(write_len, str.size()); } ASSERT_OK(out_stream->Flush()); @@ -301,7 +301,7 @@ TEST_P(FileSystemTest, TestWriteMultipleTimes) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, fs_->Open(file_path)); std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -312,7 +312,7 @@ TEST_P(FileSystemTest, TestWriteInNotExistDir) { // write process std::string content = "abcdefghijk"; ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] int32_t write_len, + ASSERT_OK_AND_ASSIGN([[maybe_unused]] int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -320,7 +320,7 @@ TEST_P(FileSystemTest, TestWriteInNotExistDir) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, fs_->Open(file_path)); std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -334,7 +334,7 @@ TEST_P(FileSystemTest, TestWriteEmptyFile) { // write process std::string content = ""; ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, 0); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -351,7 +351,7 @@ TEST_P(FileSystemTest, TestWriteEmptyFile) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, fs_->Open(file_path)); std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -362,7 +362,7 @@ TEST_P(FileSystemTest, TestWriteWithOverwrite) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -379,7 +379,7 @@ TEST_P(FileSystemTest, TestWriteWithOverwrite) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, fs_->Open(file_path)); std::string read_content(new_content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(new_content, read_content); @@ -395,7 +395,7 @@ TEST_P(FileSystemTest, TestAsyncRead) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] int32_t write_len, + ASSERT_OK_AND_ASSIGN([[maybe_unused]] int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -428,7 +428,7 @@ TEST_P(FileSystemTest, TestInvalidRead) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] int32_t write_len, + ASSERT_OK_AND_ASSIGN([[maybe_unused]] int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -441,7 +441,7 @@ TEST_P(FileSystemTest, TestInvalidRead) { // read from cur pos std::string read_content(3, '\0'); ASSERT_NOK(in_stream->Read(read_content.data(), read_content.size())); - ASSERT_OK_AND_ASSIGN(size_t actual_read, in_stream->Read(read_content.data(), 0)); + ASSERT_OK_AND_ASSIGN(int64_t actual_read, in_stream->Read(read_content.data(), 0)); ASSERT_EQ(actual_read, 0); } { @@ -465,7 +465,7 @@ TEST_P(FileSystemTest, TestInvalidAsyncRead) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN([[maybe_unused]] int32_t write_len, + ASSERT_OK_AND_ASSIGN([[maybe_unused]] int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -567,7 +567,7 @@ TEST_P(FileSystemTest, TestSeek) { std::string path = PathUtil::JoinPath(test_root_, "/test_file"); ASSERT_OK_AND_ASSIGN(std::unique_ptr out, fs_->Create(path, /*overwrite=*/true)); std::string input = "paimon"; - ASSERT_OK_AND_ASSIGN(int32_t size, out->Write(input.data(), input.size())); + ASSERT_OK_AND_ASSIGN(int64_t size, out->Write(input.data(), input.size())); ASSERT_EQ(size, input.size()); ASSERT_OK(out->Close()); @@ -585,7 +585,7 @@ TEST_P(FileSystemTest, TestSeek) { ASSERT_EQ(pos3, 1); std::string read_content(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, in->Read(read_content.data(), read_content.size())); + ASSERT_OK_AND_ASSIGN(int64_t read_len, in->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ("aim", read_content); @@ -604,7 +604,7 @@ TEST_P(FileSystemTest, TestSeek2) { std::string file_path = test_root_ + "/file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -629,7 +629,7 @@ TEST_P(FileSystemTest, TestSeek2) { // read from cur pos std::string read_content(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ("ijk", read_content); @@ -651,14 +651,14 @@ TEST_P(FileSystemTest, TestRename) { ASSERT_OK_AND_ASSIGN(std::unique_ptr out, fs_->Create(path, /*overwrite=*/true)); ASSERT_TRUE(out); std::string input = "paimon"; - ASSERT_OK_AND_ASSIGN(int32_t size, out->Write(input.data(), input.size())); + ASSERT_OK_AND_ASSIGN(int64_t size, out->Write(input.data(), input.size())); ASSERT_EQ(size, input.size()); ASSERT_OK(out->Flush()); ASSERT_OK(out->Close()); ASSERT_OK_AND_ASSIGN(std::unique_ptr in, fs_->Open(path)); ASSERT_TRUE(in); char* data = new char[input.size() * 2]; - ASSERT_OK_AND_ASSIGN(int32_t size_read, in->Read(data, input.size(), /*offset=*/0)); + ASSERT_OK_AND_ASSIGN(int64_t size_read, in->Read(data, input.size(), /*offset=*/0)); ASSERT_EQ(size_read, input.size()); std::string read_data(data, input.size()); ASSERT_EQ(read_data, input); @@ -1134,6 +1134,9 @@ TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameName) { // test for create multi dir such as "partition1" and "partition1" (relative path) TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameNameWithRelativePath) { + if (GetParam() == "jindo") { + GTEST_SKIP() << "skip jindo for relative path test"; + } uint32_t runs_count = 10; uint32_t thread_count = 10; auto executor = CreateDefaultExecutor(thread_count); diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp b/src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp index 3e222f6e..f19c294c 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index_test.cpp @@ -74,7 +74,6 @@ class BitmapGlobalIndexTest : public ::testing::Test { auto wrapper = std::dynamic_pointer_cast(global_writer); EXPECT_TRUE(wrapper); - wrapper->max_write_size_ = 128; ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp index e5e95e92..0adde6c0 100644 --- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp +++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp @@ -54,8 +54,7 @@ class BTreeCompatibilityTest : public ::testing::Test { EXPECT_OK_AND_ASSIGN(auto input, fs_->Open(path)); EXPECT_OK_AND_ASSIGN(auto length, input->Length()); std::string buffer(static_cast(length), '\0'); - EXPECT_OK_AND_ASSIGN([[maybe_unused]] auto bytes_read, - input->Read(buffer.data(), static_cast(length))); + EXPECT_OK_AND_ASSIGN([[maybe_unused]] auto bytes_read, input->Read(buffer.data(), length)); return buffer; } @@ -117,7 +116,7 @@ class BTreeCompatibilityTest : public ::testing::Test { auto meta_str = ReadFileAsString(meta_path); std::shared_ptr meta_bytes = Bytes::AllocateBytes(meta_str, pool_.get()); PAIMON_ASSIGN_OR_RAISE(auto file_status, fs_->GetFileStatus(bin_path)); - auto file_size = static_cast(file_status->GetLen()); + auto file_size = file_status->GetLen(); GlobalIndexIOMeta io_meta(bin_path, file_size, meta_bytes); std::vector metas = {io_meta}; diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index b05a0a8c..041394bc 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -55,7 +55,7 @@ class FakeGlobalIndexFileWriter : public GlobalIndexFileWriter { Result GetFileSize(const std::string& file_name) const override { PAIMON_ASSIGN_OR_RAISE(auto file_status, fs_->GetFileStatus(base_path_ + "/" + file_name)); - return static_cast(file_status->GetLen()); + return file_status->GetLen(); } std::string ToPath(const std::string& file_name) const override { diff --git a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp index 655efa91..6a45aee3 100644 --- a/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp +++ b/src/paimon/common/global_index/btree/lazy_filtered_btree_reader_test.cpp @@ -55,7 +55,7 @@ class FakeLazyFileWriter : public GlobalIndexFileWriter { Result GetFileSize(const std::string& file_name) const override { PAIMON_ASSIGN_OR_RAISE(auto file_status, fs_->GetFileStatus(base_path_ + "/" + file_name)); - return static_cast(file_status->GetLen()); + return file_status->GetLen(); } std::string ToPath(const std::string& file_name) const override { diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp index 0e032177..01a1f842 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index_test.cpp @@ -73,7 +73,6 @@ class RangeBitmapGlobalIndexTest : public ::testing::Test { auto wrapper = std::dynamic_pointer_cast(global_writer); EXPECT_TRUE(wrapper); - wrapper->max_write_size_ = 128; ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); diff --git a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h index 90a7b84d..a85b2dc4 100644 --- a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h +++ b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h @@ -19,9 +19,7 @@ #pragma once -#include #include -#include #include #include #include @@ -63,19 +61,10 @@ class FileIndexWriterWrapper : public GlobalIndexWriter { file_manager_->NewOutputStream(file_name)); PAIMON_ASSIGN_OR_RAISE(PAIMON_UNIQUE_PTR bytes, writer_->SerializedBytes()); - uint64_t total_write_size = 0; - while (total_write_size < bytes->size()) { - uint64_t current_write_size = - std::min(bytes->size() - total_write_size, max_write_size_); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_size, - out->Write(bytes->data() + total_write_size, - static_cast(current_write_size))); - if (static_cast(actual_size) != current_write_size) { - return Status::IOError( - fmt::format("expect write len {} mismatch actual write len {}", - current_write_size, actual_size)); - } - total_write_size += current_write_size; + PAIMON_ASSIGN_OR_RAISE(int64_t actual_size, out->Write(bytes->data(), bytes->size())); + if (actual_size < 0 || static_cast(actual_size) != bytes->size()) { + return Status::IOError(fmt::format("expect write len {} mismatch actual write len {}", + bytes->size(), actual_size)); } PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); @@ -85,11 +74,8 @@ class FileIndexWriterWrapper : public GlobalIndexWriter { } private: - static constexpr uint64_t kMaxWriteSize = std::numeric_limits::max(); - std::string index_type_; int64_t count_ = 0; - uint64_t max_write_size_ = kMaxWriteSize; std::shared_ptr file_manager_; std::shared_ptr writer_; }; diff --git a/src/paimon/common/io/buffered_input_stream.cpp b/src/paimon/common/io/buffered_input_stream.cpp index d2b50b73..e02620c0 100644 --- a/src/paimon/common/io/buffered_input_stream.cpp +++ b/src/paimon/common/io/buffered_input_stream.cpp @@ -25,13 +25,14 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/memory/bytes.h" namespace paimon { class MemoryPool; BufferedInputStream::BufferedInputStream(const std::shared_ptr& in, - int32_t buffer_size, MemoryPool* pool) + int64_t buffer_size, MemoryPool* pool) : buffer_size_(buffer_size), in_(in) { assert(buffer_size > 0); buffer_ = std::make_unique(buffer_size, pool); @@ -60,7 +61,7 @@ Status BufferedInputStream::Seek(int64_t offset, SeekOrigin origin) { const int64_t buf_start_abs = in_pos - count_; const int64_t buf_end_abs = in_pos; if (target_abs_offset >= buf_start_abs && target_abs_offset <= buf_end_abs) { - pos_ = static_cast(target_abs_offset - buf_start_abs); + pos_ = target_abs_offset - buf_start_abs; return Status::OK(); } } @@ -78,10 +79,11 @@ Result BufferedInputStream::GetPos() const { return in_pos - count_ + pos_; } -Result BufferedInputStream::Read(char* buffer, uint32_t size) { - uint32_t actual_read_len = 0; +Result BufferedInputStream::Read(char* buffer, int64_t size) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length")); + int64_t actual_read_len = 0; while (actual_read_len < size) { - PAIMON_ASSIGN_OR_RAISE(int32_t nread, + PAIMON_ASSIGN_OR_RAISE(int64_t nread, InnerRead(buffer + actual_read_len, size - actual_read_len)); assert(nread > 0); actual_read_len += nread; @@ -90,16 +92,16 @@ Result BufferedInputStream::Read(char* buffer, uint32_t size) { return actual_read_len; } -Result BufferedInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { +Result BufferedInputStream::Read(char* buffer, int64_t size, int64_t offset) { return Status::Invalid("BufferedInputStream does not support Read from offset"); } -void BufferedInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void BufferedInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { callback(Status::NotImplemented("BufferedInputStream do not support ReadAsync")); } -Result BufferedInputStream::Length() const { +Result BufferedInputStream::Length() const { return in_->Length(); } @@ -119,18 +121,21 @@ Status BufferedInputStream::Fill() { count_ = 0; PAIMON_ASSIGN_OR_RAISE(int64_t in_pos, in_->GetPos()); PAIMON_ASSIGN_OR_RAISE(int64_t length, in_->Length()); - int64_t left_to_read = std::min((length - in_pos), static_cast(buffer_size_)); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, in_->Read(buffer_->data(), left_to_read)); + int64_t left_to_read = std::min(length - in_pos, buffer_size_); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, in_->Read(buffer_->data(), left_to_read)); PAIMON_RETURN_NOT_OK(AssertReadLength(left_to_read, actual_read_len)); count_ = actual_read_len; return Status::OK(); } -Result BufferedInputStream::InnerRead(char* buffer, int32_t size) { +Result BufferedInputStream::InnerRead(char* buffer, int64_t size) { assert(size > 0); - int32_t avail = count_ - pos_; - if (avail <= 0) { - assert(avail == 0); + if (PAIMON_UNLIKELY(pos_ > count_)) { + return Status::Invalid(fmt::format( + "BufferedInputStream internal error: pos_ {} exceeds count_ {}", pos_, count_)); + } + int64_t avail = count_ - pos_; + if (avail == 0) { /* If the requested length is at least as large as the buffer, and if there is no mark/reset activity, do not bother to copy the bytes into the local buffer. In this way buffered streams will @@ -147,14 +152,14 @@ Result BufferedInputStream::InnerRead(char* buffer, int32_t size) { size)); } } - int32_t copy_length = std::min(avail, size); - memcpy(buffer, buffer_->data() + pos_, copy_length); + int64_t copy_length = std::min(avail, size); + memcpy(buffer, buffer_->data() + pos_, static_cast(copy_length)); pos_ += copy_length; return copy_length; } -Status BufferedInputStream::AssertReadLength(int32_t read_length, - int32_t actual_read_length) const { +Status BufferedInputStream::AssertReadLength(int64_t read_length, + int64_t actual_read_length) const { if (read_length != actual_read_length) { return Status::Invalid( fmt::format("assert read length failed: read length not match, read length {}, actual " diff --git a/src/paimon/common/io/byte_array_input_stream.cpp b/src/paimon/common/io/byte_array_input_stream.cpp index a88d6b82..06f0b0e2 100644 --- a/src/paimon/common/io/byte_array_input_stream.cpp +++ b/src/paimon/common/io/byte_array_input_stream.cpp @@ -26,9 +26,10 @@ #include "fmt/format.h" namespace paimon { -ByteArrayInputStream::ByteArrayInputStream(const char* buffer, uint64_t length) +ByteArrayInputStream::ByteArrayInputStream(const char* buffer, int64_t length) : buffer_(buffer), length_(length), position_(0) { assert(buffer_); + assert(length >= 0); } const char* ByteArrayInputStream::GetRawData() const { @@ -36,59 +37,60 @@ const char* ByteArrayInputStream::GetRawData() const { } Status ByteArrayInputStream::Seek(int64_t offset, SeekOrigin origin) { + int64_t new_position = 0; switch (origin) { case SeekOrigin::FS_SEEK_SET: { - position_ = offset; + new_position = offset; break; } case SeekOrigin::FS_SEEK_CUR: { - position_ += offset; + new_position = position_ + offset; break; } case SeekOrigin::FS_SEEK_END: { - PAIMON_ASSIGN_OR_RAISE(uint64_t length, Length()); - position_ = static_cast(length) + offset; + new_position = length_ + offset; break; } default: return Status::Invalid( "invalid SeekOrigin, only support FS_SEEK_SET, FS_SEEK_CUR, and FS_SEEK_END"); } - if (position_ < 0 || position_ > static_cast(length_)) { - return Status::Invalid( - fmt::format("invalid seek, after seek, current pos {}, length {}", position_, length_)); + if (new_position < 0 || new_position > length_) { + return Status::Invalid(fmt::format("invalid seek, after seek, current pos {}, length {}", + new_position, length_)); } + position_ = new_position; return Status::OK(); } -Result ByteArrayInputStream::Read(char* buffer, uint32_t size) { - if (position_ + static_cast(size) > static_cast(length_)) { +Result ByteArrayInputStream::Read(char* buffer, int64_t size) { + if (size < 0 || size > length_ - position_) { return Status::Invalid( fmt::format("ByteArrayInputStream assert boundary failed: need length {}, current " "position {}, exceed length {}", size, position_, length_)); } - memcpy(buffer, buffer_ + position_, size); + memcpy(buffer, buffer_ + position_, static_cast(size)); position_ += size; return size; } -Result ByteArrayInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { - if (offset + static_cast(size) > length_) { +Result ByteArrayInputStream::Read(char* buffer, int64_t size, int64_t offset) { + if (size < 0 || offset < 0 || offset > length_ || size > length_ - offset) { return Status::Invalid( - fmt::format("ByteArrayInputStream assert boundary failed: need length {}, read offset " - "{}, exceed length {}", + fmt::format("ByteArrayInputStream boundary check failed: read size {}, offset {}, " + "stream length {}", size, offset, length_)); } - memcpy(buffer, buffer_ + offset, size); + memcpy(buffer, buffer_ + offset, static_cast(size)); return size; } -void ByteArrayInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void ByteArrayInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { - Result read_size = Read(buffer, size, offset); + Result read_size = Read(buffer, size, offset); Status status = Status::OK(); - if (read_size.ok() && static_cast(read_size.value()) != size) { + if (read_size.ok() && read_size.value() != size) { status = Status::Invalid(fmt::format( "ByteArrayInputStream async read size {} != expected {}", read_size.value(), size)); } else if (!read_size.ok()) { diff --git a/src/paimon/common/io/cache_input_stream.h b/src/paimon/common/io/cache_input_stream.h index 3c48df30..9ccbf260 100644 --- a/src/paimon/common/io/cache_input_stream.h +++ b/src/paimon/common/io/cache_input_stream.h @@ -22,6 +22,7 @@ #include #include +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/utils/read_ahead_cache.h" @@ -39,12 +40,14 @@ class CacheInputStream : public InputStream { Result GetPos() const override { return input_stream_->GetPos(); } - Result Read(char* buffer, uint32_t size) override { + Result Read(char* buffer, int64_t size) override { return input_stream_->Read(buffer, size); } - Result Read(char* buffer, uint32_t size, uint64_t offset) override { + Result Read(char* buffer, int64_t size, int64_t offset) override { if (cache_) { - ByteRange range{offset, static_cast(size)}; + PAIMON_RETURN_NOT_OK(ValidateValueInRange(offset, "read offset")); + PAIMON_RETURN_NOT_OK(ValidateValueInRange(size, "read size")); + ByteRange range{static_cast(offset), static_cast(size)}; PAIMON_ASSIGN_OR_RAISE(ByteSlice slice, cache_->Read(range)); if (slice.buffer) { std::memcpy(buffer, slice.buffer->data() + slice.offset, slice.length); @@ -53,10 +56,20 @@ class CacheInputStream : public InputStream { } return input_stream_->Read(buffer, size, offset); } - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override { if (cache_) { - ByteRange range{offset, static_cast(size)}; + Status status = ValidateValueInRange(offset, "read offset"); + if (!status.ok()) { + callback(status); + return; + } + status = ValidateValueInRange(size, "read size"); + if (!status.ok()) { + callback(status); + return; + } + ByteRange range{static_cast(offset), static_cast(size)}; Result slice = cache_->Read(range); if (!slice.ok()) { callback(slice.status()); @@ -80,7 +93,7 @@ class CacheInputStream : public InputStream { return input_stream_->GetUri(); } - Result Length() const override { + Result Length() const override { return input_stream_->Length(); } diff --git a/src/paimon/common/io/cache_input_stream_test.cpp b/src/paimon/common/io/cache_input_stream_test.cpp index 2c2e36cf..d4a61854 100644 --- a/src/paimon/common/io/cache_input_stream_test.cpp +++ b/src/paimon/common/io/cache_input_stream_test.cpp @@ -80,7 +80,7 @@ TEST_F(CacheInputStreamTest, TestProxyMethods) { CacheInputStream stream(std::move(underlying), /*cache=*/nullptr); // Length - ASSERT_OK_AND_ASSIGN(uint64_t length, stream.Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, stream.Length()); ASSERT_EQ(length, content_.size()); // GetUri @@ -94,7 +94,7 @@ TEST_F(CacheInputStreamTest, TestProxyMethods) { // Read (sequential, no offset) std::string buffer(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t bytes_read, stream.Read(buffer.data(), 3)); + ASSERT_OK_AND_ASSIGN(int64_t bytes_read, stream.Read(buffer.data(), 3)); ASSERT_EQ(bytes_read, 3); ASSERT_EQ(buffer, "fgh"); @@ -108,7 +108,7 @@ TEST_F(CacheInputStreamTest, TestReadWithOffsetNullCache) { CacheInputStream stream(std::move(underlying), /*cache=*/nullptr); std::string buffer(5, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t bytes_read, stream.Read(buffer.data(), 5, /*offset=*/2)); + ASSERT_OK_AND_ASSIGN(int64_t bytes_read, stream.Read(buffer.data(), 5, /*offset=*/2)); ASSERT_EQ(bytes_read, 5); ASSERT_EQ(buffer, "cdefg"); } @@ -121,7 +121,7 @@ TEST_F(CacheInputStreamTest, TestReadWithOffsetCacheHit) { CacheInputStream stream(std::move(underlying), cache); std::string buffer(5, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t bytes_read, stream.Read(buffer.data(), 5, /*offset=*/2)); + ASSERT_OK_AND_ASSIGN(int64_t bytes_read, stream.Read(buffer.data(), 5, /*offset=*/2)); ASSERT_EQ(bytes_read, 5); ASSERT_EQ(buffer, "cdefg"); } @@ -134,7 +134,7 @@ TEST_F(CacheInputStreamTest, TestReadWithOffsetCacheMiss) { CacheInputStream stream(std::move(underlying), cache); std::string buffer(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t bytes_read, stream.Read(buffer.data(), 3, /*offset=*/10)); + ASSERT_OK_AND_ASSIGN(int64_t bytes_read, stream.Read(buffer.data(), 3, /*offset=*/10)); ASSERT_EQ(bytes_read, 3); ASSERT_EQ(buffer, "klm"); } diff --git a/src/paimon/common/io/data_input_stream.cpp b/src/paimon/common/io/data_input_stream.cpp index 63f83381..9ffe75b0 100644 --- a/src/paimon/common/io/data_input_stream.cpp +++ b/src/paimon/common/io/data_input_stream.cpp @@ -42,10 +42,10 @@ Status DataInputStream::Seek(int64_t offset) const { template Result DataInputStream::ReadValue() const { static_assert(std::is_trivially_copyable_v, "T must be trivially copyable"); - int32_t read_length = sizeof(T); + int64_t read_length = sizeof(T); PAIMON_RETURN_NOT_OK(AssertBoundary(read_length)); T value; - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_length, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_length, input_stream_->Read(reinterpret_cast(&value), read_length)); PAIMON_RETURN_NOT_OK(AssertReadLength(read_length, actual_read_length)); if (NeedSwap()) { @@ -55,17 +55,17 @@ Result DataInputStream::ReadValue() const { } Status DataInputStream::ReadBytes(Bytes* bytes) const { - int32_t read_length = bytes->size(); + int64_t read_length = bytes->size(); PAIMON_RETURN_NOT_OK(AssertBoundary(read_length)); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_length, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_length, input_stream_->Read(bytes->data(), read_length)); PAIMON_RETURN_NOT_OK(AssertReadLength(read_length, actual_read_length)); return Status::OK(); } -Status DataInputStream::Read(char* data, uint32_t size) const { +Status DataInputStream::Read(char* data, int64_t size) const { PAIMON_RETURN_NOT_OK(AssertBoundary(size)); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_length, input_stream_->Read(data, size)); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_length, input_stream_->Read(data, size)); PAIMON_RETURN_NOT_OK(AssertReadLength(size, actual_read_length)); return Status::OK(); } @@ -75,7 +75,7 @@ Result DataInputStream::ReadString() const { PAIMON_ASSIGN_OR_RAISE(read_length, ReadValue()); PAIMON_RETURN_NOT_OK(AssertBoundary(read_length)); std::string value(read_length, '\0'); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_length, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_length, input_stream_->Read(value.data(), read_length)); PAIMON_RETURN_NOT_OK(AssertReadLength(read_length, actual_read_length)); return value; @@ -85,11 +85,11 @@ Result DataInputStream::GetPos() const { return input_stream_->GetPos(); } -Result DataInputStream::Length() const { +Result DataInputStream::Length() const { return input_stream_->Length(); } -Status DataInputStream::AssertReadLength(int32_t read_length, int32_t actual_read_length) const { +Status DataInputStream::AssertReadLength(int64_t read_length, int64_t actual_read_length) const { if (read_length != actual_read_length) { return Status::Invalid( fmt::format("assert read length failed: read length not match, read length {}, actual " @@ -99,15 +99,16 @@ Status DataInputStream::AssertReadLength(int32_t read_length, int32_t actual_rea return Status::OK(); } -Status DataInputStream::AssertBoundary(int32_t need_length) const { +Status DataInputStream::AssertBoundary(int64_t need_length) const { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(need_length, "DataInputStream need length")); // TODO(jinli.zjw): Store current_pos and file_length as member variables to reduce the overhead // of I/O calls. PAIMON_ASSIGN_OR_RAISE(int64_t pos, input_stream_->GetPos()); - PAIMON_ASSIGN_OR_RAISE(uint64_t length, input_stream_->Length()); - if (pos + need_length > static_cast(length)) { + PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream_->Length()); + if (pos < 0 || length < 0 || pos > length || need_length > length - pos) { return Status::Invalid( - fmt::format("DataInputStream assert boundary failed: need length {}, current position " - "{}, exceed length {}", + fmt::format("DataInputStream boundary check failed: read size {}, position {}, " + "stream length {}", need_length, pos, length)); } return Status::OK(); diff --git a/src/paimon/common/io/data_output_stream.cpp b/src/paimon/common/io/data_output_stream.cpp index 14d7463f..53825a05 100644 --- a/src/paimon/common/io/data_output_stream.cpp +++ b/src/paimon/common/io/data_output_stream.cpp @@ -30,8 +30,8 @@ DataOutputStream::DataOutputStream(const std::shared_ptr& output_s } Status DataOutputStream::WriteBytes(const std::shared_ptr& bytes) { - int32_t write_length = bytes->size(); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_write_length, + int64_t write_length = bytes->size(); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_write_length, output_stream_->Write(bytes->data(), write_length)); PAIMON_RETURN_NOT_OK(AssertWriteLength(write_length, actual_write_length)); return Status::OK(); @@ -40,14 +40,14 @@ Status DataOutputStream::WriteBytes(const std::shared_ptr& bytes) { Status DataOutputStream::WriteString(const std::string& value) { uint16_t write_length = value.size(); PAIMON_RETURN_NOT_OK(WriteValue(write_length)); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_write_length, + PAIMON_ASSIGN_OR_RAISE(int64_t actual_write_length, output_stream_->Write(value.data(), write_length)); PAIMON_RETURN_NOT_OK(AssertWriteLength(write_length, actual_write_length)); return Status::OK(); } -Status DataOutputStream::AssertWriteLength(int32_t write_length, - int32_t actual_write_length) const { +Status DataOutputStream::AssertWriteLength(int64_t write_length, + int64_t actual_write_length) const { if (write_length != actual_write_length) { return Status::Invalid(fmt::format( "assert write length failed: write length not match, write length {}, actual " diff --git a/src/paimon/common/io/data_output_stream.h b/src/paimon/common/io/data_output_stream.h index ace5976a..dfb6456b 100644 --- a/src/paimon/common/io/data_output_stream.h +++ b/src/paimon/common/io/data_output_stream.h @@ -48,9 +48,9 @@ class PAIMON_EXPORT DataOutputStream { if (NeedSwap()) { write_value = EndianSwapValue(value); } - int32_t write_length = sizeof(T); + int64_t write_length = sizeof(T); PAIMON_ASSIGN_OR_RAISE( - int32_t actual_write_length, + int64_t actual_write_length, output_stream_->Write(reinterpret_cast(&write_value), write_length)); PAIMON_RETURN_NOT_OK(AssertWriteLength(write_length, actual_write_length)); return Status::OK(); @@ -66,7 +66,7 @@ class PAIMON_EXPORT DataOutputStream { } private: - Status AssertWriteLength(int32_t write_length, int32_t actual_write_length) const; + Status AssertWriteLength(int64_t write_length, int64_t actual_write_length) const; bool NeedSwap() const; diff --git a/src/paimon/common/io/offset_input_stream.cpp b/src/paimon/common/io/offset_input_stream.cpp index 69593cd1..2df88adc 100644 --- a/src/paimon/common/io/offset_input_stream.cpp +++ b/src/paimon/common/io/offset_input_stream.cpp @@ -22,6 +22,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/macros.h" namespace paimon { @@ -30,28 +31,29 @@ Result> OffsetInputStream::Create( if (PAIMON_UNLIKELY(wrapped == nullptr)) { return Status::Invalid("input stream is null pointer"); } - if (PAIMON_UNLIKELY(offset < 0)) { - return Status::Invalid(fmt::format("offset {} is less than 0", offset)); - } - if (PAIMON_UNLIKELY(length < -1)) { - return Status::Invalid(fmt::format("length {} is less than -1", length)); + PAIMON_ASSIGN_OR_RAISE(int64_t total_length, wrapped->Length()); + return Create(wrapped, length, offset, total_length); +} + +Result> OffsetInputStream::Create( + const std::shared_ptr& wrapped, int64_t length, int64_t offset, + int64_t total_length) { + if (PAIMON_UNLIKELY(wrapped == nullptr)) { + return Status::Invalid("input stream is null pointer"); } - PAIMON_ASSIGN_OR_RAISE(uint64_t total_length, wrapped->Length()); - if (PAIMON_UNLIKELY((uint64_t)offset > total_length)) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(offset, "offset")); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "length")); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(total_length, "total length")); + if (PAIMON_UNLIKELY(offset > total_length)) { return Status::Invalid( fmt::format("offset {} exceed total length {}", offset, total_length)); } - if (length == -1) { - // length == -1 means it's dynamic length, should read to the end - length = total_length - offset; - } - if (PAIMON_UNLIKELY((uint64_t)offset + (uint64_t)length > total_length)) { + if (PAIMON_UNLIKELY(length > total_length - offset)) { return Status::Invalid(fmt::format("offset {} + length {} exceed total length {}", offset, length, total_length)); } PAIMON_RETURN_NOT_OK(wrapped->Seek(offset, SeekOrigin::FS_SEEK_SET)); - return std::unique_ptr( - new OffsetInputStream(std::move(wrapped), length, offset)); + return std::unique_ptr(new OffsetInputStream(wrapped, length, offset)); } OffsetInputStream::OffsetInputStream(const std::shared_ptr& wrapped, int64_t length, @@ -59,42 +61,48 @@ OffsetInputStream::OffsetInputStream(const std::shared_ptr& wrapped : wrapped_(wrapped), length_(length), offset_(offset) {} Status OffsetInputStream::Seek(int64_t offset, SeekOrigin origin) { + int64_t new_position = 0; switch (origin) { case SeekOrigin::FS_SEEK_SET: { - inner_position_ = offset; - PAIMON_RETURN_NOT_OK(AssertBoundary(inner_position_)); - return wrapped_->Seek(offset_ + inner_position_, SeekOrigin::FS_SEEK_SET); + new_position = offset; + PAIMON_RETURN_NOT_OK(AssertBoundary(new_position)); + PAIMON_RETURN_NOT_OK(wrapped_->Seek(offset_ + new_position, SeekOrigin::FS_SEEK_SET)); + break; } case SeekOrigin::FS_SEEK_CUR: { - inner_position_ += offset; - PAIMON_RETURN_NOT_OK(AssertBoundary(inner_position_)); - return wrapped_->Seek(offset, SeekOrigin::FS_SEEK_CUR); + new_position = inner_position_ + offset; + PAIMON_RETURN_NOT_OK(AssertBoundary(new_position)); + PAIMON_RETURN_NOT_OK(wrapped_->Seek(offset, SeekOrigin::FS_SEEK_CUR)); + break; } case SeekOrigin::FS_SEEK_END: { - inner_position_ = length_ + offset; - PAIMON_RETURN_NOT_OK(AssertBoundary(inner_position_)); - return wrapped_->Seek(offset_ + inner_position_, SeekOrigin::FS_SEEK_SET); + new_position = length_ + offset; + PAIMON_RETURN_NOT_OK(AssertBoundary(new_position)); + PAIMON_RETURN_NOT_OK(wrapped_->Seek(offset_ + new_position, SeekOrigin::FS_SEEK_SET)); + break; } default: return Status::Invalid( "invalid SeekOrigin, only support FS_SEEK_SET, FS_SEEK_CUR, and FS_SEEK_END"); } + inner_position_ = new_position; return Status::OK(); } -Result OffsetInputStream::Read(char* buffer, uint32_t size) { +Result OffsetInputStream::Read(char* buffer, int64_t size) { PAIMON_RETURN_NOT_OK(AssertBoundary(inner_position_ + size)); - inner_position_ += size; - return wrapped_->Read(buffer, size); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, wrapped_->Read(buffer, size)); + inner_position_ += actual_read_len; + return actual_read_len; } -Result OffsetInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { +Result OffsetInputStream::Read(char* buffer, int64_t size, int64_t offset) { PAIMON_RETURN_NOT_OK(AssertBoundary(offset)); PAIMON_RETURN_NOT_OK(AssertBoundary(offset + size)); return wrapped_->Read(buffer, size, offset_ + offset); } -void OffsetInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void OffsetInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { auto status = AssertBoundary(offset); if (!status.ok()) { @@ -121,11 +129,11 @@ Result OffsetInputStream::GetPos() const { return inner_position_; } -Result OffsetInputStream::Length() const { +Result OffsetInputStream::Length() const { return length_; } -Status OffsetInputStream::AssertBoundary(int32_t inner_pos) const { +Status OffsetInputStream::AssertBoundary(int64_t inner_pos) const { if (inner_pos < 0 || inner_pos > length_) { return Status::Invalid( fmt::format("OffsetInputStream assert boundary failed: inner pos {} exceed length {}", diff --git a/src/paimon/common/io/offset_input_stream.h b/src/paimon/common/io/offset_input_stream.h index 38f97ef9..9fb6f4c0 100644 --- a/src/paimon/common/io/offset_input_stream.h +++ b/src/paimon/common/io/offset_input_stream.h @@ -35,20 +35,23 @@ class PAIMON_EXPORT OffsetInputStream : public InputStream { public: static Result> Create( const std::shared_ptr& wrapped, int64_t length, int64_t offset); + static Result> Create( + const std::shared_ptr& wrapped, int64_t length, int64_t offset, + int64_t total_length); ~OffsetInputStream() override = default; Status Seek(int64_t offset, SeekOrigin origin) override; Result GetPos() const override; - Result Read(char* buffer, uint32_t size) override; + Result Read(char* buffer, int64_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; - Result Length() const override; + Result Length() const override; Status Close() override; @@ -56,7 +59,7 @@ class PAIMON_EXPORT OffsetInputStream : public InputStream { private: OffsetInputStream(const std::shared_ptr& wrapped, int64_t length, int64_t offset); - Status AssertBoundary(int32_t inner_pos) const; + Status AssertBoundary(int64_t inner_pos) const; private: std::shared_ptr wrapped_; diff --git a/src/paimon/common/io/offset_input_stream_test.cpp b/src/paimon/common/io/offset_input_stream_test.cpp index 3734650b..734c814f 100644 --- a/src/paimon/common/io/offset_input_stream_test.cpp +++ b/src/paimon/common/io/offset_input_stream_test.cpp @@ -146,43 +146,6 @@ TEST(OffsetInputStreamTest, TestBoundaryValidation) { "assert boundary failed: inner pos 9 exceed length 6"); } -TEST(OffsetInputStreamTest, TestReadWithUnspecifiedLength) { - auto inner_stream = std::make_unique("abcdefghij", /*length=*/10); - // Use -1 for length to test dynamic length calculation - ASSERT_OK_AND_ASSIGN( - auto offset_stream, - OffsetInputStream::Create(std::move(inner_stream), /*length=*/-1, /*offset=*/2)); - - // Test that length is calculated correctly - ASSERT_OK_AND_ASSIGN(auto length, offset_stream->Length()); - // Should be total length (10) minus offset (2) = 8 - ASSERT_EQ(8, length); - - // Test sequential read within the calculated bounds - std::string buffer(4, '\0'); - ASSERT_OK_AND_ASSIGN(auto bytes_read, offset_stream->Read(buffer.data(), /*size=*/4)); - ASSERT_EQ(4, bytes_read); - ASSERT_EQ("cdef", buffer); - - ASSERT_OK_AND_ASSIGN(auto pos, offset_stream->GetPos()); - ASSERT_EQ(4, pos); - - // Test read with offset within the calculated bounds - std::string buffer2(3, '\0'); - ASSERT_OK_AND_ASSIGN(bytes_read, offset_stream->Read(buffer2.data(), /*size=*/3, /*offset=*/5)); - ASSERT_EQ(3, bytes_read); - ASSERT_EQ("hij", buffer2); - - // Position should not change after offset read - ASSERT_OK_AND_ASSIGN(pos, offset_stream->GetPos()); - ASSERT_EQ(4, pos); - - // Test boundary validation with dynamic length - std::string buffer3(10, '\0'); - ASSERT_NOK_WITH_MSG(offset_stream->Read(buffer3.data(), /*size=*/10), - "assert boundary failed: inner pos 14 exceed length 8"); -} - TEST(OffsetInputStreamTest, TestInvalidParameters) { // Test null wrapped stream ASSERT_NOK_WITH_MSG(OffsetInputStream::Create(nullptr, /*length=*/6, /*offset=*/2), @@ -194,11 +157,11 @@ TEST(OffsetInputStreamTest, TestInvalidParameters) { OffsetInputStream::Create(std::move(inner_stream), /*length=*/6, /*offset=*/-1), "offset -1 is less than 0"); - // Test length less than -1 + // Test negative length inner_stream = std::make_unique("abcdefghij", /*length=*/10); ASSERT_NOK_WITH_MSG( OffsetInputStream::Create(std::move(inner_stream), /*length=*/-2, /*offset=*/2), - "length -2 is less than -1"); + "length -2 is less than 0"); // Test length + offset beyond wrapped stream length inner_stream = std::make_unique("abcdefghij", /*length=*/10); @@ -206,10 +169,10 @@ TEST(OffsetInputStreamTest, TestInvalidParameters) { OffsetInputStream::Create(std::move(inner_stream), /*length=*/8, /*offset=*/7), "offset 7 + length 8 exceed total length 10"); - // Test dynamic length with offset beyond wrapped stream length + // Test offset beyond wrapped stream length inner_stream = std::make_unique("abcdefghij", /*length=*/10); ASSERT_NOK_WITH_MSG( - OffsetInputStream::Create(std::move(inner_stream), /*length=*/-1, /*offset=*/15), + OffsetInputStream::Create(std::move(inner_stream), /*length=*/1, /*offset=*/15), "offset 15 exceed total length 10"); } diff --git a/src/paimon/common/memory/bytes.cpp b/src/paimon/common/memory/bytes.cpp index bfe7b28e..d1093f52 100644 --- a/src/paimon/common/memory/bytes.cpp +++ b/src/paimon/common/memory/bytes.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -33,7 +32,7 @@ const std::shared_ptr& Bytes::EmptyBytes() { return empty_bytes; } -PAIMON_UNIQUE_PTR Bytes::AllocateBytes(int32_t length, MemoryPool* pool) { +PAIMON_UNIQUE_PTR Bytes::AllocateBytes(size_t length, MemoryPool* pool) { return pool->AllocateUnique(length, pool); } diff --git a/src/paimon/common/memory/memory_segment_utils.cpp b/src/paimon/common/memory/memory_segment_utils.cpp index 7f56792d..fe406f9a 100644 --- a/src/paimon/common/memory/memory_segment_utils.cpp +++ b/src/paimon/common/memory/memory_segment_utils.cpp @@ -24,9 +24,6 @@ #include "paimon/common/utils/murmurhash_utils.h" namespace paimon { -std::shared_ptr MemorySegmentUtils::AllocateBytes(int32_t length, MemoryPool* pool) { - return Bytes::AllocateBytes(length, pool); -} void MemorySegmentUtils::CopyFromBytes(std::vector* segments, int32_t offset, const Bytes& bytes, int32_t bytes_offset, @@ -316,7 +313,7 @@ int32_t MemorySegmentUtils::HashByWords(const std::vector& segmen int32_t MemorySegmentUtils::HashMultiSegByWords(const std::vector& segments, int32_t offset, int32_t num_bytes, MemoryPool* pool) { - std::shared_ptr bytes = AllocateBytes(num_bytes, pool); + std::shared_ptr bytes = Bytes::AllocateBytes(num_bytes, pool); CopyMultiSegmentsToBytes(segments, offset, bytes.get(), 0, num_bytes); return MurmurHashUtils::HashUnsafeBytesByWords(reinterpret_cast(bytes->data()), 0, num_bytes); @@ -324,7 +321,7 @@ int32_t MemorySegmentUtils::HashMultiSegByWords(const std::vector int32_t MemorySegmentUtils::HashMultiSeg(const std::vector& segments, int32_t offset, int32_t num_bytes, MemoryPool* pool) { - std::shared_ptr bytes = AllocateBytes(num_bytes, pool); + std::shared_ptr bytes = Bytes::AllocateBytes(num_bytes, pool); CopyMultiSegmentsToBytes(segments, offset, bytes.get(), 0, num_bytes); return MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(bytes->data()), 0, num_bytes); diff --git a/src/paimon/common/memory/memory_segment_utils.h b/src/paimon/common/memory/memory_segment_utils.h index b904728f..4f1fdc65 100644 --- a/src/paimon/common/memory/memory_segment_utils.h +++ b/src/paimon/common/memory/memory_segment_utils.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -43,9 +44,6 @@ class PAIMON_EXPORT MemorySegmentUtils { MemorySegmentUtils() = delete; ~MemorySegmentUtils() = delete; - /// Allocate bytes in pool - static std::shared_ptr AllocateBytes(int32_t length, MemoryPool* pool); - /// Copy target segments from source byte[]. /// /// @param segments target segments. diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp index 165b9bf7..036b1d3b 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp @@ -531,7 +531,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, WorkloopSetReadStatusWhenCacheInitFailed MockFormatReaderBuilder reader_builder(data_array, data_type_, batch_size); CacheConfig invalid_cache_config( /*buffer_size_limit=*/512 * 1024, - /*range_size_limit=*/static_cast(std::numeric_limits::max()) + 1, + /*range_size_limit=*/4 * 1024, /*hole_size_limit=*/8 * 1024, /*pre_buffer_limit=*/128 * 1024); @@ -547,9 +547,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, WorkloopSetReadStatusWhenCacheInitFailed auto prefetch_reader = dynamic_cast(reader.get()); prefetch_reader->Workloop(); - Status status = prefetch_reader->GetReadStatus(); - ASSERT_FALSE(status.ok()); - ASSERT_TRUE(status.IsInvalid()); + ASSERT_NOK_WITH_MSG(prefetch_reader->GetReadStatus(), + "range size limit 4096 should be larger than hole size limit 8192"); } TEST_F(PrefetchFileBatchReaderImplTest, DoReadBatchReturnOkWhenShutdown) { diff --git a/src/paimon/common/sst/bloom_filter_handle.h b/src/paimon/common/sst/bloom_filter_handle.h index 9796bcba..1974604e 100644 --- a/src/paimon/common/sst/bloom_filter_handle.h +++ b/src/paimon/common/sst/bloom_filter_handle.h @@ -18,11 +18,10 @@ #pragma once -#include +#include +#include -#include "paimon/common/memory/memory_segment.h" -#include "paimon/memory/bytes.h" -#include "paimon/result.h" +#include "paimon/visibility.h" namespace paimon { diff --git a/src/paimon/common/sst/sst_file_reader.cpp b/src/paimon/common/sst/sst_file_reader.cpp index 9639bcd3..315bc199 100644 --- a/src/paimon/common/sst/sst_file_reader.cpp +++ b/src/paimon/common/sst/sst_file_reader.cpp @@ -67,7 +67,7 @@ Result> SstFileReader::Create( Result> SstFileReader::CreateForSortLookupStore( const std::shared_ptr& in, MemorySlice::SliceComparator comparator, const std::shared_ptr& block_cache, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(uint64_t file_len, in->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_len, in->Length()); PAIMON_RETURN_NOT_OK( in->Seek(file_len - SortLookupStoreFooter::ENCODED_LENGTH, SeekOrigin::FS_SEEK_SET)); auto footer_bytes = Bytes::AllocateBytes(SortLookupStoreFooter::ENCODED_LENGTH, pool.get()); diff --git a/src/paimon/common/sst/sst_file_utils.h b/src/paimon/common/sst/sst_file_utils.h index 24c61788..25a9f314 100644 --- a/src/paimon/common/sst/sst_file_utils.h +++ b/src/paimon/common/sst/sst_file_utils.h @@ -17,11 +17,9 @@ */ #pragma once -#include #include "fmt/format.h" #include "paimon/common/compression/block_compression_type.h" -#include "paimon/common/memory/memory_slice.h" namespace paimon { diff --git a/src/paimon/common/sst/sst_file_writer.cpp b/src/paimon/common/sst/sst_file_writer.cpp index 003a386b..ec736e33 100644 --- a/src/paimon/common/sst/sst_file_writer.cpp +++ b/src/paimon/common/sst/sst_file_writer.cpp @@ -18,7 +18,6 @@ #include "paimon/common/sst/sst_file_writer.h" -#include "paimon/common/sst/sst_file_utils.h" #include "paimon/common/utils/crc32c.h" #include "paimon/common/utils/murmurhash_utils.h" diff --git a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp index adf3cd8f..1dab4e48 100644 --- a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp +++ b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp @@ -24,10 +24,8 @@ #include #include "arrow/api.h" -#include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/math.h" -#include "paimon/common/utils/options_utils.h" #include "paimon/fs/file_system.h" #include "paimon/macros.h" #include "paimon/result.h" @@ -37,11 +35,10 @@ namespace paimon { namespace { -template -arrow::Status ValidateArrowIoRange(From value, const char* name) { - if (!InRange(value)) { - return arrow::Status::Invalid(fmt::format("{} value {} is out of bound of type {}", name, - value, OptionsUtils::GetTypeName())); +arrow::Status ValidateArrowIoRange(int64_t value, const char* name) { + Status status = ValidateValueNonNegative(value, name); + if (!status.ok()) { + return ToArrowStatus(status); } return arrow::Status::OK(); } @@ -50,8 +47,10 @@ arrow::Status ValidateArrowIoRange(From value, const char* name) { ArrowInputStreamAdapter::ArrowInputStreamAdapter( const std::shared_ptr& input_stream, - const std::shared_ptr& pool, uint64_t file_size) - : input_stream_(input_stream), pool_(pool), file_size_(file_size) {} + const std::shared_ptr& pool, int64_t file_size) + : input_stream_(input_stream), pool_(pool), file_size_(file_size) { + assert(file_size >= 0); +} ArrowInputStreamAdapter::~ArrowInputStreamAdapter() { [[maybe_unused]] auto status = DoClose(); @@ -62,9 +61,8 @@ arrow::Status ArrowInputStreamAdapter::Seek(int64_t position) { } arrow::Result ArrowInputStreamAdapter::Read(int64_t nbytes, void* out) { - ARROW_RETURN_NOT_OK(ValidateArrowIoRange(nbytes, "nbytes")); - Result read_bytes = - input_stream_->Read(static_cast(out), static_cast(nbytes)); + ARROW_RETURN_NOT_OK(ValidateArrowIoRange(nbytes, "nbytes")); + Result read_bytes = input_stream_->Read(static_cast(out), nbytes); if (!read_bytes.ok()) { return ToArrowStatus(read_bytes.status()); } @@ -83,10 +81,9 @@ arrow::Result> ArrowInputStreamAdapter::Read(int6 arrow::Result ArrowInputStreamAdapter::ReadAt(int64_t position, int64_t nbytes, void* out) { - ARROW_RETURN_NOT_OK(ValidateArrowIoRange(position, "position")); - ARROW_RETURN_NOT_OK(ValidateArrowIoRange(nbytes, "nbytes")); - Result read_bytes = input_stream_->Read( - static_cast(out), static_cast(nbytes), static_cast(position)); + ARROW_RETURN_NOT_OK(ValidateArrowIoRange(position, "position")); + ARROW_RETURN_NOT_OK(ValidateArrowIoRange(nbytes, "nbytes")); + Result read_bytes = input_stream_->Read(static_cast(out), nbytes, position); if (!read_bytes.ok()) { return ToArrowStatus(read_bytes.status()); } @@ -107,12 +104,12 @@ arrow::Result> ArrowInputStreamAdapter::ReadAt(in arrow::Future> ArrowInputStreamAdapter::ReadAsync( const arrow::io::IOContext& io_context, int64_t position, int64_t nbytes) { auto fut = arrow::Future>::Make(); - auto range_status = ValidateArrowIoRange(position, "position"); + auto range_status = ValidateArrowIoRange(position, "position"); if (!range_status.ok()) { fut.MarkFinished(range_status); return fut; } - range_status = ValidateArrowIoRange(nbytes, "nbytes"); + range_status = ValidateArrowIoRange(nbytes, "nbytes"); if (!range_status.ok()) { fut.MarkFinished(range_status); return fut; @@ -125,8 +122,7 @@ arrow::Future> ArrowInputStreamAdapter::ReadAsync return fut; } std::shared_ptr buffer = std::move(buffer_result).ValueUnsafe(); - input_stream_->ReadAsync(reinterpret_cast(buffer->mutable_data()), - static_cast(nbytes), static_cast(position), + input_stream_->ReadAsync(reinterpret_cast(buffer->mutable_data()), nbytes, position, [fut, buffer](Status callback_status) mutable { if (callback_status.ok()) { fut.MarkFinished(std::move(buffer)); @@ -146,7 +142,7 @@ arrow::Result ArrowInputStreamAdapter::Tell() const { } arrow::Result ArrowInputStreamAdapter::GetSize() { - return static_cast(file_size_); + return file_size_; } bool ArrowInputStreamAdapter::closed() const { diff --git a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h index 00ee1648..134568b2 100644 --- a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h +++ b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h @@ -33,7 +33,7 @@ class InputStream; class PAIMON_EXPORT ArrowInputStreamAdapter : public arrow::io::RandomAccessFile { public: ArrowInputStreamAdapter(const std::shared_ptr& input_stream, - const std::shared_ptr& pool, uint64_t file_size); + const std::shared_ptr& pool, int64_t file_size); ~ArrowInputStreamAdapter() override; // NOTE: In paimon file system definition, position + nbytes should not exceed file_size_. @@ -57,7 +57,7 @@ class PAIMON_EXPORT ArrowInputStreamAdapter : public arrow::io::RandomAccessFile std::shared_ptr input_stream_; std::shared_ptr pool_; - uint64_t file_size_; + int64_t file_size_; bool closed_ = false; }; diff --git a/src/paimon/common/utils/arrow/arrow_output_stream_adapter.cpp b/src/paimon/common/utils/arrow/arrow_output_stream_adapter.cpp index c74c090f..f89e0810 100644 --- a/src/paimon/common/utils/arrow/arrow_output_stream_adapter.cpp +++ b/src/paimon/common/utils/arrow/arrow_output_stream_adapter.cpp @@ -22,7 +22,6 @@ #include "arrow/result.h" #include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/result.h" @@ -51,15 +50,17 @@ bool ArrowOutputStreamAdapter::closed() const { } arrow::Status ArrowOutputStreamAdapter::Write(const void* data, int64_t nbytes) { - if (!InRange(nbytes)) { - return arrow::Status::Invalid( - fmt::format("nbytes value {} is out of bound of uint32_t", nbytes)); + if (nbytes < 0) { + return arrow::Status::Invalid(fmt::format("write size {} is less than 0", nbytes)); } - Result len = - out_->Write(static_cast(data), static_cast(nbytes)); + Result len = out_->Write(static_cast(data), nbytes); if (!len.ok()) { return ToArrowStatus(len.status()); } + if (len.value() != nbytes) { + return arrow::Status::IOError( + fmt::format("expect write len {} mismatch actual write len {}", nbytes, len.value())); + } return arrow::Status::OK(); } diff --git a/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp b/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp index 04fba953..87c1e0b2 100644 --- a/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp @@ -60,7 +60,7 @@ TEST(ArrowStreamAdapterTest, TestInputAndOutputStream) { // in stream ASSERT_OK_AND_ASSIGN(std::shared_ptr in, file_system->Open(file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_unique(in, GetArrowPool(GetDefaultPool()), length); ASSERT_EQ(in_stream->GetSize().ValueOrDie(), static_cast(data.length())); diff --git a/src/paimon/common/utils/math.h b/src/paimon/common/utils/math.h index 575ed41e..54ad6cf7 100644 --- a/src/paimon/common/utils/math.h +++ b/src/paimon/common/utils/math.h @@ -34,6 +34,10 @@ #include #include +#include "fmt/format.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/status.h" + namespace paimon { template @@ -61,6 +65,23 @@ constexpr bool InRange(From value) { } } +template +Status ValidateValueInRange(From value, const char* value_name) { + if (PAIMON_UNLIKELY(!InRange(value))) { + return Status::Invalid(fmt::format("{} {} is out of bound of type {}", value_name, value, + OptionsUtils::GetTypeName())); + } + return Status::OK(); +} + +template +Status ValidateValueNonNegative(T value, const char* value_name) { + if (PAIMON_UNLIKELY(value < 0)) { + return Status::Invalid(fmt::format("{} {} is less than 0", value_name, value)); + } + return Status::OK(); +} + // Swaps between big and little endian. Can be used in combination with the // little-endian encoding/decoding functions in coding_lean.h and coding.h to // encode/decode big endian. diff --git a/src/paimon/common/utils/math_test.cpp b/src/paimon/common/utils/math_test.cpp index ae4a5037..36df1cba 100644 --- a/src/paimon/common/utils/math_test.cpp +++ b/src/paimon/common/utils/math_test.cpp @@ -22,6 +22,7 @@ #include #include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -74,6 +75,15 @@ TEST(MathTest, InRange) { ASSERT_FALSE(InRange(std::numeric_limits::max())); ASSERT_TRUE(InRange(std::numeric_limits::max())); ASSERT_FALSE(InRange(static_cast(std::numeric_limits::max()) + 1)); + + ASSERT_OK(ValidateValueInRange( + static_cast(std::numeric_limits::lowest()), "signed value")); + ASSERT_NOK_WITH_MSG(ValidateValueInRange(-1, "negative value"), + "negative value -1 is out of bound of type"); + + ASSERT_OK(ValidateValueNonNegative(0, "non-negative value")); + ASSERT_NOK_WITH_MSG(ValidateValueNonNegative(-1, "negative value"), + "negative value -1 is less than 0"); } } // namespace paimon::test diff --git a/src/paimon/common/utils/read_ahead_cache.cpp b/src/paimon/common/utils/read_ahead_cache.cpp index aa7e5cb6..b0001189 100644 --- a/src/paimon/common/utils/read_ahead_cache.cpp +++ b/src/paimon/common/utils/read_ahead_cache.cpp @@ -28,6 +28,7 @@ #include #include "paimon/common/utils/byte_range_combiner.h" +#include "paimon/common/utils/math.h" namespace paimon { @@ -126,18 +127,13 @@ Status ReadAheadCache::Impl::Init(std::vector&& ranges) { if (is_initialized_) { return Status::Invalid("Cache has already been initialized"); } - if (config_.GetRangeSizeLimit() > static_cast(std::numeric_limits::max())) { - return Status::Invalid("CacheConfig range_size_limit exceeds uint32_t max"); - } - PAIMON_ASSIGN_OR_RAISE( std::vector pending_ranges, ByteRangeCombiner::CoalesceByteRanges(std::move(ranges), config_.GetHoleSizeLimit(), config_.GetRangeSizeLimit())); for (const auto& pending_range : pending_ranges) { - if (pending_range.length > static_cast(std::numeric_limits::max())) { - return Status::Invalid("range length should not be larger than uint32_t max"); - } + PAIMON_RETURN_NOT_OK(ValidateValueInRange(pending_range.offset, "range offset")); + PAIMON_RETURN_NOT_OK(ValidateValueInRange(pending_range.length, "range length")); } pending_ranges_ = pending_ranges; is_cached_ = std::vector>(pending_ranges_.size()); @@ -228,8 +224,10 @@ std::vector ReadAheadCache::Impl::MakeCacheEntries( auto promise = std::make_shared>(); auto future = promise->get_future(); auto buffer = std::make_shared(range.length, memory_pool_.get()); + auto read_size = static_cast(buffer->size()); + auto read_offset = static_cast(range.offset); stream_->ReadAsync( - buffer->data(), static_cast(buffer->size()), range.offset, + buffer->data(), read_size, read_offset, [promise, buffer](Status status) mutable { promise->set_value(status); }); new_entries.emplace_back(range, std::move(buffer), std::move(future)); } diff --git a/src/paimon/common/utils/stream_utils.h b/src/paimon/common/utils/stream_utils.h index 5537c83b..b8b67088 100644 --- a/src/paimon/common/utils/stream_utils.h +++ b/src/paimon/common/utils/stream_utils.h @@ -46,11 +46,11 @@ class StreamUtils { static Result> ReadFully(std::unique_ptr input_stream, const std::shared_ptr& pool) { PAIMON_RETURN_NOT_OK(input_stream->Seek(0, FS_SEEK_SET)); - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length()); PAIMON_UNIQUE_PTR content = Bytes::AllocateBytes(file_length, pool.get()); - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, - input_stream->Read(content->data(), content->size())); - if (static_cast(actual_read_len) != file_length) { + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, + input_stream->Read(content->data(), file_length)); + if (actual_read_len != file_length) { return Status::Invalid("actual read length {}, not match with expect length {}", actual_read_len, file_length); } @@ -59,7 +59,7 @@ class StreamUtils { static Result> ReadAsyncFully( std::unique_ptr input_stream, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length()); PAIMON_UNIQUE_PTR content = Bytes::AllocateBytes(file_length, pool.get()); PAIMON_RETURN_NOT_OK(ReadAsyncFully(std::move(input_stream), content->data())); return content; @@ -67,10 +67,10 @@ class StreamUtils { static Status ReadAsyncFully(std::unique_ptr input_stream, char* content) { PAIMON_RETURN_NOT_OK(input_stream->Seek(0, FS_SEEK_SET)); - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length()); - uint64_t read_offset = 0; - uint32_t read_len = std::min(file_length, kDefaultReadChunkSize); + int64_t read_offset = 0; + int64_t read_len = std::min(file_length, kDefaultReadChunkSize); std::vector> futures; futures.reserve(file_length / kDefaultReadChunkSize + 1); while (read_len > 0) { @@ -96,7 +96,7 @@ class StreamUtils { } private: - static constexpr uint64_t kDefaultReadChunkSize = 1024 * 1024; + static constexpr int64_t kDefaultReadChunkSize = 1024 * 1024; }; } // namespace paimon diff --git a/src/paimon/common/utils/stream_utils_test.cpp b/src/paimon/common/utils/stream_utils_test.cpp index 9786a3eb..74c92fe9 100644 --- a/src/paimon/common/utils/stream_utils_test.cpp +++ b/src/paimon/common/utils/stream_utils_test.cpp @@ -49,8 +49,8 @@ class StreamUtilsTest : public ::testing::Test { std::string CreateTestFile(const std::string& content) { std::string file_path = dir_->Str() + "/test_file.txt"; EXPECT_OK_AND_ASSIGN(auto output_stream, file_system_->Create(file_path, true)); - EXPECT_OK_AND_ASSIGN(int32_t length, output_stream->Write(content.data(), content.size())); - EXPECT_EQ(length, static_cast(content.size())); + EXPECT_OK_AND_ASSIGN(int64_t length, output_stream->Write(content.data(), content.size())); + EXPECT_EQ(length, static_cast(content.size())); EXPECT_OK(output_stream->Close()); return file_path; } diff --git a/src/paimon/core/deletionvectors/bitmap_deletion_vector.cpp b/src/paimon/core/deletionvectors/bitmap_deletion_vector.cpp index 533d4acd..ed8c3ba2 100644 --- a/src/paimon/core/deletionvectors/bitmap_deletion_vector.cpp +++ b/src/paimon/core/deletionvectors/bitmap_deletion_vector.cpp @@ -21,6 +21,7 @@ #include "arrow/util/crc32.h" #include "fmt/format.h" #include "paimon/common/io/memory_segment_output_stream.h" +#include "paimon/common/utils/math.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/io/data_input_stream.h" @@ -29,10 +30,9 @@ namespace paimon { Result BitmapDeletionVector::SerializeTo(const std::shared_ptr& pool, DataOutputStream* out) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data, SerializeToBytes(pool)); - int64_t size = data->size(); - if (size < 0 || size > std::numeric_limits::max()) { - return Status::Invalid("BitmapDeletionVector serialize size out of range: ", size); - } + size_t size = data->size(); + PAIMON_RETURN_NOT_OK( + ValidateValueInRange(size, "BitmapDeletionVector serialize size")); PAIMON_RETURN_NOT_OK(out->WriteValue(static_cast(size))); PAIMON_RETURN_NOT_OK(out->WriteBytes(data)); uint32_t crc32 = 0; diff --git a/src/paimon/core/deletionvectors/deletion_file_writer.cpp b/src/paimon/core/deletionvectors/deletion_file_writer.cpp index 9942708d..32f1ce52 100644 --- a/src/paimon/core/deletionvectors/deletion_file_writer.cpp +++ b/src/paimon/core/deletionvectors/deletion_file_writer.cpp @@ -19,6 +19,7 @@ #include "paimon/core/deletionvectors/deletion_file_writer.h" #include "paimon/common/io/data_output_stream.h" +#include "paimon/common/utils/path_util.h" #include "paimon/core/deletionvectors/deletion_vectors_index_file.h" namespace paimon { @@ -49,10 +50,9 @@ Status DeletionFileWriter::Write(const std::string& key, } Result> DeletionFileWriter::GetResult() const { - int64_t length = output_bytes_; - if (length < 0 || length > std::numeric_limits::max()) { + if (output_bytes_ < 0 || output_bytes_ > std::numeric_limits::max()) { return Status::Invalid( - fmt::format("Deletion file result length {} out of int32 range.", length)); + fmt::format("Deletion file result length {} out of int32 range.", output_bytes_)); } std::optional final_path; if (is_external_path_) { @@ -60,8 +60,8 @@ Result> DeletionFileWriter::GetResult() const { final_path = external_path.ToString(); } return std::make_unique(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, - PathUtil::GetName(path_), length, dv_metas_.size(), - dv_metas_, final_path); + PathUtil::GetName(path_), output_bytes_, + dv_metas_.size(), dv_metas_, final_path); } } // namespace paimon diff --git a/src/paimon/core/deletionvectors/deletion_file_writer.h b/src/paimon/core/deletionvectors/deletion_file_writer.h index 9a04b235..5c48490c 100644 --- a/src/paimon/core/deletionvectors/deletion_file_writer.h +++ b/src/paimon/core/deletionvectors/deletion_file_writer.h @@ -21,9 +21,7 @@ #include #include -#include "fmt/format.h" #include "paimon/common/utils/linked_hash_map.h" -#include "paimon/common/utils/path_util.h" #include "paimon/core/deletionvectors/deletion_vector.h" #include "paimon/core/index/deletion_vector_meta.h" #include "paimon/core/index/index_path_factory.h" diff --git a/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp b/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp index 93777f03..09673d84 100644 --- a/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp +++ b/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp @@ -94,7 +94,7 @@ TEST(DeletionFileWriterTest, GetResultWithoutCloseShouldFail) { auto pool = GetDefaultPool(); ASSERT_OK_AND_ASSIGN(auto writer, DeletionFileWriter::Create(path_factory, fs, pool)); - ASSERT_NOK_WITH_MSG(writer->GetResult(), "result length -1 out of int32 range"); + ASSERT_NOK_WITH_MSG(writer->GetResult(), "Deletion file result length -1 out of int32 range"); } TEST(DeletionFileWriterTest, ExternalPathInResult) { diff --git a/src/paimon/core/index/index_file.h b/src/paimon/core/index/index_file.h index e76c19d9..f3829b4e 100644 --- a/src/paimon/core/index/index_file.h +++ b/src/paimon/core/index/index_file.h @@ -41,11 +41,11 @@ class IndexFile { return path_factory_->ToPath(file); } - virtual Result FileSize(const std::shared_ptr& file) const { + virtual Result FileSize(const std::shared_ptr& file) const { return FileSize(Path(file)); } - virtual Result FileSize(const std::string& file) const { + virtual Result FileSize(const std::string& file) const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, fs_->GetFileStatus(file)); return file_status->GetLen(); } diff --git a/src/paimon/core/manifest/manifest_committable_test.cpp b/src/paimon/core/manifest/manifest_committable_test.cpp index 007f3366..886ccedf 100644 --- a/src/paimon/core/manifest/manifest_committable_test.cpp +++ b/src/paimon/core/manifest/manifest_committable_test.cpp @@ -61,7 +61,7 @@ class ManifestCommittableTest : public testing::Test { EXPECT_OK_AND_ASSIGN(auto in_stream, file_system->Open(path)); EXPECT_OK_AND_ASSIGN( - [[maybe_unused]] int32_t read_bytes, + [[maybe_unused]] int64_t read_bytes, in_stream->Read(reinterpret_cast(buffer.data()), buffer.size())); EXPECT_OK(in_stream->Close()); diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp index 52f4dee4..2d686852 100644 --- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp +++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp @@ -53,7 +53,7 @@ Result> RemoteLookupFileManager::GenRemoteLookupFi // Get the file size from the local file system PAIMON_ASSIGN_OR_RAISE(std::unique_ptr local_file_status, file_system_->GetFileStatus(local_file_path)); - auto length = static_cast(local_file_status->GetLen()); + int64_t length = local_file_status->GetLen(); std::string remote_sst_name = lookup_levels->NewRemoteSst(file, length); std::string remote_sst_path = RemoteSstPath(file, remote_sst_name); @@ -106,24 +106,25 @@ Status RemoteLookupFileManager::CopyFromInputToOutput( std::unique_ptr&& input_stream, std::unique_ptr&& output_stream) const { auto buffer = std::make_shared(kBufferSize, pool_.get()); - PAIMON_ASSIGN_OR_RAISE(uint64_t total_length, input_stream->Length()); - uint64_t write_size = 0; + PAIMON_ASSIGN_OR_RAISE(int64_t total_length, input_stream->Length()); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(total_length, "input stream length")); + int64_t write_size = 0; while (write_size < total_length) { - uint64_t current_read_size = std::min(total_length - write_size, kBufferSize); - PAIMON_ASSIGN_OR_RAISE(int32_t bytes_read, + int64_t current_read_size = + std::min(total_length - write_size, static_cast(kBufferSize)); + PAIMON_ASSIGN_OR_RAISE(int64_t bytes_read, input_stream->Read(buffer->data(), current_read_size)); - if (static_cast(bytes_read) != current_read_size) { - return Status::Invalid( - fmt::format("CopyFromInputToOutput failed: expected read {} bytes, while " - "actual read {} bytes", - current_read_size, bytes_read)); + if (bytes_read != current_read_size) { + return Status::Invalid(fmt::format( + "CopyFromInputToOutput failed: expected read {} bytes, while actual read {} bytes", + current_read_size, bytes_read)); } - PAIMON_ASSIGN_OR_RAISE(int32_t bytes_written, + PAIMON_ASSIGN_OR_RAISE(int64_t bytes_written, output_stream->Write(buffer->data(), bytes_read)); if (bytes_written != bytes_read) { return Status::Invalid( - fmt::format("CopyFromInputToOutput failed: expected write {} bytes, while " - "actual write {} bytes", + fmt::format("CopyFromInputToOutput failed: expected write {} bytes, while actual " + "write {} bytes", bytes_read, bytes_written)); } write_size += current_read_size; diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp index 54ed805d..cd8f3e18 100644 --- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp +++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp @@ -277,9 +277,9 @@ TEST_F(RemoteLookupFileManagerTest, TryToDownloadLargeFileAcrossMultipleBuffers) for (uint64_t i = 0; i < file_size; ++i) { write_buffer[i] = static_cast(i % 251); } - ASSERT_OK_AND_ASSIGN(int32_t bytes_written, + ASSERT_OK_AND_ASSIGN(int64_t bytes_written, output_stream->Write(write_buffer.data(), file_size)); - ASSERT_EQ(static_cast(bytes_written), file_size); + ASSERT_EQ(bytes_written, file_size); ASSERT_OK(output_stream->Flush()); ASSERT_OK(output_stream->Close()); } @@ -297,8 +297,8 @@ TEST_F(RemoteLookupFileManagerTest, TryToDownloadLargeFileAcrossMultipleBuffers) { ASSERT_OK_AND_ASSIGN(auto input_stream, fs_->Open(local_file_path)); std::vector read_buffer(file_size); - ASSERT_OK_AND_ASSIGN(int32_t bytes_read, input_stream->Read(read_buffer.data(), file_size)); - ASSERT_EQ(static_cast(bytes_read), file_size); + ASSERT_OK_AND_ASSIGN(int64_t bytes_read, input_stream->Read(read_buffer.data(), file_size)); + ASSERT_EQ(bytes_read, file_size); for (uint64_t i = 0; i < file_size; ++i) { ASSERT_EQ(read_buffer[i], static_cast(i % 251)) << "Data mismatch at byte offset " << i; diff --git a/src/paimon/core/mergetree/spill_reader.cpp b/src/paimon/core/mergetree/spill_reader.cpp index f4bb9c1d..4b5d3f23 100644 --- a/src/paimon/core/mergetree/spill_reader.cpp +++ b/src/paimon/core/mergetree/spill_reader.cpp @@ -54,7 +54,7 @@ Status SpillReader::Open(const FileIOChannel::ID& channel_id) { const std::string& file_path = channel_id.GetPath(); PAIMON_ASSIGN_OR_RAISE(in_stream_, fs_->Open(file_path)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, fs_->GetFileStatus(file_path)); - uint64_t file_len = file_status->GetLen(); + int64_t file_len = file_status->GetLen(); arrow_input_stream_adapter_ = std::make_shared(in_stream_, arrow_pool_, file_len); auto ipc_read_options = arrow::ipc::IpcReadOptions::Defaults(); diff --git a/src/paimon/core/mergetree/spill_writer.cpp b/src/paimon/core/mergetree/spill_writer.cpp index 3e9b917c..170057fc 100644 --- a/src/paimon/core/mergetree/spill_writer.cpp +++ b/src/paimon/core/mergetree/spill_writer.cpp @@ -117,7 +117,7 @@ Result SpillWriter::GetFileSize() const { } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, fs_->GetFileStatus(channel_id_.GetPath())); - return static_cast(file_status->GetLen()); + return file_status->GetLen(); } const FileIOChannel::ID& SpillWriter::GetChannelId() const { diff --git a/src/paimon/core/mergetree/write_buffer_test.cpp b/src/paimon/core/mergetree/write_buffer_test.cpp index 0f082737..26f02933 100644 --- a/src/paimon/core/mergetree/write_buffer_test.cpp +++ b/src/paimon/core/mergetree/write_buffer_test.cpp @@ -102,7 +102,7 @@ class WriteBufferTest : public ::testing::Test { if (spill_files.size() != 1 || spill_files[0]->IsDir()) { return Status::Invalid("expected exactly one spill file"); } - return static_cast(spill_files[0]->GetLen()); + return spill_files[0]->GetLen(); } Result ReadReaderResult(KeyValueRecordReader* reader) const { diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index 34b44f33..63e58062 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -256,7 +256,7 @@ class FileStoreCommitImplTest : public testing::Test { EXPECT_OK_AND_ASSIGN(std::unique_ptr in_stream, file_system->Open(path)); EXPECT_TRUE(in_stream); EXPECT_OK_AND_ASSIGN( - [[maybe_unused]] int32_t length, + [[maybe_unused]] int64_t length, in_stream->Read(reinterpret_cast(buffer.data()), buffer.size())); EXPECT_OK(in_stream->Close()); auto pool = GetDefaultPool(); diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp index d2d2f538..2a8aa53c 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader.cpp @@ -18,6 +18,7 @@ #include "paimon/format/avro/avro_file_batch_reader.h" +#include #include #include diff --git a/src/paimon/format/avro/avro_input_stream_impl.cpp b/src/paimon/format/avro/avro_input_stream_impl.cpp index b1d6a01c..d2dfddc6 100644 --- a/src/paimon/format/avro/avro_input_stream_impl.cpp +++ b/src/paimon/format/avro/avro_input_stream_impl.cpp @@ -27,6 +27,7 @@ #include #include "avro/Exception.hh" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" @@ -36,9 +37,10 @@ namespace paimon::avro { Result> AvroInputStreamImpl::Create( const std::shared_ptr& input_stream, size_t buffer_size, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(uint64_t length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream->Length()); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "file length")); return std::unique_ptr( - new AvroInputStreamImpl(input_stream, buffer_size, length, pool)); + new AvroInputStreamImpl(input_stream, buffer_size, static_cast(length), pool)); } AvroInputStreamImpl::AvroInputStreamImpl(const std::shared_ptr& input_stream, @@ -69,9 +71,12 @@ bool AvroInputStreamImpl::next(const uint8_t** data, size_t* len) { if (remaining == 0) { return false; // eof } - auto read_length = - in_->Read(reinterpret_cast(buffer_), - static_cast(std::min(buffer_size_, remaining))); + uint64_t read_size = std::min(buffer_size_, remaining); + Status status = ValidateValueInRange(read_size, "read length"); + if (!status.ok()) { + throw ::avro::Exception("Read failed: {}", status.ToString()); + } + auto read_length = in_->Read(reinterpret_cast(buffer_), static_cast(read_size)); if (!read_length.ok()) { throw ::avro::Exception("Read failed: {}", read_length.status().ToString()); } diff --git a/src/paimon/format/avro/avro_output_stream_impl.cpp b/src/paimon/format/avro/avro_output_stream_impl.cpp index d70d2f97..6e3ce32e 100644 --- a/src/paimon/format/avro/avro_output_stream_impl.cpp +++ b/src/paimon/format/avro/avro_output_stream_impl.cpp @@ -22,6 +22,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" @@ -64,11 +65,16 @@ void AvroOutputStreamImpl::backup(size_t len) { void AvroOutputStreamImpl::FlushBuffer() { size_t length = buffer_size_ - available_; - Result write_len = out_->Write(reinterpret_cast(buffer_), length); + Status validate_status = ValidateValueInRange(length, "write length"); + if (!validate_status.ok()) { + throw std::runtime_error("write failed, status: " + validate_status.ToString()); + } + Result write_len = + out_->Write(reinterpret_cast(buffer_), static_cast(length)); if (!write_len.ok()) { throw std::runtime_error("write failed, status: " + write_len.status().ToString()); } - if (static_cast(write_len.value()) != length) { + if (write_len.value() != static_cast(length)) { throw std::runtime_error( fmt::format("write failed, expected length: {}, actual write length: {}", length, write_len.value())); diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index 52df7979..4cddf625 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -52,12 +52,12 @@ Result> BlobFileBatchReader::Create( batch_size)); } - PAIMON_ASSIGN_OR_RAISE(uint64_t file_size, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, input_stream->Length()); PAIMON_RETURN_NOT_OK( input_stream->Seek(file_size - BlobDefs::kBlobFileHeaderLength, FS_SEEK_SET)); int8_t header[BlobDefs::kBlobFileHeaderLength]; PAIMON_ASSIGN_OR_RAISE( - int32_t actual_size, + int64_t actual_size, input_stream->Read(reinterpret_cast(header), BlobDefs::kBlobFileHeaderLength)); if (actual_size != BlobDefs::kBlobFileHeaderLength) { return Status::Invalid( diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index e5e3c8cf..aa8dd248 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -179,19 +179,19 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { } else { in = std::make_unique(blob_data.data(), blob_data.size()); } - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, in->Length()); - uint64_t total_read_length = 0; - auto read_len = static_cast(std::min(file_length, tmp_buffer_->size())); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, in->Length()); + int64_t total_read_length = 0; + int64_t read_len = std::min(file_length, static_cast(tmp_buffer_->size())); while (read_len > 0) { - PAIMON_ASSIGN_OR_RAISE(int32_t actual_read_len, in->Read(tmp_buffer_->data(), read_len)); - if (static_cast(actual_read_len) != read_len) { + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, in->Read(tmp_buffer_->data(), read_len)); + if (actual_read_len != read_len) { return Status::Invalid("actual read length {}, not match with expect length {}", actual_read_len, read_len); } PAIMON_RETURN_NOT_OK(WriteWithCrc32(tmp_buffer_->data(), actual_read_len)); total_read_length += actual_read_len; - read_len = static_cast( - std::min(file_length - total_read_length, tmp_buffer_->size())); + read_len = + std::min(file_length - total_read_length, static_cast(tmp_buffer_->size())); } // write bin length @@ -211,8 +211,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { return Status::OK(); } -Status BlobFormatWriter::WriteBytes(const char* data, int32_t length) { - PAIMON_ASSIGN_OR_RAISE(int32_t actual, out_->Write(data, length)); +Status BlobFormatWriter::WriteBytes(const char* data, int64_t length) { + PAIMON_ASSIGN_OR_RAISE(int64_t actual, out_->Write(data, length)); if (actual != length) { return Status::Invalid("not suppose actual length {} not match with expect {}", actual, length); @@ -220,7 +220,7 @@ Status BlobFormatWriter::WriteBytes(const char* data, int32_t length) { return Status::OK(); } -Status BlobFormatWriter::WriteWithCrc32(const char* data, int32_t length) { +Status BlobFormatWriter::WriteWithCrc32(const char* data, int64_t length) { crc32_ = arrow::internal::crc32(crc32_, data, length); return WriteBytes(data, length); } diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 7680611d..b586e0d4 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -81,8 +81,8 @@ class BlobFormatWriter : public FormatWriter { Status WriteBlob(std::string_view blob_data); - Status WriteBytes(const char* data, int32_t length); - Status WriteWithCrc32(const char* data, int32_t length); + Status WriteBytes(const char* data, int64_t length); + Status WriteWithCrc32(const char* data, int64_t length); template static PAIMON_UNIQUE_PTR IntegerToLittleEndian(T value, diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index 9a5f013e..81ee781f 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -319,7 +319,7 @@ TEST_P(BlobFormatWriterTest, TestEmptyWriter) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system_->Open(dir_->Str() + "/file.blob")); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(uint64_t file_length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length()); ASSERT_EQ(file_length, 5); // Should have footer even if no data std::vector buffer(file_length); ASSERT_OK_AND_ASSIGN(auto read_length, input_stream->Read(buffer.data(), buffer.size())); @@ -341,7 +341,7 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { // Write data larger than TMP_BUFFER_SIZE (1MB) const size_t large_size = BlobFormatWriter::kTmpBufferSize * 2 + 1000; // ~2MB std::vector large_data(large_size, 'A'); - ASSERT_OK_AND_ASSIGN(int32_t written, large_file_stream->Write(large_data.data(), large_size)); + ASSERT_OK_AND_ASSIGN(int64_t written, large_file_stream->Write(large_data.data(), large_size)); ASSERT_EQ(written, large_size); ASSERT_OK(large_file_stream->Flush()); ASSERT_OK(large_file_stream->Close()); @@ -465,7 +465,7 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system_->Open(dir_->Str() + "/file.blob")); ASSERT_TRUE(input_stream); - ASSERT_OK_AND_ASSIGN(uint64_t file_length, input_stream->Length()); + ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length()); ASSERT_EQ(file_length, 22); std::vector buffer(file_length); ASSERT_OK_AND_ASSIGN(auto read_length, diff --git a/src/paimon/format/orc/orc_input_stream_impl.cpp b/src/paimon/format/orc/orc_input_stream_impl.cpp index bc5dec27..572ae814 100644 --- a/src/paimon/format/orc/orc_input_stream_impl.cpp +++ b/src/paimon/format/orc/orc_input_stream_impl.cpp @@ -24,6 +24,7 @@ #include "fmt/format.h" #include "orc/Exceptions.hh" #include "orc/Reader.hh" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/status.h" @@ -31,9 +32,10 @@ namespace paimon::orc { Result> OrcInputStreamImpl::Create( const std::shared_ptr& input_stream, uint64_t natural_read_size) { PAIMON_ASSIGN_OR_RAISE(std::string name, input_stream->GetUri()); - PAIMON_ASSIGN_OR_RAISE(uint64_t length, input_stream->Length()); - return std::unique_ptr( - new OrcInputStreamImpl(input_stream, name, length, natural_read_size)); + PAIMON_ASSIGN_OR_RAISE(int64_t length, input_stream->Length()); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "file length")); + return std::unique_ptr(new OrcInputStreamImpl( + input_stream, name, static_cast(length), natural_read_size)); } OrcInputStreamImpl::OrcInputStreamImpl(const std::shared_ptr& input_stream, @@ -63,11 +65,20 @@ void OrcInputStreamImpl::read(void* buf, uint64_t length, uint64_t offset) { metrics_->IOCount.fetch_add(1); } - Result read_bytes = input_stream_->Read(static_cast(buf), length, offset); + Status status = ValidateValueInRange(length, "read length"); + if (!status.ok()) { + throw ::orc::ParseError(status.ToString()); + } + status = ValidateValueInRange(offset, "read offset"); + if (!status.ok()) { + throw ::orc::ParseError(status.ToString()); + } + Result read_bytes = input_stream_->Read( + static_cast(buf), static_cast(length), static_cast(offset)); if (!read_bytes.ok()) { throw ::orc::ParseError("read failed, status: " + read_bytes.status().ToString()); } - if (static_cast(read_bytes.value()) != length) { + if (read_bytes.value() != static_cast(length)) { throw ::orc::ParseError( fmt::format("read failed, expected length: {}, actual read length: {}", length, read_bytes.value())); diff --git a/src/paimon/format/orc/orc_output_stream_impl.cpp b/src/paimon/format/orc/orc_output_stream_impl.cpp index 1168e457..abe92edb 100644 --- a/src/paimon/format/orc/orc_output_stream_impl.cpp +++ b/src/paimon/format/orc/orc_output_stream_impl.cpp @@ -23,6 +23,7 @@ #include #include "fmt/format.h" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/status.h" @@ -44,17 +45,26 @@ uint64_t OrcOutputStreamImpl::getLength() const { if (!pos.ok()) { throw std::runtime_error(fmt::format("get length failed, file name {}, error msg {}", file_name_, pos.status().ToString())); - } else { - return pos.value(); } + Status status = ValidateValueInRange(pos.value(), "file position"); + if (!status.ok()) { + throw std::runtime_error(fmt::format("get length failed, file name {}, error msg {}", + file_name_, status.ToString())); + } + return static_cast(pos.value()); } void OrcOutputStreamImpl::write(const void* buf, size_t length) { - Result write_len = output_stream_->Write(static_cast(buf), length); + Status status = ValidateValueInRange(length, "write length"); + if (!status.ok()) { + throw std::runtime_error("write failed, status: " + status.ToString()); + } + Result write_len = + output_stream_->Write(static_cast(buf), static_cast(length)); if (!write_len.ok()) { throw std::runtime_error("write failed, status: " + write_len.status().ToString()); } - if (static_cast(write_len.value()) != length) { + if (write_len.value() != static_cast(length)) { throw std::runtime_error( fmt::format("write failed, expected length: {}, actual write length: {}", length, write_len.value())); diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp index f5d84389..8129e8f3 100644 --- a/src/paimon/format/parquet/column_index_filter_test.cpp +++ b/src/paimon/format/parquet/column_index_filter_test.cpp @@ -251,7 +251,7 @@ class ColumnIndexFilterTest : public ::testing::Test { // Open as raw ParquetFileReader ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name_)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, arrow_pool_, length); parquet_reader_ = ::parquet::ParquetFileReader::Open(in_stream); ASSERT_TRUE(parquet_reader_); diff --git a/src/paimon/format/parquet/file_reader_wrapper_test.cpp b/src/paimon/format/parquet/file_reader_wrapper_test.cpp index c8bf5add..b18d0277 100644 --- a/src/paimon/format/parquet/file_reader_wrapper_test.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper_test.cpp @@ -120,7 +120,7 @@ class FileReaderWrapperTest : public ::testing::Test { Result> PrepareReaderWrapper( const std::string& file_path, int64_t wrapper_batch_size = 0) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr in, fs_->Open(file_path)); - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, in->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, in->Length()); auto input_stream = std::make_unique(in, arrow_pool_, file_length); ::parquet::arrow::FileReaderBuilder file_reader_builder; ::parquet::ReaderProperties reader_properties; diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 11cb0ddb..041839ba 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -116,7 +116,7 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { std::shared_ptr* out, int32_t batch_size = 1024) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, arrow_pool_, length); std::map options; @@ -519,7 +519,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesPartialMatch) { // Open as raw ParquetFileReader ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, arrow_pool_, length); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); ASSERT_TRUE(parquet_reader); @@ -544,7 +544,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesAllMatch) { WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, arrow_pool_, length); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); @@ -570,7 +570,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesNoMatch) { WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, arrow_pool_, length); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); @@ -589,7 +589,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiColumn) { WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, arrow_pool_, length); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); @@ -615,7 +615,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) { WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, arrow_pool_, length); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); diff --git a/src/paimon/format/parquet/parquet_reader_builder.h b/src/paimon/format/parquet/parquet_reader_builder.h index c0d99315..dadbb8cf 100644 --- a/src/paimon/format/parquet/parquet_reader_builder.h +++ b/src/paimon/format/parquet/parquet_reader_builder.h @@ -45,7 +45,7 @@ class ParquetReaderBuilder : public ReaderBuilder { Result> Build( const std::shared_ptr& path) const override { - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, path->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, path->Length()); std::shared_ptr arrow_pool = GetArrowPool(pool_); auto input_stream = std::make_unique(path, arrow_pool, file_length); diff --git a/src/paimon/format/parquet/parquet_stats_extractor.cpp b/src/paimon/format/parquet/parquet_stats_extractor.cpp index 8f8b82ad..f5789161 100644 --- a/src/paimon/format/parquet/parquet_stats_extractor.cpp +++ b/src/paimon/format/parquet/parquet_stats_extractor.cpp @@ -275,7 +275,7 @@ ParquetStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& fi const std::shared_ptr& pool) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr input_stream, file_system->Open(path)); assert(input_stream); - PAIMON_ASSIGN_OR_RAISE(uint64_t file_length, input_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length()); std::shared_ptr parquet_memory_pool = GetArrowPool(pool); auto parquet_input_file = std::make_shared( std::move(input_stream), parquet_memory_pool, file_length); diff --git a/src/paimon/format/parquet/predicate_pushdown_test.cpp b/src/paimon/format/parquet/predicate_pushdown_test.cpp index 7ccbaa2c..4399f21b 100644 --- a/src/paimon/format/parquet/predicate_pushdown_test.cpp +++ b/src/paimon/format/parquet/predicate_pushdown_test.cpp @@ -106,7 +106,7 @@ class PredicatePushdownTest : public ::testing::Test { uint32_t predicate_node_count_limit = paimon::parquet::DEFAULT_PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name_)); - ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, arrow_pool_, length); std::map options; diff --git a/src/paimon/fs/jindo/jindo_file_status.h b/src/paimon/fs/jindo/jindo_file_status.h index 5ac14cfd..bc291eb0 100644 --- a/src/paimon/fs/jindo/jindo_file_status.h +++ b/src/paimon/fs/jindo/jindo_file_status.h @@ -49,7 +49,7 @@ class JindoFileStatus : public FileStatus { return file_info_.getPath(); } - uint64_t GetLen() const override { + int64_t GetLen() const override { return file_info_.getLength(); } diff --git a/src/paimon/fs/jindo/jindo_file_system.cpp b/src/paimon/fs/jindo/jindo_file_system.cpp index afd5a383..aaa6fa90 100644 --- a/src/paimon/fs/jindo/jindo_file_system.cpp +++ b/src/paimon/fs/jindo/jindo_file_system.cpp @@ -27,6 +27,7 @@ #include "JdoStatus.hpp" // NOLINT(build/include_subdir) #include "fmt/format.h" #include "jdo_error.h" // NOLINT(build/include_subdir) +#include "paimon/common/utils/math.h" #include "paimon/fs/jindo/jindo_file_status.h" #include "paimon/fs/jindo/jindo_utils.h" @@ -192,7 +193,7 @@ Status JindoInputStream::Seek(int64_t offset, SeekOrigin origin) { PAIMON_ASSIGN_OR_RAISE(int64_t pos, GetPos()); PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->seek(offset + pos)); } else if (origin == FS_SEEK_END) { - PAIMON_ASSIGN_OR_RAISE(uint64_t len, Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t len, Length()); PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->seek(len + offset)); } else { return Status::Invalid("unsupported seek origin"); @@ -203,33 +204,48 @@ Status JindoInputStream::Seek(int64_t offset, SeekOrigin origin) { Result JindoInputStream::GetPos() const { int64_t pos = -1; PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->tell(pos)); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(pos, "jindo input position")); return pos; } -Result JindoInputStream::Length() const { +Result JindoInputStream::Length() const { int64_t len = -1; PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->getFileLength(len)); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(len, "jindo input length")); return len; } -Result JindoInputStream::Read(char* buffer, uint32_t size) { +Result JindoInputStream::Read(char* buffer, int64_t size) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length")); PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->read(size, &result_, buffer)); return result_.length(); } -Result JindoInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { +Result JindoInputStream::Read(char* buffer, int64_t size, int64_t offset) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length")); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(offset, "read offset")); PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->pread(offset, size, &result_, buffer)); return result_.length(); } -void JindoInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void JindoInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { + Status validate_status = ValidateValueNonNegative(size, "read length"); + if (!validate_status.ok()) { + callback(validate_status); + return; + } + validate_status = ValidateValueNonNegative(offset, "read offset"); + if (!validate_status.ok()) { + callback(validate_status); + return; + } auto outer_callback = [=](JdoStatus status) { callback(status.ok() ? Status::OK() : Status::IOError(status.errMsg())); }; auto task = reader_->preadAsync(offset, size, &result_, buffer, outer_callback); assert(task); - auto status = task->perform(); + [[maybe_unused]] auto perform_status = task->perform(); } Status JindoInputStream::Close() { @@ -250,10 +266,12 @@ JindoOutputStream::JindoOutputStream(const std::shared_ptr& Result JindoOutputStream::GetPos() const { int64_t pos = -1; PAIMON_RETURN_NOT_OK_FROM_JINDO(writer_->tell(pos)); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(pos, "jindo output position")); return pos; } -Result JindoOutputStream::Write(const char* buffer, uint32_t size) { +Result JindoOutputStream::Write(const char* buffer, int64_t size) { + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "write length")); std::string_view data(buffer, size); PAIMON_RETURN_NOT_OK_FROM_JINDO(writer_->write(data)); return size; diff --git a/src/paimon/fs/jindo/jindo_file_system.h b/src/paimon/fs/jindo/jindo_file_system.h index ce5d1edf..f3dccc04 100644 --- a/src/paimon/fs/jindo/jindo_file_system.h +++ b/src/paimon/fs/jindo/jindo_file_system.h @@ -69,13 +69,13 @@ class JindoInputStream : public InputStream { std::unique_ptr&& reader); Status Seek(int64_t offset, SeekOrigin origin) override; Result GetPos() const override; - Result Read(char* buffer, uint32_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + Result Read(char* buffer, int64_t size) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; Status Close() override; Result GetUri() const override; - Result Length() const override; + Result Length() const override; private: // The lifecycle of the fs used to create the Jindo Reader must be longer than the lifecycle of @@ -91,7 +91,7 @@ class JindoOutputStream : public OutputStream { std::unique_ptr&& writer); Result GetPos() const override; - Result Write(const char* buffer, uint32_t size) override; + Result Write(const char* buffer, int64_t size) override; Status Flush() override; Status Close() override; Result GetUri() const override; diff --git a/src/paimon/fs/jindo/jindo_file_system_test.cpp b/src/paimon/fs/jindo/jindo_file_system_test.cpp index 39b8b577..437e4ba4 100644 --- a/src/paimon/fs/jindo/jindo_file_system_test.cpp +++ b/src/paimon/fs/jindo/jindo_file_system_test.cpp @@ -52,7 +52,7 @@ TEST_F(JindoFileSystemTest, TestLifeCycle) { // read process ASSERT_OK_AND_ASSIGN(auto in_stream, tmp_fs->Open(file_path)); std::string read_content(content.size(), '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ(content, read_content); @@ -79,7 +79,7 @@ TEST_F(JindoFileSystemTest, TestSeek) { std::string file_path = test_dir_ + "file.data"; // write process ASSERT_OK_AND_ASSIGN(auto out_stream, fs_->Create(file_path, /*overwrite=*/true)); - ASSERT_OK_AND_ASSIGN(int32_t write_len, out_stream->Write(content.data(), content.size())); + ASSERT_OK_AND_ASSIGN(int64_t write_len, out_stream->Write(content.data(), content.size())); ASSERT_EQ(write_len, content.size()); ASSERT_OK(out_stream->Flush()); ASSERT_OK(out_stream->Close()); @@ -109,7 +109,7 @@ TEST_F(JindoFileSystemTest, TestSeek) { // read from cur pos std::string read_content(3, '\0'); - ASSERT_OK_AND_ASSIGN(int32_t read_len, + ASSERT_OK_AND_ASSIGN(int64_t read_len, in_stream->Read(read_content.data(), read_content.size())); ASSERT_EQ(read_len, read_content.size()); ASSERT_EQ("ijk", read_content); diff --git a/src/paimon/fs/local/local_file.cpp b/src/paimon/fs/local/local_file.cpp index 0291c59d..c603154c 100644 --- a/src/paimon/fs/local/local_file.cpp +++ b/src/paimon/fs/local/local_file.cpp @@ -28,6 +28,7 @@ #include "fmt/format.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/utils/math.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/fs/local/local_file_status.h" @@ -171,7 +172,7 @@ Result> LocalFile::GetFileStatus() const { S_ISDIR(buf.st_mode)); } -Result LocalFile::Length() const { +Result LocalFile::Length() const { CHECK_HOOK(); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, GetFileStatus()); return file_status->GetLen(); @@ -197,18 +198,16 @@ const std::string& LocalFile::GetPath() const { return path_; } -Result LocalFile::Read(char* buffer, uint32_t length, uint64_t offset) { +Result LocalFile::Read(char* buffer, int64_t length, int64_t offset) { if (file_) { CHECK_HOOK(); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "read length")); + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(offset, "read offset")); int32_t fd = fileno(file_); - auto more = static_cast(length); - if (more < 0) { - return Status::IOError(fmt::format( - "pread file '{}' fail, length overflow int32_t, ec: EC_BADARGS", path_)); - } - uint64_t off = 0; - int32_t ret = 0; + int64_t more = length; + int64_t off = 0; + int64_t ret = 0; while (more > 0) { ret = ::pread(fd, buffer + off, more, offset + off); if (ret == -1) { @@ -227,17 +226,14 @@ Result LocalFile::Read(char* buffer, uint32_t length, uint64_t offset) "read file '{}' fail, can not read file which is opened fail, ec: EBADF", path_)); } -Result LocalFile::Read(char* buffer, uint32_t length) { +Result LocalFile::Read(char* buffer, int64_t length) { if (file_) { CHECK_HOOK(); - auto more = static_cast(length); - if (more < 0) { - return Status::IOError( - fmt::format("fileName '{}', length '{}', ec: EC_BADARGS", path_, length)); - } + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "read length")); - int32_t ret = 0; - uint64_t off = 0; + int64_t more = length; + int64_t ret = 0; + int64_t off = 0; while (more > 0) { ret = fread(buffer + off, 1, more, file_); if (ferror(file_) != 0) { @@ -257,23 +253,25 @@ Result LocalFile::Read(char* buffer, uint32_t length) { "read file '{}' fail, can not read file which is opened fail, ec: EBADF", path_)); } -Result LocalFile::Write(const char* buffer, uint32_t length) { +Result LocalFile::Write(const char* buffer, int64_t length) { if (file_) { CHECK_HOOK(); - auto more = static_cast(length); - if (more < 0) { - return Status::IOError(fmt::format( - "write file '{}' fail, length overflow int32_t, ec: EC_BADARGS", path_)); - } + PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(length, "write length")); - int32_t ret = 0; - uint64_t off = 0; + int64_t more = length; + int64_t ret = 0; + int64_t off = 0; while (more > 0) { ret = fwrite(buffer + off, 1, more, file_); if (ferror(file_) != 0) { return Status::IOError(fmt::format("write file '{}' fail at off {}, ec: {}", path_, off, std::strerror(errno))); } + if (ret == 0) { + return Status::IOError( + fmt::format("write file '{}' fail at off {}, wrote zero bytes, ec: {}", path_, + off, std::strerror(errno))); + } more -= ret; off += ret; } diff --git a/src/paimon/fs/local/local_file.h b/src/paimon/fs/local/local_file.h index 655c4141..7ebfd13f 100644 --- a/src/paimon/fs/local/local_file.h +++ b/src/paimon/fs/local/local_file.h @@ -53,12 +53,12 @@ class LocalFile { std::unique_ptr GetParentFile() const; Result Mkdir() const; Result> GetFileStatus() const; - Result Length() const; + Result Length() const; Result LastModifiedTimeMs() const; Status OpenFile(bool is_read_file); - Result Read(char* buffer, uint32_t length); - Result Read(char* buffer, uint32_t length, uint64_t offset); - Result Write(const char* buffer, uint32_t length); + Result Read(char* buffer, int64_t length); + Result Read(char* buffer, int64_t length, int64_t offset); + Result Write(const char* buffer, int64_t length); Status Flush(); Status Close(); Status Seek(int64_t offset, int32_t seek_origin); diff --git a/src/paimon/fs/local/local_file_status.h b/src/paimon/fs/local/local_file_status.h index eb1af1fd..8987d0be 100644 --- a/src/paimon/fs/local/local_file_status.h +++ b/src/paimon/fs/local/local_file_status.h @@ -43,7 +43,7 @@ class LocalBasicFileStatus : public BasicFileStatus { class LocalFileStatus : public FileStatus { public: - LocalFileStatus(const std::string& path, uint64_t length, int64_t last_modification_time, + LocalFileStatus(const std::string& path, int64_t length, int64_t last_modification_time, bool is_dir) : path_(path), length_(length), @@ -54,7 +54,7 @@ class LocalFileStatus : public FileStatus { return path_; } - uint64_t GetLen() const override { + int64_t GetLen() const override { return length_; } @@ -68,7 +68,7 @@ class LocalFileStatus : public FileStatus { private: std::string path_; - uint64_t length_; + int64_t length_; int64_t last_modification_time_; bool is_dir_; }; diff --git a/src/paimon/fs/local/local_file_system.cpp b/src/paimon/fs/local/local_file_system.cpp index 258a39f4..917506be 100644 --- a/src/paimon/fs/local/local_file_system.cpp +++ b/src/paimon/fs/local/local_file_system.cpp @@ -256,37 +256,37 @@ Result LocalInputStream::GetPos() const { return file_->Tell(); } -Result LocalInputStream::Read(char* buffer, uint32_t size) { - PAIMON_ASSIGN_OR_RAISE(int32_t read_length, file_->Read(buffer, size)); - if (read_length != static_cast(size)) { +Result LocalInputStream::Read(char* buffer, int64_t size) { + PAIMON_ASSIGN_OR_RAISE(int64_t read_length, file_->Read(buffer, size)); + if (read_length != size) { return Status::IOError(fmt::format("file '{}' read size {} != expected {}", file_->GetPath(), read_length, size)); } return read_length; } -Result LocalInputStream::Read(char* buffer, uint32_t size, uint64_t offset) { - PAIMON_ASSIGN_OR_RAISE(int32_t read_length, file_->Read(buffer, size, offset)); - if (read_length != static_cast(size)) { +Result LocalInputStream::Read(char* buffer, int64_t size, int64_t offset) { + PAIMON_ASSIGN_OR_RAISE(int64_t read_length, file_->Read(buffer, size, offset)); + if (read_length != size) { return Status::IOError(fmt::format("file '{}' read size {} != expected {}", file_->GetPath(), read_length, size)); } return read_length; } -void LocalInputStream::ReadAsync(char* buffer, uint32_t size, uint64_t offset, +void LocalInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) { - Result read_size = Read(buffer, size, offset); + Result read_size = Read(buffer, size, offset); Status status = Status::OK(); if (!read_size.ok()) { status = read_size.status(); } else { - assert(read_size.value() == static_cast(size)); + assert(read_size.value() == size); } callback(status); } -Result LocalInputStream::Length() const { +Result LocalInputStream::Length() const { return file_->Length(); } @@ -306,7 +306,7 @@ LocalOutputStream::LocalOutputStream(std::unique_ptr&& file) : file_( Result LocalOutputStream::GetPos() const { return file_->Tell(); } -Result LocalOutputStream::Write(const char* buffer, uint32_t size) { +Result LocalOutputStream::Write(const char* buffer, int64_t size) { return file_->Write(buffer, size); } Status LocalOutputStream::Flush() { diff --git a/src/paimon/fs/local/local_file_system.h b/src/paimon/fs/local/local_file_system.h index 3a4bc7ab..748f2463 100644 --- a/src/paimon/fs/local/local_file_system.h +++ b/src/paimon/fs/local/local_file_system.h @@ -69,16 +69,16 @@ class LocalInputStream : public InputStream { Status Seek(int64_t offset, SeekOrigin origin) override; Result GetPos() const override; - Result Read(char* buffer, uint32_t size) override; - Result Read(char* buffer, uint32_t size, uint64_t offset) override; - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + Result Read(char* buffer, int64_t size) override; + Result Read(char* buffer, int64_t size, int64_t offset) override; + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override; Status Close() override; Result GetUri() const override { return file_->GetPath(); } - Result Length() const override; + Result Length() const override; private: explicit LocalInputStream(std::unique_ptr&& file); @@ -91,7 +91,7 @@ class LocalOutputStream : public OutputStream { static Result> Create(std::unique_ptr file); Result GetPos() const override; - Result Write(const char* buffer, uint32_t size) override; + Result Write(const char* buffer, int64_t size) override; Status Flush() override; Status Close() override; Result GetUri() const override { diff --git a/src/paimon/fs/local/local_file_test.cpp b/src/paimon/fs/local/local_file_test.cpp index 4ad3b971..f22095c9 100644 --- a/src/paimon/fs/local/local_file_test.cpp +++ b/src/paimon/fs/local/local_file_test.cpp @@ -48,8 +48,8 @@ TEST(LocalFileTest, TestReadWriteEmptyContent) { ASSERT_OK(file->OpenFile(/*is_read_file=*/false)); const char* str = ""; - const int32_t str_size = 0; - ASSERT_OK_AND_ASSIGN(int32_t write_size, file->Write(str, str_size)); + constexpr int64_t str_size = 0; + ASSERT_OK_AND_ASSIGN(int64_t write_size, file->Write(str, str_size)); ASSERT_EQ(write_size, str_size); ASSERT_OK(file->Flush()); @@ -60,7 +60,7 @@ TEST(LocalFileTest, TestReadWriteEmptyContent) { ASSERT_OK_AND_ASSIGN(auto file2, LocalFile::Create(path)); ASSERT_OK(file2->OpenFile(/*is_read_file=*/true)); char buffer[10]; - ASSERT_OK_AND_ASSIGN(int32_t read_len, file2->Read(buffer, 10)); + ASSERT_OK_AND_ASSIGN(int64_t read_len, file2->Read(buffer, 10)); ASSERT_EQ(0, read_len); } @@ -85,8 +85,8 @@ TEST(LocalFileTest, TestSimple) { ASSERT_OK(file->OpenFile(/*is_read_file=*/false)); const char* str = "test_data"; - const int32_t str_size = 9; - ASSERT_OK_AND_ASSIGN(int32_t write_size, file->Write(str, str_size)); + constexpr int64_t str_size = 9; + ASSERT_OK_AND_ASSIGN(int64_t write_size, file->Write(str, str_size)); ASSERT_EQ(write_size, str_size); ASSERT_OK(file->Flush()); @@ -101,7 +101,7 @@ TEST(LocalFileTest, TestSimple) { std::vector file_list; ASSERT_NOK(file->List(&file_list)); - ASSERT_OK_AND_ASSIGN(size_t len, file->Length()); + ASSERT_OK_AND_ASSIGN(int64_t len, file->Length()); ASSERT_EQ(len, str_size); ASSERT_OK_AND_ASSIGN(auto file2, LocalFile::Create(path)); @@ -110,7 +110,7 @@ TEST(LocalFileTest, TestSimple) { ASSERT_OK(file2->OpenFile(true)); char str_read[str_size + 1]; { - ASSERT_OK_AND_ASSIGN(int32_t read_size, file2->Read(str_read, 4)); + ASSERT_OK_AND_ASSIGN(int64_t read_size, file2->Read(str_read, 4)); ASSERT_EQ(read_size, 4); str_read[read_size] = '\0'; ASSERT_EQ(strcmp(str_read, "test"), 0); @@ -119,13 +119,13 @@ TEST(LocalFileTest, TestSimple) { ASSERT_OK_AND_ASSIGN(int64_t pos, file2->Tell()); ASSERT_EQ(pos, 4); ASSERT_OK(file2->Seek(5, SEEK_SET)); - ASSERT_OK_AND_ASSIGN(int32_t read_size, file2->Read(str_read, 4)); + ASSERT_OK_AND_ASSIGN(int64_t read_size, file2->Read(str_read, 4)); ASSERT_EQ(read_size, 4); str_read[read_size] = '\0'; ASSERT_EQ(strcmp(str_read, "data"), 0); } { - ASSERT_OK_AND_ASSIGN(int32_t read_size, file2->Read(str_read, str_size, 0)); + ASSERT_OK_AND_ASSIGN(int64_t read_size, file2->Read(str_read, str_size, 0)); ASSERT_EQ(read_size, str_size); str_read[read_size] = '\0'; ASSERT_EQ(strcmp(str_read, "test_data"), 0); diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 2cd52911..14a66584 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -207,10 +207,9 @@ Result LuceneGlobalIndexWriter::FlushIndexToFinal() { static_cast(kDefaultReadBufferSize)); input->readBytes(reinterpret_cast(buffer->data()), /*offset=*/0, static_cast(current_write_size)); - PAIMON_ASSIGN_OR_RAISE( - int32_t actual_write_size, - out->Write(buffer->data(), static_cast(current_write_size))); - if (static_cast(actual_write_size) != current_write_size) { + PAIMON_ASSIGN_OR_RAISE(int64_t actual_write_size, + out->Write(buffer->data(), current_write_size)); + if (actual_write_size != current_write_size) { return Status::Invalid( fmt::format("invalid write, try to write {} while actual write {}", current_write_size, actual_write_size)); diff --git a/src/paimon/global_index/lucene/lucene_input.h b/src/paimon/global_index/lucene/lucene_input.h index 22f60df2..f30acd61 100644 --- a/src/paimon/global_index/lucene/lucene_input.h +++ b/src/paimon/global_index/lucene/lucene_input.h @@ -46,7 +46,7 @@ class LuceneIndexInput : public Lucene::BufferedIndexInput { if (!result.ok()) { throw Lucene::IOException(LuceneUtils::StringToWstring(result.status().ToString())); } - return static_cast(result.value()); + return result.value(); } void close() override { if (is_clone_) { @@ -67,7 +67,7 @@ class LuceneIndexInput : public Lucene::BufferedIndexInput { throw Lucene::IOException( LuceneUtils::StringToWstring(read_result.status().ToString())); } - if (read_result.value() != length) { + if (read_result.value() != static_cast(length)) { throw Lucene::IOException(L"actual read len and expect read len mismatch"); } } diff --git a/src/paimon/global_index/lumina/lumina_file_io_test.cpp b/src/paimon/global_index/lumina/lumina_file_io_test.cpp index 571703c6..39dd1b2b 100644 --- a/src/paimon/global_index/lumina/lumina_file_io_test.cpp +++ b/src/paimon/global_index/lumina/lumina_file_io_test.cpp @@ -28,50 +28,42 @@ class LuminaFileIOTest : public ::testing::Test { }; TEST_F(LuminaFileIOTest, TestSimple) { - auto check_write_and_read = [](uint64_t max_write_size, uint64_t max_read_size) { - std::string content = "Hello World."; - auto dir = paimon::test::UniqueTestDirectory::Create("local"); - auto fs = dir->GetFileSystem(); - std::string index_path = dir->Str() + "/lumina_test.index"; - // write content - ASSERT_OK_AND_ASSIGN(std::shared_ptr out, - fs->Create(index_path, /*overwrite=*/false)); - auto writer = std::make_shared(out); - writer->max_write_size_ = max_write_size; - ASSERT_EQ(writer->GetLength().Value(), 0); - ASSERT_TRUE(writer->Write(content.data(), content.length()).IsOk()); - ASSERT_TRUE(writer->Close().IsOk()); - - // check file exist - ASSERT_OK_AND_ASSIGN(bool exist, fs->Exists(index_path)); - ASSERT_TRUE(exist); - ASSERT_OK_AND_ASSIGN(std::unique_ptr file_status, - fs->GetFileStatus(index_path)); - ASSERT_FALSE(file_status->IsDir()); - ASSERT_EQ(file_status->GetLen(), content.length()); + std::string content = "Hello World."; + auto dir = paimon::test::UniqueTestDirectory::Create("local"); + auto fs = dir->GetFileSystem(); + std::string index_path = dir->Str() + "/lumina_test.index"; + // write content + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs->Create(index_path, /*overwrite=*/false)); + auto writer = std::make_shared(out); + ASSERT_EQ(writer->GetLength().Value(), 0); + ASSERT_TRUE(writer->Write(content.data(), content.length()).IsOk()); + ASSERT_TRUE(writer->Close().IsOk()); - // read content - ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs->Open(index_path)); - auto reader = std::make_shared(in); - reader->max_read_size_ = max_read_size; - ASSERT_EQ(reader->GetLength().Value(), content.length()); - ASSERT_EQ(reader->GetPosition().Value(), 0); - std::string read_content(content.size(), 0); - ASSERT_TRUE(reader->Read(read_content.data(), read_content.size()).IsOk()); - ASSERT_EQ(read_content, content); - ASSERT_EQ(reader->GetPosition().Value(), content.size()); + // check file exist + ASSERT_OK_AND_ASSIGN(bool exist, fs->Exists(index_path)); + ASSERT_TRUE(exist); + ASSERT_OK_AND_ASSIGN(std::unique_ptr file_status, fs->GetFileStatus(index_path)); + ASSERT_FALSE(file_status->IsDir()); + ASSERT_EQ(file_status->GetLen(), content.length()); - // test seek - ASSERT_TRUE(reader->Seek(2).IsOk()); - std::string read_content2(3, 0); - ASSERT_TRUE(reader->Read(read_content2.data(), read_content2.size()).IsOk()); - ASSERT_EQ(read_content2, "llo"); - ASSERT_EQ(reader->GetPosition().Value(), 5); - ASSERT_TRUE(reader->Close().IsOk()); - }; + // read content + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs->Open(index_path)); + auto reader = std::make_shared(in); + ASSERT_EQ(reader->GetLength().Value(), content.length()); + ASSERT_EQ(reader->GetPosition().Value(), 0); + std::string read_content(content.size(), 0); + ASSERT_TRUE(reader->Read(read_content.data(), read_content.size()).IsOk()); + ASSERT_EQ(read_content, content); + ASSERT_EQ(reader->GetPosition().Value(), content.size()); - check_write_and_read(LuminaFileWriter::kMaxWriteSize, LuminaFileReader::kMaxReadSize); - check_write_and_read(2, 2); + // test seek + ASSERT_TRUE(reader->Seek(2).IsOk()); + std::string read_content2(3, 0); + ASSERT_TRUE(reader->Read(read_content2.data(), read_content2.size()).IsOk()); + ASSERT_EQ(read_content2, "llo"); + ASSERT_EQ(reader->GetPosition().Value(), 5); + ASSERT_TRUE(reader->Close().IsOk()); } TEST_F(LuminaFileIOTest, TestReadAsync) { @@ -84,8 +76,7 @@ TEST_F(LuminaFileIOTest, TestReadAsync) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs->Open(index_path)); auto reader = std::make_shared(in); - auto check_read_result = [&](uint64_t max_read_size, std::string& read_content) { - reader->max_read_size_ = max_read_size; + auto check_read_result = [&](std::string& read_content) { bool read_finished = false; std::promise promise; std::future future = promise.get_future(); @@ -105,12 +96,10 @@ TEST_F(LuminaFileIOTest, TestReadAsync) { ASSERT_EQ(content.substr(0, read_content.size()), read_content); }; - for (auto max_read_size : {1, 2, 3, 4, 5, 10, 100}) { - std::string read_content(content.size(), '\0'); - check_read_result(max_read_size, read_content); - // test read empty - std::string empty_content; - check_read_result(max_read_size, empty_content); - } + std::string read_content(content.size(), '\0'); + check_read_result(read_content); + // test read empty + std::string empty_content; + check_read_result(empty_content); } } // namespace paimon::lumina::test diff --git a/src/paimon/global_index/lumina/lumina_file_reader.h b/src/paimon/global_index/lumina/lumina_file_reader.h index db6d2fbb..0ae11692 100644 --- a/src/paimon/global_index/lumina/lumina_file_reader.h +++ b/src/paimon/global_index/lumina/lumina_file_reader.h @@ -18,13 +18,12 @@ #pragma once -#include -#include +#include #include -#include #include "fmt/format.h" #include "lumina/io/FileReader.h" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/lumina/lumina_utils.h" namespace paimon::lumina { @@ -34,12 +33,16 @@ class LuminaFileReader : public ::lumina::io::FileReader { ~LuminaFileReader() override = default; ::lumina::core::Result GetLength() const noexcept override { - Result length_result = in_->Length(); + Result length_result = in_->Length(); if (!length_result.ok()) { return ::lumina::core::Result::Err( PaimonToLuminaStatus(length_result.status())); } - return ::lumina::core::Result::Ok(length_result.value()); + Status status = ValidateValueInRange(length_result.value(), "file length"); + if (!status.ok()) { + return ::lumina::core::Result::Err(PaimonToLuminaStatus(status)); + } + return ::lumina::core::Result::Ok(static_cast(length_result.value())); } ::lumina::core::Result GetPosition() const noexcept override { @@ -47,29 +50,36 @@ class LuminaFileReader : public ::lumina::io::FileReader { if (!pos_result.ok()) { return ::lumina::core::Result::Err(PaimonToLuminaStatus(pos_result.status())); } + Status status = ValidateValueInRange(pos_result.value(), "file position"); + if (!status.ok()) { + return ::lumina::core::Result::Err(PaimonToLuminaStatus(status)); + } return ::lumina::core::Result::Ok(static_cast(pos_result.value())); } ::lumina::core::Status Seek(uint64_t position) noexcept override { - return PaimonToLuminaStatus(in_->Seek(position, SeekOrigin::FS_SEEK_SET)); + Status status = ValidateValueInRange(position, "seek position"); + if (!status.ok()) { + return PaimonToLuminaStatus(status); + } + return PaimonToLuminaStatus( + in_->Seek(static_cast(position), SeekOrigin::FS_SEEK_SET)); } ::lumina::core::Status Read(char* data, uint64_t size) noexcept override { - uint64_t total_read_size = 0; - while (total_read_size < size) { - uint64_t current_read_size = std::min(size - total_read_size, max_read_size_); - Result read_result = - in_->Read(data + total_read_size, static_cast(current_read_size)); - if (!read_result.ok()) { - return PaimonToLuminaStatus(read_result.status()); - } - if (static_cast(read_result.value()) != current_read_size) { - return ::lumina::core::Status( - ::lumina::core::ErrorCode::IoError, - fmt::format("expect read len {} mismatch actual read len {}", current_read_size, - read_result.value())); - } - total_read_size += current_read_size; + Status status = ValidateValueInRange(size, "read size"); + if (!status.ok()) { + return PaimonToLuminaStatus(status); + } + Result read_result = in_->Read(data, static_cast(size)); + if (!read_result.ok()) { + return PaimonToLuminaStatus(read_result.status()); + } + if (read_result.value() != static_cast(size)) { + return ::lumina::core::Status( + ::lumina::core::ErrorCode::IoError, + fmt::format("expect read len {} mismatch actual read len {}", size, + read_result.value())); } return ::lumina::core::Status::Ok(); } @@ -81,50 +91,20 @@ class LuminaFileReader : public ::lumina::io::FileReader { return; } - struct ReadContext { - char* current_data; - uint64_t remaining; - uint64_t current_offset; - std::function final_call_back; - std::shared_ptr in; - }; - - auto ctx = std::make_shared( - ReadContext{data, size, offset, std::move(call_back), in_}); - - // recursive lambda to read next chunk - std::function read_next; - read_next = [ctx, max_read_size = max_read_size_, &read_next]() { - if (ctx->remaining == 0) { - // all done - ctx->final_call_back(::lumina::core::Status::Ok()); - return; - } - - // determine this chunk's size - uint64_t chunk_size = std::min(ctx->remaining, max_read_size); - auto safe_size = static_cast(chunk_size); - - // issue async read for this chunk - ctx->in->ReadAsync(ctx->current_data, safe_size, ctx->current_offset, - [ctx, safe_size, read_next](const Status& status) { - if (!status.ok()) { - // propagate error immediately - ctx->final_call_back(PaimonToLuminaStatus(status)); - return; - } - // advance pointers and counters - ctx->current_data += safe_size; - ctx->current_offset += safe_size; - ctx->remaining -= safe_size; - - // continue with next chunk - read_next(); - }); - }; - - // start the first read - read_next(); + Status status = ValidateValueInRange(size, "read size"); + if (!status.ok()) { + call_back(PaimonToLuminaStatus(status)); + return; + } + status = ValidateValueInRange(offset, "read offset"); + if (!status.ok()) { + call_back(PaimonToLuminaStatus(status)); + return; + } + in_->ReadAsync(data, static_cast(size), static_cast(offset), + [call_back = std::move(call_back)](const Status& status) { + call_back(PaimonToLuminaStatus(status)); + }); } ::lumina::core::Status Close() noexcept override { @@ -132,10 +112,6 @@ class LuminaFileReader : public ::lumina::io::FileReader { } private: - static constexpr uint64_t kMaxReadSize = std::numeric_limits::max(); - - private: - uint64_t max_read_size_ = kMaxReadSize; std::shared_ptr in_; }; } // namespace paimon::lumina diff --git a/src/paimon/global_index/lumina/lumina_file_writer.h b/src/paimon/global_index/lumina/lumina_file_writer.h index 16323778..a2514cc6 100644 --- a/src/paimon/global_index/lumina/lumina_file_writer.h +++ b/src/paimon/global_index/lumina/lumina_file_writer.h @@ -18,12 +18,11 @@ #pragma once -#include -#include #include #include "fmt/format.h" #include "lumina/io/FileWriter.h" +#include "paimon/common/utils/math.h" #include "paimon/fs/file_system.h" #include "paimon/global_index/lumina/lumina_utils.h" namespace paimon::lumina { @@ -41,21 +40,19 @@ class LuminaFileWriter : public ::lumina::io::FileWriter { } ::lumina::core::Status Write(const char* data, uint64_t size) noexcept override { - uint64_t total_write_size = 0; - while (total_write_size < size) { - uint64_t current_write_size = std::min(size - total_write_size, max_write_size_); - Result write_result = - out_->Write(data + total_write_size, static_cast(current_write_size)); - if (!write_result.ok()) { - return PaimonToLuminaStatus(write_result.status()); - } - if (static_cast(write_result.value()) != current_write_size) { - return ::lumina::core::Status( - ::lumina::core::ErrorCode::IoError, - fmt::format("expect write len {} mismatch actual write len {}", - current_write_size, write_result.value())); - } - total_write_size += current_write_size; + Status status = ValidateValueInRange(size, "write size"); + if (!status.ok()) { + return PaimonToLuminaStatus(status); + } + Result write_result = out_->Write(data, static_cast(size)); + if (!write_result.ok()) { + return PaimonToLuminaStatus(write_result.status()); + } + if (write_result.value() != static_cast(size)) { + return ::lumina::core::Status( + ::lumina::core::ErrorCode::IoError, + fmt::format("expect write len {} mismatch actual write len {}", size, + write_result.value())); } return ::lumina::core::Status::Ok(); } @@ -69,10 +66,6 @@ class LuminaFileWriter : public ::lumina::io::FileWriter { } private: - static constexpr uint64_t kMaxWriteSize = std::numeric_limits::max(); - - private: - uint64_t max_write_size_ = kMaxWriteSize; std::shared_ptr out_; }; } // namespace paimon::lumina diff --git a/src/paimon/testing/mock/mock_file_system.h b/src/paimon/testing/mock/mock_file_system.h index e0164f1b..484ed1b3 100644 --- a/src/paimon/testing/mock/mock_file_system.h +++ b/src/paimon/testing/mock/mock_file_system.h @@ -37,13 +37,13 @@ class MockInputStream : public InputStream { Result GetPos() const override { return 0; } - Result Read(char* buffer, uint32_t size) override { + Result Read(char* buffer, int64_t size) override { return 0; } - Result Read(char* buffer, uint32_t size, uint64_t offset) override { + Result Read(char* buffer, int64_t size, int64_t offset) override { return 0; } - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override {} Status Close() override { @@ -52,7 +52,7 @@ class MockInputStream : public InputStream { Result GetUri() const override { return std::string(); } - Result Length() const override { + Result Length() const override { return 0; } }; @@ -65,7 +65,7 @@ class MockOutputStream : public OutputStream { Result GetPos() const override { return 0; } - Result Write(const char* buffer, uint32_t size) override { + Result Write(const char* buffer, int64_t size) override { return 0; } Status Flush() override { @@ -87,7 +87,7 @@ class MockFileStatus : public FileStatus { std::string GetPath() const override { return ""; } - uint64_t GetLen() const override { + int64_t GetLen() const override { return 0; } int64_t GetModificationTime() const override { diff --git a/src/paimon/testing/mock/mock_format_writer.cpp b/src/paimon/testing/mock/mock_format_writer.cpp index 07feccdd..61f7482b 100644 --- a/src/paimon/testing/mock/mock_format_writer.cpp +++ b/src/paimon/testing/mock/mock_format_writer.cpp @@ -40,8 +40,8 @@ MockFormatWriter::MockFormatWriter(const std::shared_ptr& out, Status MockFormatWriter::AddBatch(ArrowArray* batch) { ArrowArrayRelease(batch); std::string str = std::to_string(DateTimeUtils::GetCurrentUTCTimeUs()) + "\n"; - PAIMON_ASSIGN_OR_RAISE(int32_t res, out_->Write(str.data(), str.size())); - if (res != static_cast(str.size())) { + PAIMON_ASSIGN_OR_RAISE(int64_t res, out_->Write(str.data(), str.size())); + if (res != static_cast(str.size())) { return Status::IOError("write size does not match"); } counter_++; diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index b18a1016..1af886e3 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -186,8 +186,8 @@ class TestHelper { PAIMON_ASSIGN_OR_RAISE(auto result_stream, result_blobs[i]->NewInputStream(fs)); PAIMON_ASSIGN_OR_RAISE(auto expected_stream, expected_blobs[i]->NewInputStream(fs)); - PAIMON_ASSIGN_OR_RAISE(uint64_t result_length, result_stream->Length()); - PAIMON_ASSIGN_OR_RAISE(uint64_t expected_length, expected_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t result_length, result_stream->Length()); + PAIMON_ASSIGN_OR_RAISE(int64_t expected_length, expected_stream->Length()); if (result_length != expected_length) { auto result_descriptor_bytes = result_blobs[i]->ToDescriptor(GetDefaultPool()); auto expected_descriptor_bytes = expected_blobs[i]->ToDescriptor(GetDefaultPool()); diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 1689e7cf..4db53384 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -328,13 +328,12 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter [&](const std::string_view& raw_value, arrow::LargeBinaryBuilder* builder) -> Status { std::string file_path = blob_dir_->Str() + "/blob_" + std::to_string(blob_file_counter_++) + ".bin"; + auto raw_size = static_cast(raw_value.size()); PAIMON_ASSIGN_OR_RAISE(auto out, fs->Create(file_path, /*overwrite=*/true)); - PAIMON_ASSIGN_OR_RAISE( - auto written, - out->Write(raw_value.data(), static_cast(raw_value.size()))); + PAIMON_ASSIGN_OR_RAISE(auto written, out->Write(raw_value.data(), raw_size)); PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); - if (static_cast(written) != raw_value.size()) { + if (written != raw_size) { return Status::Invalid("Short write: expected {}, wrote {}", raw_value.size(), written); } diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 5a9d1e4b..0d40181a 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -193,7 +193,7 @@ class ReadInteTest : public testing::Test, public ::testing::WithParamInterface< auto file_system = std::make_shared(); EXPECT_OK_AND_ASSIGN(auto input_stream, file_system->Open(split_file_name)); std::vector split_bytes(input_stream->Length().value_or(0), 0); - EXPECT_OK_AND_ASSIGN([[maybe_unused]] int32_t read_len, + EXPECT_OK_AND_ASSIGN([[maybe_unused]] int64_t read_len, input_stream->Read(split_bytes.data(), split_bytes.size())); EXPECT_OK(input_stream->Close()); @@ -2957,15 +2957,15 @@ TEST_P(ReadInteTest, TestSpecificFs) { Result GetPos() const override { return input_->GetPos(); } - Result Read(char* buffer, uint32_t size) override { + Result Read(char* buffer, int64_t size) override { (*io_count_)++; return input_->Read(buffer, size); } - Result Read(char* buffer, uint32_t size, uint64_t offset) override { + Result Read(char* buffer, int64_t size, int64_t offset) override { (*io_count_)++; return input_->Read(buffer, size, offset); } - void ReadAsync(char* buffer, uint32_t size, uint64_t offset, + void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override { (*io_count_)++; return input_->ReadAsync(buffer, size, offset, std::move(callback)); @@ -2977,7 +2977,7 @@ TEST_P(ReadInteTest, TestSpecificFs) { Result GetUri() const override { return input_->GetUri(); } - Result Length() const override { + Result Length() const override { return input_->Length(); } diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 8aa07faf..b52ba8c2 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -864,9 +864,9 @@ TEST_P(WriteAndReadInteTest, TestWriteSamePartitionTwiceWithAllBasicTypesForPk) std::vector> GetTestValuesForWriteAndReadInteTest() { std::vector> values = {{"parquet", "local"}}; + // values.emplace_back("parquet", "jindo"); #ifdef PAIMON_ENABLE_ORC values.emplace_back("orc", "local"); - // values.emplace_back("parquet", "jindo"); #endif #ifdef PAIMON_ENABLE_AVRO values.emplace_back("avro", "local"); From f84e3760072c1768802db3ff91100a2918922cfb Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Wed, 10 Jun 2026 21:07:18 +0800 Subject: [PATCH 046/138] fix: small fixes for parquet reader including RowRanges, PreBuffer, ColumnIndexFilter and code cleanup --- .../format/parquet/column_index_filter.cpp | 32 ++++++----------- .../parquet/column_index_filter_test.cpp | 22 ++++++++++++ .../format/parquet/file_reader_wrapper.cpp | 12 +------ .../page_filtered_row_group_reader.cpp | 2 ++ .../parquet/parquet_file_batch_reader.cpp | 35 +++++++------------ .../format/parquet/parquet_format_defs.h | 14 ++++++++ src/paimon/format/parquet/row_ranges.h | 3 ++ 7 files changed, 65 insertions(+), 55 deletions(-) diff --git a/src/paimon/format/parquet/column_index_filter.cpp b/src/paimon/format/parquet/column_index_filter.cpp index 43a0e9df..5dd3af9c 100644 --- a/src/paimon/format/parquet/column_index_filter.cpp +++ b/src/paimon/format/parquet/column_index_filter.cpp @@ -99,6 +99,11 @@ Result ColumnIndexFilter::VisitLeafPredicate( const auto& literals = leaf_predicate->Literals(); FieldType field_type = leaf_predicate->GetFieldType(); + if (function_type != Function::Type::IS_NULL && function_type != Function::Type::IS_NOT_NULL && + literals.empty()) { + return Status::Invalid( + fmt::format("predicate on column '{}' requires at least one literal", field_name)); + } std::vector matching_pages; switch (function_type) { @@ -109,37 +114,22 @@ Result ColumnIndexFilter::VisitLeafPredicate( matching_pages = FilterPagesByIsNotNull(column_index_ptr); break; case Function::Type::EQUAL: - if (!literals.empty()) { - matching_pages = FilterPagesByEqual(column_index_ptr, literals[0], field_type); - } + matching_pages = FilterPagesByEqual(column_index_ptr, literals[0], field_type); break; case Function::Type::NOT_EQUAL: - if (!literals.empty()) { - matching_pages = FilterPagesByNotEqual(column_index_ptr, literals[0], field_type); - } + matching_pages = FilterPagesByNotEqual(column_index_ptr, literals[0], field_type); break; case Function::Type::LESS_THAN: - if (!literals.empty()) { - matching_pages = FilterPagesByLessThan(column_index_ptr, literals[0], field_type); - } + matching_pages = FilterPagesByLessThan(column_index_ptr, literals[0], field_type); break; case Function::Type::LESS_OR_EQUAL: - if (!literals.empty()) { - matching_pages = - FilterPagesByLessOrEqual(column_index_ptr, literals[0], field_type); - } + matching_pages = FilterPagesByLessOrEqual(column_index_ptr, literals[0], field_type); break; case Function::Type::GREATER_THAN: - if (!literals.empty()) { - matching_pages = - FilterPagesByGreaterThan(column_index_ptr, literals[0], field_type); - } + matching_pages = FilterPagesByGreaterThan(column_index_ptr, literals[0], field_type); break; case Function::Type::GREATER_OR_EQUAL: - if (!literals.empty()) { - matching_pages = - FilterPagesByGreaterOrEqual(column_index_ptr, literals[0], field_type); - } + matching_pages = FilterPagesByGreaterOrEqual(column_index_ptr, literals[0], field_type); break; case Function::Type::IN: matching_pages = FilterPagesByIn(column_index_ptr, literals, field_type); diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp index 8129e8f3..b9003436 100644 --- a/src/paimon/format/parquet/column_index_filter_test.cpp +++ b/src/paimon/format/parquet/column_index_filter_test.cpp @@ -29,6 +29,9 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/common/predicate/equal.h" +#include "paimon/common/predicate/in.h" +#include "paimon/common/predicate/leaf_predicate_impl.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/defs.h" @@ -483,4 +486,23 @@ TEST_F(ColumnIndexFilterTest, NullPredicateReturnsAllRows) { EXPECT_EQ(row_group_row_count_, ranges.RowCount()); } +/// Predicates other than IsNull/IsNotNull are not allowed without a literal. +/// PredicateBuilder (public API) does not support constructing them without +/// a literal, so the filter should return an error for this invalid input. +TEST_F(ColumnIndexFilterTest, EmptyLiteralsReturnsError) { + auto pred = std::make_shared(paimon::Equal::Instance(), 0, "val", + FieldType::INT, std::vector()); + auto result = Filter(pred); + EXPECT_FALSE(result.ok()); +} + +/// Empty literals for IN predicate — same rule applies: non-IS_NULL/IS_NOT_NULL +/// predicates without literals are invalid and should return an error. +TEST_F(ColumnIndexFilterTest, EmptyLiteralsInReturnsError) { + auto pred = std::make_shared(paimon::In::Instance(), 0, "val", + FieldType::INT, std::vector()); + auto result = Filter(pred); + EXPECT_FALSE(result.ok()); +} + } // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 3e019598..8829a829 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -28,23 +28,13 @@ #include "fmt/format.h" #include "paimon/format/parquet/column_index_filter.h" #include "paimon/format/parquet/page_filtered_row_group_reader.h" +#include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/macros.h" #include "parquet/arrow/reader.h" #include "parquet/file_reader.h" #include "parquet/metadata.h" #include "parquet/page_index.h" -// Convert any std::exception thrown by underlying Parquet/Arrow APIs into a -// Status. Used as the trailing catch clauses of a try block in every public -// method that calls into the parquet C++ API, so the read layer never throws. -#define PAIMON_PARQUET_CATCH_AND_RETURN_STATUS(context) \ - catch (const std::exception& e) { \ - return Status::Invalid(fmt::format("{}: {}", (context), e.what())); \ - } \ - catch (...) { \ - return Status::UnknownError((context), ": unknown error"); \ - } - namespace paimon::parquet { namespace { diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 2e9d9b36..f729d729 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -252,6 +252,8 @@ Result> PageFilteredRowGroupReader::Re // Pre-buffering failed, fall back to row-group level PreBuffer ::arrow::io::IOContext io_ctx(pool); parquet_reader->PreBuffer(rg_vec, col_vec, io_ctx, cache_options); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + parquet_reader->WhenBuffered(rg_vec, col_vec).status()); } } else { PAIMON_RETURN_NOT_OK_FROM_ARROW(parquet_reader->WhenBuffered(rg_vec, col_vec).status()); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 7eb3066e..e64f481a 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -50,17 +50,6 @@ #include "parquet/arrow/reader.h" #include "parquet/properties.h" -// Convert any std::exception thrown by underlying Parquet/Arrow APIs into a -// Status. Used as the trailing catch clauses of a try block in every public -// method that calls into the parquet C++ API, so the read layer never throws. -#define PAIMON_PARQUET_CATCH_AND_RETURN_STATUS(context) \ - catch (const std::exception& e) { \ - return Status::Invalid(fmt::format("{}: {}", (context), e.what())); \ - } \ - catch (...) { \ - return Status::UnknownError((context), ": unknown error"); \ - } - namespace arrow { class MemoryPool; } // namespace arrow @@ -161,18 +150,6 @@ Status ParquetFileBatchReader::SetReadSchema( } } - // Build column name to index map for page-level filtering. - // For leaf columns, indices[0] is the correct leaf column index in Parquet. - // For nested types (struct/list/map), FlattenSchema produces multiple leaf indices, - // but predicate pushdown only targets leaf columns with simple types, so indices[0] - // is always the correct single leaf index for predicate evaluation. - std::map column_name_to_index; - for (const auto& [name, indices] : field_index_map) { - if (!indices.empty()) { - column_name_to_index[name] = indices[0]; - } - } - std::vector row_groups = arrow::internal::Iota(reader_->GetNumberOfRowGroups()); if (predicate) { PAIMON_ASSIGN_OR_RAISE(row_groups, @@ -190,6 +167,18 @@ Status ParquetFileBatchReader::SetReadSchema( OptionsUtils::GetValueFromMap(options_, PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER)); if (enable_page_index_filter) { + // Build column name to index map for page-level filtering. + // For leaf columns, indices[0] is the correct leaf column index in Parquet. + // For nested types (struct/list/map), FlattenSchema produces multiple leaf indices, + // but predicate pushdown only targets leaf columns with simple types, so indices[0] + // is always the correct single leaf index for predicate evaluation. + std::map column_name_to_index; + for (const auto& [name, indices] : field_index_map) { + if (!indices.empty()) { + column_name_to_index[name] = indices[0]; + } + } + PAIMON_ASSIGN_OR_RAISE( auto page_filter_result, FilterRowGroupsByPageIndex(predicate, column_name_to_index, row_groups)); diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h index 90cd716a..69894092 100644 --- a/src/paimon/format/parquet/parquet_format_defs.h +++ b/src/paimon/format/parquet/parquet_format_defs.h @@ -21,8 +21,22 @@ #include #include +#include "fmt/format.h" +#include "paimon/status.h" + namespace paimon::parquet { +// Convert any std::exception thrown by underlying Parquet/Arrow APIs into a +// Status. Used as the trailing catch clauses of a try block in every public +// method that calls into the parquet C++ API, so the read layer never throws. +#define PAIMON_PARQUET_CATCH_AND_RETURN_STATUS(context) \ + catch (const std::exception& e) { \ + return Status::Invalid(fmt::format("{}: {}", (context), e.what())); \ + } \ + catch (...) { \ + return Status::UnknownError(fmt::format("{}: unknown error", (context))); \ + } + // write static inline const char PARQUET_BLOCK_SIZE[] = "parquet.block.size"; static inline const char PARQUET_PAGE_SIZE[] = "parquet.page.size"; diff --git a/src/paimon/format/parquet/row_ranges.h b/src/paimon/format/parquet/row_ranges.h index 956622d3..2f49f4f4 100644 --- a/src/paimon/format/parquet/row_ranges.h +++ b/src/paimon/format/parquet/row_ranges.h @@ -46,6 +46,9 @@ class RowRanges { /// Creates a RowRanges from a list of ranges. explicit RowRanges(const std::vector& ranges) : ranges_(ranges) {} + /// Creates a RowRanges from a list of ranges, taking ownership of the vector. + explicit RowRanges(std::vector&& ranges) : ranges_(std::move(ranges)) {} + /// Creates a RowRanges with a single range [0, row_count - 1]. static RowRanges CreateSingle(int64_t row_count) { if (row_count <= 0) { From 79e19aefccc4fe8dfc6ab8d13017d3e719e4d72f Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Wed, 10 Jun 2026 18:27:29 -0700 Subject: [PATCH 047/138] feat(blob view): support multi-thread reading upstream table & add inte test --- include/paimon/catalog/catalog.h | 5 +- include/paimon/defs.h | 6 + include/paimon/executor.h | 3 + src/paimon/common/data/blob_utils.cpp | 26 +- src/paimon/common/data/blob_utils.h | 6 +- src/paimon/common/data/blob_utils_test.cpp | 133 +++-- .../common/data/blob_view_struct_test.cpp | 46 ++ src/paimon/common/defs.cpp | 1 + src/paimon/common/executor/executor.cpp | 8 +- src/paimon/core/append/append_only_writer.cpp | 17 +- src/paimon/core/append/append_only_writer.h | 1 + .../core/append/append_only_writer_test.cpp | 67 +++ .../core/catalog/file_system_catalog.cpp | 34 +- src/paimon/core/catalog/file_system_catalog.h | 5 +- .../core/catalog/file_system_catalog_test.cpp | 12 +- src/paimon/core/core_options.cpp | 8 + src/paimon/core/core_options.h | 1 + src/paimon/core/core_options_test.cpp | 4 + .../global_index/global_index_scan_impl.cpp | 23 +- .../merge_tree_compact_manager_test.cpp | 8 + .../operation/data_evolution_split_read.cpp | 18 +- src/paimon/core/utils/blob_view_lookup.cpp | 191 +++++-- src/paimon/core/utils/blob_view_lookup.h | 20 +- .../core/utils/blob_view_lookup_test.cpp | 81 ++- test/inte/blob_table_inte_test.cpp | 492 +++++++++++++++++- test/inte/read_inte_test.cpp | 2 +- .../README | 9 +- ...c772a51-21ba-4c17-b464-6f1a314d0950-0.orc} | Bin ...5e684ea-c245-4719-858d-eb6d7e3a8446-0.orc} | Bin ...e684ea-c245-4719-858d-eb6d7e3a8446-1.blob} | Bin ...e684ea-c245-4719-858d-eb6d7e3a8446-2.blob} | Bin ...c7f689b-b9ec-4e8a-b0ae-4844ca3aa499-0.orc} | Bin ...7f689b-b9ec-4e8a-b0ae-4844ca3aa499-1.blob} | Bin ...7f689b-b9ec-4e8a-b0ae-4844ca3aa499-2.blob} | Bin ...0ddcf17-3ea2-4c90-8995-3e0ee4d3b5a9-0.orc} | Bin ...est-6ee90f75-0567-4692-b375-db3f5abd2b96-0 | Bin 2285 -> 0 bytes ...est-70d9ec54-00dc-47b6-85d8-944c25110315-0 | Bin 0 -> 2960 bytes ...est-c1e6e601-fc3b-4ab1-8b53-dd661f91a59c-0 | Bin 0 -> 3258 bytes ...est-fbe3fd00-750f-446d-81c3-bff220caed22-0 | Bin 0 -> 3104 bytes ...ist-3d1147dc-1860-48ab-a450-fce85ac5e9da-0 | Bin 0 -> 1543 bytes ...ist-3d1147dc-1860-48ab-a450-fce85ac5e9da-1 | Bin 0 -> 1565 bytes ...ist-7cf0d548-c508-4619-823c-10b0a842e6c2-0 | Bin 0 -> 392 bytes ...ist-7cf0d548-c508-4619-823c-10b0a842e6c2-1 | Bin 0 -> 1543 bytes ...ist-8046c4af-62fd-466e-8199-29d90455f97d-0 | Bin 0 -> 1723 bytes ...ist-8046c4af-62fd-466e-8199-29d90455f97d-1 | Bin 0 -> 1569 bytes ...ist-b54f1ecf-7c13-4dba-a1c5-19f988924166-2 | Bin 1034 -> 0 bytes ...ist-b54f1ecf-7c13-4dba-a1c5-19f988924166-3 | Bin 996 -> 0 bytes .../schema/schema-0 | 11 +- .../schema/schema-1 | 13 +- .../snapshot/snapshot-1 | 17 +- .../snapshot/snapshot-2 | 17 +- .../snapshot/snapshot-3 | 17 +- .../README | 9 +- ...9127-b7b2-4950-aaca-f77216b0a46b-0.parquet | Bin 0 -> 1479 bytes ...989127-b7b2-4950-aaca-f77216b0a46b-1.blob} | Bin ...989127-b7b2-4950-aaca-f77216b0a46b-2.blob} | Bin ...b1e9-7186-463f-82f5-8d1b004386f2-0.parquet | Bin 0 -> 1160 bytes ...2df8-9f32-4c97-9b43-de9ef946a1ed-0.parquet | Bin 1295 -> 0 bytes ...e758-1ff4-45c3-94f6-27a4e7c47172-0.parquet | Bin 0 -> 1297 bytes ...3de758-1ff4-45c3-94f6-27a4e7c47172-1.blob} | Bin ...3de758-1ff4-45c3-94f6-27a4e7c47172-2.blob} | Bin ...b979-5e51-4112-92f2-ff3036b332c6-0.parquet | Bin 1374 -> 0 bytes ...1455-da2f-4e8b-a026-f001aace3d58-0.parquet | Bin 1232 -> 0 bytes ...4206-06f9-4411-b69f-749f7868ac2d-0.parquet | Bin 1223 -> 0 bytes ...65ab-9916-4a9e-a13a-5e5298933a18-0.parquet | Bin 0 -> 1232 bytes ...st-04dfe945-c569-4a24-b830-c69d4f426e64-0} | Bin 2164 -> 2162 bytes ...st-26f82168-b0c0-43fc-86af-1a15e18be25b-0} | Bin 2232 -> 2283 bytes ...est-3d16fcf1-d81b-4d69-b118-f581c55e2278-0 | Bin 2278 -> 0 bytes ...st-90a8ba7d-9c17-4a24-a2e6-0b67534a6961-0} | Bin 2171 -> 2226 bytes ...est-f7c12f92-163b-45c2-9160-76375875e57b-1 | Bin 2222 -> 0 bytes ...ist-084c1aa1-da4f-4232-8e28-9d935ec4cea6-0 | Bin 995 -> 0 bytes ...ist-084c1aa1-da4f-4232-8e28-9d935ec4cea6-1 | Bin 996 -> 0 bytes ...ist-084c1aa1-da4f-4232-8e28-9d935ec4cea6-2 | Bin 1032 -> 0 bytes ...ist-084c1aa1-da4f-4232-8e28-9d935ec4cea6-3 | Bin 997 -> 0 bytes ...st-6d497f15-c843-4136-b18c-25c34fa32b31-0} | Bin 996 -> 1119 bytes ...st-6d497f15-c843-4136-b18c-25c34fa32b31-1} | Bin 884 -> 1121 bytes ...st-ba29e330-ee07-4bd0-b5ee-02f21c647791-0} | Bin 995 -> 1006 bytes ...st-ba29e330-ee07-4bd0-b5ee-02f21c647791-1} | Bin 995 -> 1119 bytes ...ist-c802eb01-61a7-4f42-98fe-4f674b3c4966-0 | Bin 0 -> 1155 bytes ...ist-c802eb01-61a7-4f42-98fe-4f674b3c4966-1 | Bin 0 -> 1124 bytes ...ist-f96bc893-0979-4689-a3d0-cd9cd2f1a203-0 | Bin 884 -> 0 bytes ...ist-f96bc893-0979-4689-a3d0-cd9cd2f1a203-1 | Bin 995 -> 0 bytes .../schema/schema-0 | 9 +- .../schema/schema-1 | 11 +- .../snapshot/snapshot-1 | 17 +- .../snapshot/snapshot-2 | 17 +- .../snapshot/snapshot-3 | 17 +- 87 files changed, 1128 insertions(+), 254 deletions(-) rename test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-17cd7b66-4ca1-42b3-a569-d7b0b889d314-0.orc => data-2c772a51-21ba-4c17-b464-6f1a314d0950-0.orc} (100%) rename test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-330a942b-a3dd-408e-9042-293c03c6cba5-0.orc => data-a5e684ea-c245-4719-858d-eb6d7e3a8446-0.orc} (100%) rename test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-330a942b-a3dd-408e-9042-293c03c6cba5-1.blob => data-a5e684ea-c245-4719-858d-eb6d7e3a8446-1.blob} (100%) rename test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-330a942b-a3dd-408e-9042-293c03c6cba5-2.blob => data-a5e684ea-c245-4719-858d-eb6d7e3a8446-2.blob} (100%) rename test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-2b2e00bc-77fd-428d-87d1-8bea1063991c-0.orc => data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-0.orc} (100%) rename test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-2b2e00bc-77fd-428d-87d1-8bea1063991c-1.blob => data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-1.blob} (100%) rename test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-2b2e00bc-77fd-428d-87d1-8bea1063991c-2.blob => data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-2.blob} (100%) rename test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-35fe228b-1e40-44dd-9832-ff078fc60150-0.orc => data-d0ddcf17-3ea2-4c90-8995-3e0ee4d3b5a9-0.orc} (100%) delete mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-6ee90f75-0567-4692-b375-db3f5abd2b96-0 create mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-70d9ec54-00dc-47b6-85d8-944c25110315-0 create mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-c1e6e601-fc3b-4ab1-8b53-dd661f91a59c-0 create mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-fbe3fd00-750f-446d-81c3-bff220caed22-0 create mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-0 create mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-1 create mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-0 create mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-1 create mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-0 create mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-1 delete mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-2 delete mode 100644 test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-3 create mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-0.parquet rename test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-cea41455-da2f-4e8b-a026-f001aace3d58-1.blob => data-50989127-b7b2-4950-aaca-f77216b0a46b-1.blob} (100%) rename test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-cea41455-da2f-4e8b-a026-f001aace3d58-2.blob => data-50989127-b7b2-4950-aaca-f77216b0a46b-2.blob} (100%) create mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-5bb7b1e9-7186-463f-82f5-8d1b004386f2-0.parquet delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-aaf02df8-9f32-4c97-9b43-de9ef946a1ed-0.parquet create mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-0.parquet rename test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-c8aab979-5e51-4112-92f2-ff3036b332c6-1.blob => data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-1.blob} (100%) rename test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/{data-c8aab979-5e51-4112-92f2-ff3036b332c6-2.blob => data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-2.blob} (100%) delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-0.parquet delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-0.parquet delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-f9d44206-06f9-4411-b69f-749f7868ac2d-0.parquet create mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-fcdc65ab-9916-4a9e-a13a-5e5298933a18-0.parquet rename test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/{manifest-f7c12f92-163b-45c2-9160-76375875e57b-0 => manifest-04dfe945-c569-4a24-b830-c69d4f426e64-0} (88%) rename test/test_data/{orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-1 => parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-26f82168-b0c0-43fc-86af-1a15e18be25b-0} (84%) delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-3d16fcf1-d81b-4d69-b118-f581c55e2278-0 rename test/test_data/{orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-0 => parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-90a8ba7d-9c17-4a24-a2e6-0b67534a6961-0} (86%) delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-f7c12f92-163b-45c2-9160-76375875e57b-1 delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-0 delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-1 delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-2 delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-3 rename test/test_data/{orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-1 => parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-0} (68%) rename test/test_data/{orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-0 => parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-1} (68%) rename test/test_data/{orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-1 => parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-0} (76%) rename test/test_data/{orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-0 => parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-1} (68%) create mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-0 create mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-1 delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-0 delete mode 100644 test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-1 diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 4b0fd1e7..5f1f04b9 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -171,8 +171,9 @@ class PAIMON_EXPORT Catalog { /// @note This does not check whether the table actually exists. /// /// @param identifier The table identifier containing database and table name. - /// @return A string representing the expected location of the table. - virtual std::string GetTableLocation(const Identifier& identifier) const = 0; + /// @return A result containing the expected location of the table, or an error status on + /// failure. + virtual Result GetTableLocation(const Identifier& identifier) const = 0; /// Returns the root path of the catalog. /// diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 17ec196d..b0266e43 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -394,7 +394,13 @@ struct PAIMON_EXPORT Options { /// "blob-external-storage-path" - The external storage path where raw BLOB data from fields /// configured by 'blob-external-storage-field' is written at write time. Orphan file cleanup is /// not applied to this path. No default value. + /// @note: this option differs from the Java paimon and will be deprecated once + /// RestCatalog is supported. static const char BLOB_EXTERNAL_STORAGE_PATH[]; + /// "blob-view-upstream-warehouse" - Since the catalog capabilities are partially missing, when + /// Blob View is enabled, cpp paimon cannot automatically obtain the upstream table warehouse + /// path and requires manual configuration by the user. No default value. + static const char BLOB_VIEW_UPSTREAM_WAREHOUSE[]; /// "global-index.enabled" - Whether to enable global index for scan. Default value is "true". static const char GLOBAL_INDEX_ENABLED[]; /// "global-index.thread-num" - The maximum number of concurrent scanner for global index. No diff --git a/include/paimon/executor.h b/include/paimon/executor.h index d4203e9e..c835ef6d 100644 --- a/include/paimon/executor.h +++ b/include/paimon/executor.h @@ -57,6 +57,9 @@ class PAIMON_EXPORT Executor { /// Shutdown the executor immediately, discarding all pending tasks. virtual void ShutdownNow() = 0; + + /// Get thread number. + virtual uint32_t GetThreadNum() const = 0; }; } // namespace paimon diff --git a/src/paimon/common/data/blob_utils.cpp b/src/paimon/common/data/blob_utils.cpp index f33c220d..8d6236b4 100644 --- a/src/paimon/common/data/blob_utils.cpp +++ b/src/paimon/common/data/blob_utils.cpp @@ -29,6 +29,7 @@ #include "fmt/format.h" #include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_view_struct.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/string_utils.h" @@ -130,16 +131,18 @@ std::shared_ptr BlobUtils::ToArrowField( std::make_shared(metadata)); } -Status BlobUtils::ValidateInlineBlobDescriptors( - const std::shared_ptr& struct_array, - const std::set& inline_descriptor_fields) { - if (inline_descriptor_fields.empty()) { +Status BlobUtils::ValidateBlobInlineFields(const std::shared_ptr& struct_array, + const std::set& field_names, + const std::string& config_label) { + if (field_names.empty()) { return Status::OK(); } if (!struct_array) { - return Status::Invalid("array in ValidateInlineBlobDescriptors must be a struct_array"); + return Status::Invalid("array in ValidateBlobInlineFields must be a struct_array"); } - for (const auto& field_name : inline_descriptor_fields) { + + bool is_descriptor = (config_label == "blob-descriptor-field"); + for (const auto& field_name : field_names) { auto field_array = struct_array->GetFieldByName(field_name); if (!field_array) { continue; @@ -155,12 +158,13 @@ Status BlobUtils::ValidateInlineBlobDescriptors( continue; } auto value = binary_array->GetView(row); - PAIMON_ASSIGN_OR_RAISE(bool is_descriptor, - BlobDescriptor::IsBlobDescriptor(value.data(), value.size())); - if (!is_descriptor) { + Result valid = is_descriptor + ? BlobDescriptor::IsBlobDescriptor(value.data(), value.size()) + : BlobViewStruct::IsBlobViewStruct(value.data(), value.size()); + PAIMON_ASSIGN_OR_RAISE(bool is_valid, std::move(valid)); + if (!is_valid) { return Status::Invalid(fmt::format( - "BLOB inline field {} configured by blob-descriptor-field or blob-view-field " - "require values to be a BlobDescriptor or BlobViewStruct.", + "BLOB inline field {} require values to be set as corresponding type.", field_name)); } } diff --git a/src/paimon/common/data/blob_utils.h b/src/paimon/common/data/blob_utils.h index f9ac0b18..1c244089 100644 --- a/src/paimon/common/data/blob_utils.h +++ b/src/paimon/common/data/blob_utils.h @@ -79,9 +79,9 @@ class PAIMON_EXPORT BlobUtils { const std::string& field_name, bool nullable = false, std::unordered_map metadata = {}); - static Status ValidateInlineBlobDescriptors( - const std::shared_ptr& struct_array, - const std::set& inline_descriptor_fields); + static Status ValidateBlobInlineFields(const std::shared_ptr& struct_array, + const std::set& field_names, + const std::string& config_label); /// Converts inline blob DataFields from large_binary to binary type. /// Inline blob fields use large_binary in the table schema (because they are BLOB type), diff --git a/src/paimon/common/data/blob_utils_test.cpp b/src/paimon/common/data/blob_utils_test.cpp index 9b4284d6..3074c54d 100644 --- a/src/paimon/common/data/blob_utils_test.cpp +++ b/src/paimon/common/data/blob_utils_test.cpp @@ -22,8 +22,10 @@ #include "arrow/api.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/catalog/identifier.h" #include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_view_struct.h" #include "paimon/common/types/data_field.h" #include "paimon/data/blob.h" #include "paimon/memory/memory_pool.h" @@ -32,44 +34,47 @@ namespace paimon::test { class BlobUtilsTest : public ::testing::Test { - private: + public: std::shared_ptr CreateBlobMetadata() { std::unordered_map blob_metadata_map = { {BlobDefs::kExtensionTypeKey, BlobDefs::kExtensionTypeValue}}; return std::make_shared(blob_metadata_map); } + + private: + std::shared_ptr pool_ = GetDefaultPool(); }; TEST_F(BlobUtilsTest, IsBlobMetadata) { auto correct_metadata = CreateBlobMetadata(); - EXPECT_TRUE(BlobUtils::IsBlobMetadata(correct_metadata)); - EXPECT_FALSE(BlobUtils::IsBlobMetadata(nullptr)); + ASSERT_TRUE(BlobUtils::IsBlobMetadata(correct_metadata)); + ASSERT_FALSE(BlobUtils::IsBlobMetadata(nullptr)); std::unordered_map wrong_metadata_map = { {BlobDefs::kExtensionTypeKey, "paimon.type.varchar"}}; auto wrong_metadata = std::make_shared(wrong_metadata_map); - EXPECT_FALSE(BlobUtils::IsBlobMetadata(wrong_metadata)); + ASSERT_FALSE(BlobUtils::IsBlobMetadata(wrong_metadata)); std::unordered_map no_extension_metadata_map = { {"other_key", BlobDefs::kExtensionTypeValue}}; auto no_extension_metadata = std::make_shared(no_extension_metadata_map); - EXPECT_FALSE(BlobUtils::IsBlobMetadata(no_extension_metadata)); + ASSERT_FALSE(BlobUtils::IsBlobMetadata(no_extension_metadata)); } TEST_F(BlobUtilsTest, IsBlobField) { std::shared_ptr blob_field = BlobUtils::ToArrowField("f1", true); - EXPECT_TRUE(BlobUtils::IsBlobField(blob_field)); + ASSERT_TRUE(BlobUtils::IsBlobField(blob_field)); auto int_field = arrow::field("i_int", arrow::int32()); - EXPECT_FALSE(BlobUtils::IsBlobField(int_field)); + ASSERT_FALSE(BlobUtils::IsBlobField(int_field)); auto binary_field_no_meta = arrow::field("b_no_meta", arrow::large_binary()); - EXPECT_FALSE(BlobUtils::IsBlobField(binary_field_no_meta)); + ASSERT_FALSE(BlobUtils::IsBlobField(binary_field_no_meta)); auto wrong_meta = std::make_shared( std::unordered_map{{"other_key", "value"}}); auto binary_field_wrong_meta = arrow::field("b_wrong_meta", arrow::large_binary(), false, wrong_meta); - EXPECT_FALSE(BlobUtils::IsBlobField(binary_field_wrong_meta)); + ASSERT_FALSE(BlobUtils::IsBlobField(binary_field_wrong_meta)); } TEST_F(BlobUtilsTest, SeparateBlobSchema) { @@ -236,7 +241,7 @@ TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsEmptyFields) { auto struct_array = arrow::StructArray::Make({array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); auto sa = std::dynamic_pointer_cast(struct_array); - ASSERT_OK(BlobUtils::ValidateInlineBlobDescriptors(sa, {})); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {}, "blob-descriptor-field")); } TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsFieldNotPresent) { @@ -248,14 +253,13 @@ TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsFieldNotPresent) { arrow::StructArray::Make({int_array}, {arrow::field("f0", arrow::int32())}).ValueOrDie(); auto sa = std::dynamic_pointer_cast(struct_array); // "b0" does not exist in the struct -> should pass - ASSERT_OK(BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"})); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field")); } TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithValidDescriptor) { // Valid BlobDescriptor bytes -> OK - auto pool = GetDefaultPool(); ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); - auto serialized = descriptor->Serialize(pool); + auto serialized = descriptor->Serialize(pool_); arrow::LargeBinaryBuilder builder; ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); @@ -263,7 +267,7 @@ TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithValidDescriptor) { auto struct_array = arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); auto sa = std::dynamic_pointer_cast(struct_array); - ASSERT_OK(BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"})); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field")); } TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithNullValue) { @@ -274,7 +278,7 @@ TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithNullValue) { auto struct_array = arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); auto sa = std::dynamic_pointer_cast(struct_array); - ASSERT_OK(BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"})); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field")); } TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithRawBytes) { @@ -285,18 +289,14 @@ TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsWithRawBytes) { auto struct_array = arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); auto sa = std::dynamic_pointer_cast(struct_array); - ASSERT_NOK_WITH_MSG( - BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"}), - "BLOB inline field b0 configured by blob-descriptor-field or blob-view-field " - "require values to be a BlobDescriptor or BlobViewStruct."); + ASSERT_NOK_WITH_MSG(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field"), + "BLOB inline field b0 require values to be set as corresponding type."); } TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsMixedValidAndInvalid) { // First row is valid descriptor, second row is raw bytes -> error on row 1 - auto pool = GetDefaultPool(); ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); - auto serialized = descriptor->Serialize(pool); - + auto serialized = descriptor->Serialize(pool_); arrow::LargeBinaryBuilder builder; ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); ASSERT_TRUE(builder.Append("raw_bytes_not_descriptor").ok()); @@ -304,17 +304,14 @@ TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsMixedValidAndInvalid) { auto struct_array = arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("b0")}).ValueOrDie(); auto sa = std::dynamic_pointer_cast(struct_array); - ASSERT_NOK_WITH_MSG( - BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0"}), - "BLOB inline field b0 configured by blob-descriptor-field or blob-view-field " - "require values to be a BlobDescriptor or BlobViewStruct."); + ASSERT_NOK_WITH_MSG(BlobUtils::ValidateBlobInlineFields(sa, {"b0"}, "blob-descriptor-field"), + "BLOB inline field b0 require values to be set as corresponding type."); } TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsMultipleFields) { // Two inline fields: b0 is valid, b1 has raw bytes -> error on b1 - auto pool = GetDefaultPool(); ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); - auto serialized = descriptor->Serialize(pool); + auto serialized = descriptor->Serialize(pool_); arrow::LargeBinaryBuilder b0_builder; ASSERT_TRUE(b0_builder.Append(serialized->data(), serialized->size()).ok()); @@ -330,9 +327,83 @@ TEST_F(BlobUtilsTest, ValidateInlineBlobDescriptorsMultipleFields) { .ValueOrDie(); auto sa = std::dynamic_pointer_cast(struct_array); ASSERT_NOK_WITH_MSG( - BlobUtils::ValidateInlineBlobDescriptors(sa, {"b0", "b1"}), - "BLOB inline field b1 configured by blob-descriptor-field or blob-view-field " - "require values to be a BlobDescriptor or BlobViewStruct."); + BlobUtils::ValidateBlobInlineFields(sa, {"b0", "b1"}, "blob-descriptor-field"), + "BLOB inline field b1 require values to be set as corresponding type."); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsEmptyFields) { + // Empty view_fields -> always OK + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append("random_data").ok()); + auto array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {}, "blob-view-field")); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsFieldNotPresent) { + // Field not in struct_array -> skip, OK + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.Append(42).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({int_array}, {arrow::field("f0", arrow::int32())}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field")); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsWithValidViewStruct) { + // A BlobViewStruct value is accepted for a view field. + BlobViewStruct view_struct(Identifier("db", "tbl"), /*field_id=*/2, /*row_id=*/5); + auto serialized = view_struct.Serialize(pool_); + + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field")); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsWithNullValue) { + // Null values in view column -> skip, OK + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.AppendNull().ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_OK(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field")); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsWithRawBytes) { + // Raw bytes -> error + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append("raw_bytes_not_view").ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field"), + "BLOB inline field view require values to be set as corresponding type."); +} + +TEST_F(BlobUtilsTest, ValidateBlobViewFieldsRejectsBlobDescriptor) { + // A BlobDescriptor value is NOT accepted for a view field. + auto pool = GetDefaultPool(); + ASSERT_OK_AND_ASSIGN(auto descriptor, BlobDescriptor::Create("file:///tmp/test.bin", 0, 100)); + auto serialized = descriptor->Serialize(pool); + + arrow::LargeBinaryBuilder builder; + ASSERT_TRUE(builder.Append(serialized->data(), serialized->size()).ok()); + auto blob_array = builder.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({blob_array}, {BlobUtils::ToArrowField("view")}).ValueOrDie(); + auto sa = std::dynamic_pointer_cast(struct_array); + ASSERT_NOK_WITH_MSG(BlobUtils::ValidateBlobInlineFields(sa, {"view"}, "blob-view-field"), + "BLOB inline field view require values to be set as corresponding type."); } TEST_F(BlobUtilsTest, TestConvertBlobInlineDataFields) { diff --git a/src/paimon/common/data/blob_view_struct_test.cpp b/src/paimon/common/data/blob_view_struct_test.cpp index 783d621e..1e7bf4ea 100644 --- a/src/paimon/common/data/blob_view_struct_test.cpp +++ b/src/paimon/common/data/blob_view_struct_test.cpp @@ -109,4 +109,50 @@ TEST_F(BlobViewStructTest, TestEqual) { } } +TEST_F(BlobViewStructTest, TestIsBlobViewStructValid) { + auto serialized = view_struct_.Serialize(pool_); + ASSERT_OK_AND_ASSIGN(bool result, + BlobViewStruct::IsBlobViewStruct(serialized->data(), serialized->size())); + ASSERT_TRUE(result); +} + +TEST_F(BlobViewStructTest, TestIsBlobViewStructWithTooShortBuffer) { + // Buffer shorter than 9 bytes should return false + std::vector short_buffer = {0x02, 0x43, 0x53, 0x45, 0x44, 0x42, 0x4F, 0x4C}; + ASSERT_OK_AND_ASSIGN( + bool result, BlobViewStruct::IsBlobViewStruct(short_buffer.data(), short_buffer.size())); + ASSERT_FALSE(result); + + // Empty buffer + ASSERT_OK_AND_ASSIGN(bool empty_result, BlobViewStruct::IsBlobViewStruct(nullptr, 0)); + ASSERT_FALSE(empty_result); +} + +TEST_F(BlobViewStructTest, TestIsBlobViewStructWithFutureVersion) { + // Version > CURRENT_VERSION should return false (not an error) + auto serialized = view_struct_.Serialize(pool_); + (*serialized)[0] = '\x02'; // set version to 2 (> CURRENT_VERSION) + ASSERT_OK_AND_ASSIGN(bool result, + BlobViewStruct::IsBlobViewStruct(serialized->data(), serialized->size())); + ASSERT_FALSE(result); +} + +TEST_F(BlobViewStructTest, TestIsBlobViewStructWithWrongMagic) { + // Wrong magic number should return false + auto serialized = view_struct_.Serialize(pool_); + // Corrupt the magic bytes (bytes 1-8) + (*serialized)[1] = '\x00'; + (*serialized)[2] = '\x00'; + ASSERT_OK_AND_ASSIGN(bool result, + BlobViewStruct::IsBlobViewStruct(serialized->data(), serialized->size())); + ASSERT_FALSE(result); +} + +TEST_F(BlobViewStructTest, TestIsBlobViewStructWithRandomData) { + // Random data that doesn't match format + std::vector random_data = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09}; + ASSERT_OK_AND_ASSIGN(bool result, + BlobViewStruct::IsBlobViewStruct(random_data.data(), random_data.size())); + ASSERT_FALSE(result); +} } // namespace paimon::test diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index c025c1ae..0b36c37b 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -99,6 +99,7 @@ const char Options::BLOB_FIELD[] = "blob-field"; const char Options::BLOB_DESCRIPTOR_FIELD[] = "blob-descriptor-field"; const char Options::FALLBACK_BLOB_DESCRIPTOR_FIELD[] = "blob.stored-descriptor-fields"; const char Options::BLOB_VIEW_FIELD[] = "blob-view-field"; +const char Options::BLOB_VIEW_UPSTREAM_WAREHOUSE[] = "blob-view-upstream-warehouse"; const char Options::BLOB_EXTERNAL_STORAGE_FIELD[] = "blob-external-storage-field"; const char Options::BLOB_EXTERNAL_STORAGE_PATH[] = "blob-external-storage-path"; const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled"; diff --git a/src/paimon/common/executor/executor.cpp b/src/paimon/common/executor/executor.cpp index f48e6d3b..0f944b4b 100644 --- a/src/paimon/common/executor/executor.cpp +++ b/src/paimon/common/executor/executor.cpp @@ -35,14 +35,14 @@ class DefaultExecutor : public Executor { ~DefaultExecutor() override; void Add(std::function func) override; - void ShutdownNow() override; + uint32_t GetThreadNum() const override; private: void WorkerThread(); - void ShutdownInternal(bool wait_for_pending_tasks); + private: uint32_t thread_count_; std::vector workers_; std::queue> tasks_; @@ -58,6 +58,10 @@ DefaultExecutor::DefaultExecutor(uint32_t thread_count) : thread_count_(thread_c } } +uint32_t DefaultExecutor::GetThreadNum() const { + return thread_count_; +} + void DefaultExecutor::ShutdownInternal(bool wait_for_pending_tasks) { { std::unique_lock lock(queue_mutex_); diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 3d579e07..599c3078 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -96,22 +96,22 @@ Status AppendOnlyWriter::Write(std::unique_ptr&& batch) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr transformed, external_storage_writer_->TransformBatch(struct_array)); auto transformed_struct = std::dynamic_pointer_cast(transformed); - // TODO(lc.lsz): validate blob view - PAIMON_RETURN_NOT_OK(BlobUtils::ValidateInlineBlobDescriptors(transformed_struct, - inline_descriptor_fields_)); + PAIMON_RETURN_NOT_OK(BlobUtils::ValidateBlobInlineFields( + transformed_struct, inline_descriptor_fields_, "blob-descriptor-field")); ::ArrowArray c_transformed; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*transformed, &c_transformed)); return writer_->Write(&c_transformed); } - if (!inline_descriptor_fields_.empty()) { + if (!inline_descriptor_fields_.empty() || !inline_view_fields_.empty()) { auto data_type = arrow::struct_(write_schema_->fields()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, arrow::ImportArray(batch->GetData(), data_type)); auto struct_array = std::dynamic_pointer_cast(arrow_array); - // TODO(lc.lsz): validate blob view - PAIMON_RETURN_NOT_OK( - BlobUtils::ValidateInlineBlobDescriptors(struct_array, inline_descriptor_fields_)); + PAIMON_RETURN_NOT_OK(BlobUtils::ValidateBlobInlineFields( + struct_array, inline_descriptor_fields_, "blob-descriptor-field")); + PAIMON_RETURN_NOT_OK(BlobUtils::ValidateBlobInlineFields(struct_array, inline_view_fields_, + "blob-view-field")); ::ArrowArray c_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, &c_array)); return writer_->Write(&c_array); @@ -191,9 +191,10 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWrit auto blob_context = BlobFileContext::Create(write_schema_, options_); std::optional> main_write_cols = write_cols_; - // Save inline descriptor fields for validation in Write() + // Save inline descriptor and view fields for validation in Write() if (blob_context) { inline_descriptor_fields_ = blob_context->GetDescriptorFields(); + inline_view_fields_ = blob_context->GetViewFields(); } // Initialize ExternalStorageBlobWriter if needed diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index d598725a..69b8c845 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -137,6 +137,7 @@ class AppendOnlyWriter : public BatchWriter { std::unique_ptr>> writer_; std::unique_ptr external_storage_writer_; std::set inline_descriptor_fields_; + std::set inline_view_fields_; }; } // namespace paimon diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp index db1adb87..32743f8b 100644 --- a/src/paimon/core/append/append_only_writer_test.cpp +++ b/src/paimon/core/append/append_only_writer_test.cpp @@ -35,7 +35,9 @@ #include "arrow/c/helpers.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/blob_view_struct.h" #include "paimon/common/fs/external_path_provider.h" #include "paimon/core/compact/compact_deletion_file.h" #include "paimon/core/compact/compact_result.h" @@ -707,4 +709,69 @@ TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) { ASSERT_OK(writer.Close()); } +TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) { + auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BLOB_VIEW_FIELD, "view"}}); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), "orc", options); + + auto schema = + arrow::schema({arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}); + AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, + memory_pool_); + + // Build f0 column + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.AppendValues({1, 2}).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + + // Build view column with valid BlobViewStruct values + arrow::LargeBinaryBuilder view_builder; + BlobViewStruct view_struct_0(Identifier("db", "tbl"), /*field_id=*/1, /*row_id=*/0); + auto view_bytes_0 = view_struct_0.Serialize(memory_pool_); + ASSERT_TRUE(view_builder.Append(view_bytes_0->data(), view_bytes_0->size()).ok()); + + BlobViewStruct view_struct_1(Identifier("db", "tbl"), /*field_id=*/1, /*row_id=*/1); + auto view_bytes_1 = view_struct_1.Serialize(memory_pool_); + ASSERT_TRUE(view_builder.Append(view_bytes_1->data(), view_bytes_1->size()).ok()); + + auto view_array = view_builder.Finish().ValueOrDie(); + ASSERT_OK(writer.Write(CreateStructBatch(schema, {int_array, view_array}))); + ASSERT_OK_AND_ASSIGN(auto inc, writer.PrepareCommit(/*wait_compaction=*/true)); + ASSERT_FALSE(inc.GetNewFilesIncrement().NewFiles().empty()); + ASSERT_OK(writer.Close()); +} + +TEST_F(AppendOnlyWriterTest, TestWriteInvalidBlobViewFieldRejected) { + auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BLOB_VIEW_FIELD, "view"}}); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), "orc", options); + + auto schema = + arrow::schema({arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}); + AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, + memory_pool_); + + // Build f0 column + arrow::Int32Builder int_builder; + ASSERT_TRUE(int_builder.Append(1).ok()); + auto int_array = int_builder.Finish().ValueOrDie(); + + // Build view column with raw bytes + arrow::LargeBinaryBuilder view_builder; + ASSERT_TRUE(view_builder.Append("not_a_valid_blob_view_or_descriptor").ok()); + auto view_array = view_builder.Finish().ValueOrDie(); + + ASSERT_NOK_WITH_MSG(writer.Write(CreateStructBatch(schema, {int_array, view_array})), + "BLOB inline field view require values to be set as corresponding type."); + ASSERT_OK(writer.Close()); +} + } // namespace paimon::test diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index ee34275b..ea5b1087 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -119,7 +119,7 @@ std::string FileSystemCatalog::GetDatabaseLocation(const std::string& db_name) c return NewDatabasePath(warehouse_, db_name); } -std::string FileSystemCatalog::GetTableLocation(const Identifier& identifier) const { +Result FileSystemCatalog::GetTableLocation(const Identifier& identifier) const { return NewDataTablePath(warehouse_, identifier); } @@ -158,7 +158,8 @@ Status FileSystemCatalog::CreateTable(const Identifier& identifier, ArrowSchema* return Status::NotImplemented( "create table operation does not support object store file system for now"); } - SchemaManager schema_manager(fs_, NewDataTablePath(warehouse_, identifier)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, NewDataTablePath(warehouse_, identifier)); + SchemaManager schema_manager(fs_, table_path); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr table_schema, schema_manager.CreateTable(schema, partition_keys, primary_keys, options)); @@ -172,7 +173,8 @@ Result>> FileSystemCatalog::TableSche return Status::NotImplemented( "do not support checking TableSchemaExists for system table."); } - SchemaManager schema_manager(fs_, NewDataTablePath(warehouse_, identifier)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, NewDataTablePath(warehouse_, identifier)); + SchemaManager schema_manager(fs_, table_path); return schema_manager.Latest(); } @@ -204,10 +206,11 @@ std::string FileSystemCatalog::NewDatabasePath(const std::string& warehouse, return PathUtil::JoinPath(warehouse, db_name + DB_SUFFIX); } -std::string FileSystemCatalog::NewDataTablePath(const std::string& warehouse, - const Identifier& identifier) { +Result FileSystemCatalog::NewDataTablePath(const std::string& warehouse, + const Identifier& identifier) { + PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); return PathUtil::JoinPath(NewDatabasePath(warehouse, identifier.GetDatabaseName()), - identifier.GetTableName()); + data_table_name); } Result> FileSystemCatalog::ListDatabases() const { @@ -279,9 +282,9 @@ Result> FileSystemCatalog::LoadTableSchema( if (branch) { dynamic_options[Options::BRANCH] = branch.value(); } + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(data_identifier)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, - SystemTableLoader::Load(system_table_name.value(), fs_, - GetTableLocation(data_identifier), + SystemTableLoader::Load(system_table_name.value(), fs_, table_path, latest_schema.value(), dynamic_options)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, system_table->ArrowSchema()); @@ -302,7 +305,8 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide return std::make_shared(schema, identifier.GetDatabaseName(), identifier.GetTableName()); } - return Table::Create(fs_, GetTableLocation(identifier), identifier); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); + return Table::Create(fs_, table_path, identifier); } Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, @@ -377,7 +381,7 @@ Result> FileSystemCatalog::GetTableBranches( Status FileSystemCatalog::DropTableImpl(const Identifier& identifier, const std::vector& external_paths) { - std::string table_path = GetTableLocation(identifier); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); // Delete external paths first for (const auto& external_path : external_paths) { @@ -397,8 +401,7 @@ Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if if (is_system_table) { return Status::Invalid(fmt::format("Cannot drop system table {}.", identifier.ToString())); } - - std::string table_path = GetTableLocation(identifier); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); PAIMON_ASSIGN_OR_RAISE(bool exist, fs_->Exists(table_path)); if (!exist) { if (ignore_if_not_exists) { @@ -484,8 +487,8 @@ Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identi return Status::Invalid(fmt::format("target table {} already exists", to_table.ToString())); } - std::string from_path = GetTableLocation(from_table); - std::string to_path = GetTableLocation(to_table); + PAIMON_ASSIGN_OR_RAISE(std::string from_path, GetTableLocation(from_table)); + PAIMON_ASSIGN_OR_RAISE(std::string to_path, GetTableLocation(to_table)); PAIMON_RETURN_NOT_OK(fs_->Rename(from_path, to_path)); return Status::OK(); } @@ -514,8 +517,7 @@ Result> FileSystemCatalog::ListSnapshots( if (!exists) { return Status::NotExist(fmt::format("table {} does not exist", identifier.ToString())); } - - auto table_path = GetTableLocation(identifier); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); SnapshotManager mgr(fs_, table_path, branch); PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, mgr.GetAllSnapshots()); std::sort(snapshots.begin(), snapshots.end(), diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index c4f83064..0d974cac 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -59,7 +59,7 @@ class FileSystemCatalog : public Catalog { Result DatabaseExists(const std::string& db_name) const override; Result TableExists(const Identifier& identifier) const override; std::string GetDatabaseLocation(const std::string& db_name) const override; - std::string GetTableLocation(const Identifier& identifier) const override; + Result GetTableLocation(const Identifier& identifier) const override; Result> LoadTableSchema(const Identifier& identifier) const override; std::string GetRootPath() const override; std::shared_ptr GetFileSystem() const override; @@ -69,7 +69,8 @@ class FileSystemCatalog : public Catalog { private: static std::string NewDatabasePath(const std::string& warehouse, const std::string& db_name); - static std::string NewDataTablePath(const std::string& warehouse, const Identifier& identifier); + static Result NewDataTablePath(const std::string& warehouse, + const Identifier& identifier); static bool IsSystemDatabase(const std::string& db_name); static Result IsSpecifiedSystemTable(const Identifier& identifier); static Result IsSystemTable(const Identifier& identifier); diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 605a1444..87dbd5a7 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -179,8 +179,8 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_FALSE(exists); ASSERT_OK_AND_ASSIGN(exists, catalog.TableExists(Identifier("db1", "missing$options"))); ASSERT_FALSE(exists); - ASSERT_EQ(catalog.GetTableLocation(options_identifier), - PathUtil::JoinPath(PathUtil::JoinPath(dir->Str(), "db1.db"), "tbl1$options")); + ASSERT_OK_AND_ASSIGN(auto table_path, catalog.GetTableLocation(options_identifier)); + ASSERT_EQ(table_path, PathUtil::JoinPath(PathUtil::JoinPath(dir->Str(), "db1.db"), "tbl1")); ASSERT_OK_AND_ASSIGN(std::shared_ptr system_schema, catalog.LoadTableSchema(options_identifier)); @@ -544,8 +544,8 @@ TEST(FileSystemCatalogTest, TestCreateTableWhileTableExist) { ASSERT_OK(catalog.CreateTable(identifier, &schema, {"f1"}, {}, options, /*ignore_if_exists=*/true)); ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFactory::Get("local", dir->Str(), {})); - ASSERT_OK(fs->Delete( - PathUtil::JoinPath(catalog.GetTableLocation(identifier), "schema/schema-0"))); + ASSERT_OK_AND_ASSIGN(std::string table_path, catalog.GetTableLocation(identifier)); + ASSERT_OK(fs->Delete(PathUtil::JoinPath(table_path, "schema/schema-0"))); ASSERT_OK(catalog.CreateTable(identifier, &schema, {"f1"}, {}, options, /*ignore_if_exists=*/false)); } @@ -612,8 +612,8 @@ TEST(FileSystemCatalogTest, TestValidateTableSchema) { ASSERT_NOK(table_schema->GetFieldType("f4")); ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFactory::Get("local", dir->Str(), {})); - std::string schema_path = - PathUtil::JoinPath(catalog.GetTableLocation(identifier), "schema/schema-0"); + ASSERT_OK_AND_ASSIGN(std::string table_path, catalog.GetTableLocation(identifier)); + std::string schema_path = PathUtil::JoinPath(table_path, "schema/schema-0"); std::string expected_json_schema; ASSERT_OK(fs->ReadFile(schema_path, &expected_json_schema)); diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index bd94a4c0..96977e77 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -396,6 +396,7 @@ struct CoreOptions::Impl { std::optional scan_fallback_branch; std::optional data_file_external_paths; std::optional blob_external_storage_path; + std::optional blob_view_upstream_warehouse; std::map raw_options; @@ -560,6 +561,9 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.ParseList(Options::BLOB_VIEW_FIELD, Options::FIELDS_SEPARATOR, &blob_view_fields, /*need_trim=*/true)); + // Parse blob-view-upstream-warehouse - warehouse path for configured blob view fields + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, &blob_view_upstream_warehouse)); // Parse blob-external-storage-field - descriptor BLOB fields written to external storage PAIMON_RETURN_NOT_OK(parser.ParseList( Options::BLOB_EXTERNAL_STORAGE_FIELD, Options::FIELDS_SEPARATOR, @@ -1425,6 +1429,10 @@ const std::vector& CoreOptions::GetBlobViewFields() const { return impl_->blob_view_fields; } +std::optional CoreOptions::GetBlobViewUpstreamWarehouse() const { + return impl_->blob_view_upstream_warehouse; +} + std::vector CoreOptions::GetBlobInlineFields() const { std::vector blob_inline_fields = impl_->blob_descriptor_fields; blob_inline_fields.insert(blob_inline_fields.end(), impl_->blob_view_fields.begin(), diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index c047700c..b064dec2 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -185,6 +185,7 @@ class PAIMON_EXPORT CoreOptions { const std::vector& GetBlobFields() const; const std::vector& GetBlobDescriptorFields() const; const std::vector& GetBlobViewFields() const; + std::optional GetBlobViewUpstreamWarehouse() const; std::vector GetBlobInlineFields() const; const std::vector& GetBlobExternalStorageFields() const; std::optional GetBlobExternalStoragePath() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 39fb1950..5d140d21 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -120,6 +120,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_TRUE(core_options.GetBlobViewFields().empty()); ASSERT_TRUE(core_options.GetBlobInlineFields().empty()); ASSERT_TRUE(core_options.GetBlobExternalStorageFields().empty()); + ASSERT_EQ(std::nullopt, core_options.GetBlobViewUpstreamWarehouse()); ASSERT_EQ(std::nullopt, core_options.GetBlobExternalStoragePath()); ASSERT_TRUE(core_options.LegacyPartitionNameEnabled()); ASSERT_TRUE(core_options.GlobalIndexEnabled()); @@ -225,6 +226,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::BLOB_VIEW_FIELD, "blob5"}, {Options::BLOB_EXTERNAL_STORAGE_FIELD, "blob3,blob4"}, {Options::BLOB_EXTERNAL_STORAGE_PATH, "FILE:///tmp/blob_external_storage/"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, "FILE:///tmp/blob_view_upstream_warehouse/"}, {Options::PARTITION_GENERATE_LEGACY_NAME, "false"}, {Options::GLOBAL_INDEX_ENABLED, "false"}, {Options::GLOBAL_INDEX_THREAD_NUM, "4"}, @@ -362,6 +364,8 @@ TEST(CoreOptionsTest, TestFromMap) { std::vector({"blob3", "blob4"})); ASSERT_EQ(core_options.GetBlobExternalStoragePath(), std::optional("FILE:///tmp/blob_external_storage/")); + ASSERT_EQ(core_options.GetBlobViewUpstreamWarehouse(), + std::optional("FILE:///tmp/blob_view_upstream_warehouse/")); ASSERT_FALSE(core_options.LegacyPartitionNameEnabled()); ASSERT_FALSE(core_options.GlobalIndexEnabled()); ASSERT_EQ(core_options.GetGlobalIndexThreadNum(), 4); diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index 83f12395..45b2cc73 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -52,6 +52,20 @@ Result> GlobalIndexScanImpl::Create( const Snapshot& snapshot, const std::shared_ptr& partitions, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) { + auto final_executor = executor; + if (!final_executor) { + std::optional thread_num = options.GetGlobalIndexThreadNum(); + if (thread_num) { + if (thread_num.value() <= 0) { + return Status::Invalid( + fmt::format("invalid global index thread number {}", thread_num.value())); + } + } else { + uint32_t cpu_count = std::thread::hardware_concurrency(); + thread_num = cpu_count > 0 ? static_cast(cpu_count) : 1; + } + final_executor = CreateDefaultExecutor(static_cast(thread_num.value())); + } auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, @@ -103,15 +117,6 @@ Result> GlobalIndexScanImpl::Create( index_metas[index_meta->index_field_id][index_file_meta->IndexType()][range].push_back( index_file_meta); } - auto final_executor = executor; - if (!final_executor) { - std::optional thread_num = options.GetGlobalIndexThreadNum(); - if (!thread_num) { - uint32_t cpu_count = std::thread::hardware_concurrency(); - thread_num = cpu_count > 0 ? static_cast(cpu_count) : 1; - } - final_executor = CreateDefaultExecutor(static_cast(thread_num.value())); - } return std::unique_ptr(new GlobalIndexScanImpl( table_schema, options, path_factory, std::move(index_metas), final_executor, pool)); } diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp index a8f56781..f17daf44 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp @@ -50,6 +50,10 @@ class InlineExecutor final : public Executor { } void ShutdownNow() override {} + + uint32_t GetThreadNum() const override { + return 1; + } }; class QueuedExecutor final : public Executor { @@ -60,6 +64,10 @@ class QueuedExecutor final : public Executor { void ShutdownNow() override {} + uint32_t GetThreadNum() const override { + return 1; + } + void RunAll() { for (auto& task : tasks_) { task(); diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 516ce826..8fb4adb7 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -172,17 +173,24 @@ Result> DataEvolutionSplitRead::WrapWithBlobViewRes if (read_blob_view_fields.empty()) { return std::move(inner_reader); } + std::optional warehouse_path = options_.GetBlobViewUpstreamWarehouse(); + if (!warehouse_path) { + return Status::Invalid( + "invalid config for blob view, supposed to set BLOB_VIEW_UPSTREAM_WAREHOUSE"); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr pre_reader, CreateBlobViewReader(data_split, read_blob_view_fields)); PAIMON_ASSIGN_OR_RAISE(std::unordered_set blob_view_structs, ExtractBlobViewStructs(pre_reader.get())); - std::string warehouse_path = - PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(context_->GetPath())); - auto catalog_context = std::make_shared(warehouse_path, options_.ToMap(), - options_.GetFileSystem()); + auto catalog_context = std::make_shared( + warehouse_path.value(), options_.ToMap(), options_.GetFileSystem()); + // use global thread number + uint32_t cpu_count = std::thread::hardware_concurrency(); + uint32_t thread_num = cpu_count > 0 ? cpu_count : 1; + std::shared_ptr executor = CreateDefaultExecutor(thread_num); PAIMON_ASSIGN_OR_RAISE( BlobViewResolver resolver, - BlobViewLookup::CreateResolver(blob_view_structs, catalog_context, pool_)); + BlobViewLookup::CreateResolver(blob_view_structs, catalog_context, pool_, executor)); return std::make_unique( std::move(inner_reader), std::move(read_blob_view_fields), std::move(resolver), pool_); } diff --git a/src/paimon/core/utils/blob_view_lookup.cpp b/src/paimon/core/utils/blob_view_lookup.cpp index 56979de6..cf41ca81 100644 --- a/src/paimon/core/utils/blob_view_lookup.cpp +++ b/src/paimon/core/utils/blob_view_lookup.cpp @@ -20,16 +20,21 @@ #include "paimon/core/utils/blob_view_lookup.h" #include +#include #include +#include #include "arrow/array.h" #include "arrow/c/bridge.h" #include "fmt/format.h" #include "paimon/catalog/catalog.h" #include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/executor/future.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/defs.h" +#include "paimon/executor.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/memory/bytes.h" #include "paimon/read_context.h" @@ -50,6 +55,10 @@ void BlobViewLookup::TableReadPlan::Add(const BlobViewStruct& view_struct) { row_ranges_.push_back(view_struct.RowId()); } +const Identifier& BlobViewLookup::TableReadPlan::GetIdentifier() const { + return identifier_; +} + std::vector BlobViewLookup::TableReadPlan::GetFieldIds() const { return std::vector(references_by_field_id_.begin(), references_by_field_id_.end()); } @@ -80,10 +89,10 @@ std::vector BlobViewLookup::TableReadPlan::GetSortedDistinctRanges() cons Result BlobViewLookup::CreateResolver( const std::unordered_set& view_structs, - const std::shared_ptr& catalog_context, - const std::shared_ptr& pool) { + const std::shared_ptr& catalog_context, const std::shared_ptr& pool, + const std::shared_ptr& executor) { PAIMON_ASSIGN_OR_RAISE(DescriptorMapping mapping, - PreloadDescriptors(view_structs, catalog_context, pool)); + PreloadDescriptors(view_structs, catalog_context, pool, executor)); return BlobViewResolver([cached = std::move(mapping)](const BlobViewStruct& view_struct) -> Result> { auto iter = cached.find(view_struct); @@ -97,53 +106,109 @@ Result BlobViewLookup::CreateResolver( Result BlobViewLookup::PreloadDescriptors( const std::unordered_set& view_structs, - const std::shared_ptr& catalog_context, - const std::shared_ptr& pool) { + const std::shared_ptr& catalog_context, const std::shared_ptr& pool, + const std::shared_ptr& executor) { std::unordered_map plan_by_identifier = GroupByIdentifier(view_structs); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, - Catalog::Create(catalog_context->root_path, catalog_context->options, - catalog_context->file_system)); - DescriptorMapping mapping; + int64_t target_rows_per_task = TargetRowsPerTask(plan_by_identifier, executor->GetThreadNum()); + + std::vector>> futures; for (const auto& [identifier, table_read_plan] : plan_by_identifier) { - std::string source_table_path = catalog->GetTableLocation(identifier); - PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); - ScanContextBuilder scan_builder(source_table_path); - auto global_index_result = - BitmapGlobalIndexResult::FromRanges(table_read_plan.GetSortedDistinctRanges()); - scan_builder.SetGlobalIndexResult(global_index_result) - .WithMemoryPool(pool) - .WithFileSystem(catalog_context->file_system); - if (branch) { - scan_builder.AddOption(Options::BRANCH, branch.value()); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, - TableScan::Create(std::move(scan_context))); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); - - ReadContextBuilder read_builder(source_table_path); + const auto& id = identifier; std::vector field_ids = table_read_plan.GetFieldIds(); - field_ids.push_back(SpecialFieldIds::ROW_ID); - read_builder.SetReadFieldIds(field_ids) - .AddOption(Options::BLOB_AS_DESCRIPTOR, "true") - .EnablePrefetch(true) - .WithMemoryPool(pool) - .WithFileSystem(catalog_context->file_system); - if (branch) { - read_builder.WithBranch(branch.value()); - } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, - TableRead::Create(std::move(read_context))); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, - table_read->CreateReader(plan->Splits())); - PAIMON_RETURN_NOT_OK( - ExtractBlobDescriptors(identifier, field_ids, pool, reader.get(), &mapping)); + std::vector> range_chunks = + SplitRowRanges(table_read_plan.GetSortedDistinctRanges(), target_rows_per_task); + for (const auto& range_chunk : range_chunks) { + futures.push_back(Via( + executor.get(), + [catalog_context, id, field_ids, range_chunk, pool]() -> Result { + return LoadTableDescriptorChunk(catalog_context, id, field_ids, range_chunk, + pool); + })); + } } + + DescriptorMapping mapping; + std::vector> chunk_results = CollectAll(futures); + for (auto& chunk_result : chunk_results) { + if (!chunk_result.ok()) { + return chunk_result.status(); + } + for (const auto& [view_struct, descriptor] : chunk_result.value()) { + mapping[view_struct] = descriptor; + } + } + return mapping; +} + +Result BlobViewLookup::LoadTableDescriptorChunk( + const std::shared_ptr& catalog_context, const Identifier& identifier, + const std::vector& field_ids, const std::vector& row_ranges, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + if (branch) { + return Status::Invalid("do not support upstream table with branch"); + } + auto file_system = catalog_context->file_system; + PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(catalog_context, identifier)); + ScanContextBuilder scan_builder(table_path); + auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); + scan_builder.SetGlobalIndexResult(global_index_result) + .WithMemoryPool(pool) + .WithFileSystem(file_system); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + + ReadContextBuilder read_builder(table_path); + std::vector read_field_ids = field_ids; + read_field_ids.push_back(SpecialFieldIds::ROW_ID); + read_builder.SetReadFieldIds(read_field_ids) + .AddOption(Options::BLOB_AS_DESCRIPTOR, "true") + .EnablePrefetch(true) + .WithMemoryPool(pool) + .WithFileSystem(file_system); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + table_read->CreateReader(plan->Splits())); + + DescriptorMapping mapping; + PAIMON_RETURN_NOT_OK( + ExtractBlobDescriptors(identifier, read_field_ids, pool, reader.get(), &mapping)); return mapping; } +Result BlobViewLookup::GetTableLocation( + const std::shared_ptr& catalog_context, const Identifier& identifier) { + auto file_system = catalog_context->file_system; + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr catalog, + Catalog::Create(catalog_context->root_path, catalog_context->options, file_system)); + // The table path may be either xxx/test_database/test_table or + // xxx/test_database.db/test_table. If neither path exists or both paths exist, it means we + // cannot infer the table path, and an error will be reported. If only one of the paths + // exists, we will use that path. + PAIMON_ASSIGN_OR_RAISE(std::string source_table_path, catalog->GetTableLocation(identifier)); + std::string database_name = identifier.GetDatabaseName(); + PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); + std::string fallback_source_table_path = PathUtil::JoinPath( + PathUtil::JoinPath(catalog_context->root_path, database_name), data_table_name); + + PAIMON_ASSIGN_OR_RAISE(bool exist, catalog_context->file_system->Exists(source_table_path)); + PAIMON_ASSIGN_OR_RAISE(bool fallback_exist, file_system->Exists(fallback_source_table_path)); + if (exist == fallback_exist) { + return Status::Invalid( + fmt::format("Ambiguous table path: both table path {} and fallback table path {} are " + "present or absent", + source_table_path, fallback_source_table_path)); + } + std::string final_table_path = exist ? source_table_path : fallback_source_table_path; + return final_table_path; +} + Status BlobViewLookup::ExtractBlobDescriptors(const Identifier& identifier, const std::vector& field_ids, const std::shared_ptr& pool, @@ -241,4 +306,46 @@ std::unordered_map BlobViewLookup::Gr return grouped; } +int64_t BlobViewLookup::TargetRowsPerTask( + const std::unordered_map& plan_by_identifier, uint32_t thread_num) { + int64_t total_rows = 0; + for (const auto& [identifier, table_read_plan] : plan_by_identifier) { + for (const auto& row_range : table_read_plan.GetSortedDistinctRanges()) { + total_rows += row_range.Count(); + } + } + int64_t balanced_rows = (total_rows + thread_num - 1) / thread_num; + return std::max(MIN_ROW_PER_TASK, balanced_rows); +} + +std::vector> BlobViewLookup::SplitRowRanges(const std::vector& row_ranges, + int64_t target_rows_per_task) { + if (row_ranges.empty()) { + return {}; + } + std::vector> chunks; + std::vector current_chunk; + int64_t current_chunk_rows = 0; + for (const auto& row_range : row_ranges) { + int64_t next_from = row_range.from; + while (next_from <= row_range.to) { + if (current_chunk_rows == target_rows_per_task) { + chunks.push_back(current_chunk); + current_chunk.clear(); + current_chunk_rows = 0; + } + + int64_t remaining_rows = target_rows_per_task - current_chunk_rows; + int64_t next_to = std::min(row_range.to, next_from + remaining_rows - 1); + current_chunk.emplace_back(next_from, next_to); + current_chunk_rows += next_to - next_from + 1; + next_from = next_to + 1; + } + } + if (!current_chunk.empty()) { + chunks.push_back(current_chunk); + } + return chunks; +} + } // namespace paimon diff --git a/src/paimon/core/utils/blob_view_lookup.h b/src/paimon/core/utils/blob_view_lookup.h index 81602945..169d9f83 100644 --- a/src/paimon/core/utils/blob_view_lookup.h +++ b/src/paimon/core/utils/blob_view_lookup.h @@ -42,6 +42,8 @@ class BatchReader; class BlobViewLookup { public: using DescriptorMapping = std::unordered_map>; + /// The minimum number of rows handled by a single parallel task. + static constexpr int64_t MIN_ROW_PER_TASK = 100; BlobViewLookup() = delete; ~BlobViewLookup() = delete; @@ -49,7 +51,7 @@ class BlobViewLookup { static Result CreateResolver( const std::unordered_set& view_structs, const std::shared_ptr& catalog_context, - const std::shared_ptr& pool); + const std::shared_ptr& pool, const std::shared_ptr& executor); private: class TableReadPlan { @@ -57,6 +59,7 @@ class BlobViewLookup { explicit TableReadPlan(const BlobViewStruct& view_struct); void Add(const BlobViewStruct& view_struct); + const Identifier& GetIdentifier() const; std::vector GetFieldIds() const; std::vector GetSortedDistinctRanges() const; @@ -69,6 +72,11 @@ class BlobViewLookup { static Result PreloadDescriptors( const std::unordered_set& view_structs, const std::shared_ptr& catalog_context, + const std::shared_ptr& pool, const std::shared_ptr& executor); + + static Result LoadTableDescriptorChunk( + const std::shared_ptr& catalog_context, const Identifier& identifier, + const std::vector& field_ids, const std::vector& row_ranges, const std::shared_ptr& pool); static Status ExtractBlobDescriptors(const Identifier& identifier, @@ -78,6 +86,16 @@ class BlobViewLookup { static std::unordered_map GroupByIdentifier( const std::unordered_set& view_structs); + + static int64_t TargetRowsPerTask( + const std::unordered_map& plan_by_identifier, + uint32_t thread_num); + + static std::vector> SplitRowRanges(const std::vector& row_ranges, + int64_t target_rows_per_task); + + static Result GetTableLocation( + const std::shared_ptr& catalog_context, const Identifier& identifier); }; } // namespace paimon diff --git a/src/paimon/core/utils/blob_view_lookup_test.cpp b/src/paimon/core/utils/blob_view_lookup_test.cpp index 6dc2b89c..6cff2384 100644 --- a/src/paimon/core/utils/blob_view_lookup_test.cpp +++ b/src/paimon/core/utils/blob_view_lookup_test.cpp @@ -106,10 +106,8 @@ TEST_F(BlobViewLookupTest, TestGetSortedDistinctRangesMergesContiguousAndGaps) { auto ranges = plan.GetSortedDistinctRanges(); ASSERT_EQ(ranges.size(), 2U); - ASSERT_EQ(ranges[0].from, 5); - ASSERT_EQ(ranges[0].to, 7); - ASSERT_EQ(ranges[1].from, 10); - ASSERT_EQ(ranges[1].to, 11); + ASSERT_EQ(ranges[0], Range(5, 7)); + ASSERT_EQ(ranges[1], Range(10, 11)); } TEST_F(BlobViewLookupTest, TestGetSortedDistinctRangesWithNonContiguous) { @@ -119,12 +117,9 @@ TEST_F(BlobViewLookupTest, TestGetSortedDistinctRangesWithNonContiguous) { const auto ranges = plan.GetSortedDistinctRanges(); ASSERT_EQ(ranges.size(), 3U); - ASSERT_EQ(ranges[0].from, 1); - ASSERT_EQ(ranges[0].to, 1); - ASSERT_EQ(ranges[1].from, 50); - ASSERT_EQ(ranges[1].to, 50); - ASSERT_EQ(ranges[2].from, 100); - ASSERT_EQ(ranges[2].to, 100); + ASSERT_EQ(ranges[0], Range(1, 1)); + ASSERT_EQ(ranges[1], Range(50, 50)); + ASSERT_EQ(ranges[2], Range(100, 100)); } TEST_F(BlobViewLookupTest, TestEmptyInputProducesEmptyOutput) { @@ -191,4 +186,70 @@ TEST_F(BlobViewLookupTest, TestViewStructsOfDifferentTablesAreSplitIntoDistinctP ASSERT_EQ(plan_db2_t1.row_ranges_.size(), 1U); } +TEST_F(BlobViewLookupTest, TestGetIdentifier) { + BlobViewLookup::TableReadPlan plan(MakeView("db", "t", /*field_id=*/1, /*row_id=*/0)); + ASSERT_EQ(plan.GetIdentifier(), MakeIdentifier("db", "t")); +} + +TEST_F(BlobViewLookupTest, TestTargetRowsPerTaskEmptyReturnsMin) { + std::unordered_map empty; + ASSERT_EQ(BlobViewLookup::TargetRowsPerTask(empty, /*thread_num=*/100), + BlobViewLookup::MIN_ROW_PER_TASK); +} + +TEST_F(BlobViewLookupTest, TestTargetRowsPerTaskSmallTotalReturnsMin) { + std::unordered_set views; + for (int64_t row_id = 0; row_id < 10; ++row_id) { + views.emplace(MakeView("db", "t", /*field_id=*/1, row_id)); + } + auto grouped = BlobViewLookup::GroupByIdentifier(views); + // total_rows (10) is far below thread_num, so the balanced budget is clamped to + // MIN_ROW_PER_TASK. + ASSERT_EQ(BlobViewLookup::TargetRowsPerTask(grouped, /*thread_num=*/100), + BlobViewLookup::MIN_ROW_PER_TASK); +} + +TEST_F(BlobViewLookupTest, TestTargetRowsPerTaskLargeTotalBalancesAcrossThreads) { + std::unordered_set views; + const int64_t total_rows = 100001; + for (int64_t row_id = 0; row_id < total_rows; ++row_id) { + views.emplace(MakeView("db", "t", /*field_id=*/1, row_id)); + } + auto grouped = BlobViewLookup::GroupByIdentifier(views); + // ceil(100001 / 100) = 1001 + ASSERT_EQ(BlobViewLookup::TargetRowsPerTask(grouped, /*thread_num=*/100), 1001); +} + +TEST_F(BlobViewLookupTest, TestSplitRowRangesEmptyInput) { + auto chunks = BlobViewLookup::SplitRowRanges({}, /*target_rows_per_task=*/10); + ASSERT_TRUE(chunks.empty()); +} + +TEST_F(BlobViewLookupTest, TestSplitRowRangesSingleRangeFitsInOneChunk) { + auto chunks = BlobViewLookup::SplitRowRanges({Range(0, 4)}, /*target_rows_per_task=*/10); + ASSERT_EQ(chunks.size(), 1U); + ASSERT_EQ(chunks[0].size(), 1U); + ASSERT_EQ(chunks[0][0], Range(0, 4)); +} + +TEST_F(BlobViewLookupTest, TestSplitRowRangesSplitsLargeRange) { + // [0, 9] with target 4 => [0,3], [4,7], [8,9] + auto chunks = BlobViewLookup::SplitRowRanges({Range(0, 9)}, /*target_rows_per_task=*/4); + ASSERT_EQ(chunks.size(), 3U); + ASSERT_EQ(chunks[0], (std::vector{Range(0, 3)})); + ASSERT_EQ(chunks[1], (std::vector{Range(4, 7)})); + ASSERT_EQ(chunks[2], (std::vector{Range(8, 9)})); +} + +TEST_F(BlobViewLookupTest, TestSplitRowRangesPacksAcrossRanges) { + // Ranges [0,2] (3 rows) and [10,12] (3 rows) with target 4. + // chunk0 = [0,2] + part of second range [10,10] (total 4 rows) + // chunk1 = [11,12] (2 rows) + auto chunks = + BlobViewLookup::SplitRowRanges({Range(0, 2), Range(10, 12)}, /*target_rows_per_task=*/4); + ASSERT_EQ(chunks.size(), 2U); + ASSERT_EQ(chunks[0], (std::vector{Range(0, 2), Range(10, 10)})); + ASSERT_EQ(chunks[1], (std::vector{Range(11, 12)})); +} + } // namespace paimon::test diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 4db53384..00433dcd 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -18,6 +18,8 @@ #include #include +#include +#include #include #include #include @@ -408,7 +410,8 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter }; std::vector GetTestValuesForBlobTableInteTest() { - std::vector values = {"parquet"}; + std::vector values; + values.emplace_back("parquet"); #ifdef PAIMON_ENABLE_ORC values.emplace_back("orc"); #endif @@ -1262,7 +1265,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionAndAlterTable) { DataField(3, arrow::field("f1", arrow::utf8())), DataField(4, arrow::field("f2", arrow::decimal128(6, 3))), DataField(5, arrow::field("f0", arrow::boolean())), - DataField(8, BlobUtils::ToArrowField("f5")), + DataField(8, BlobUtils::ToArrowField("blob")), DataField(9, arrow::field("f6", arrow::int32())), SpecialFields::RowId(), SpecialFields::SequenceNumber()}; @@ -1274,7 +1277,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionAndAlterTable) { // only read blob column auto expected_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON( - arrow::struct_({BlobUtils::ToArrowField("f5")}), R"([ + arrow::struct_({BlobUtils::ToArrowField("blob")}), R"([ ["Lily"], ["Alice"], ["Bob"], @@ -1287,7 +1290,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionAndAlterTable) { ["Elderberry"] ])") .ValueOrDie()); - ASSERT_OK(ScanAndRead(table_path, {"f5"}, expected_array)); + ASSERT_OK(ScanAndRead(table_path, {"blob"}, expected_array)); } { auto expected_array = std::dynamic_pointer_cast( @@ -2372,9 +2375,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) { auto schema = arrow::schema(fields); ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, schema->field_names(), {raw_array}), - "BLOB inline field b0 configured by blob-descriptor-field or " - "blob-view-field require values " - "to be a BlobDescriptor or BlobViewStruct."); + "BLOB inline field b0 require values to be set as corresponding type."); } TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { @@ -2393,13 +2394,15 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}; - std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, file_format}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_VIEW_FIELD, "view"}, - {Options::FILE_SYSTEM, "local"}}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -2410,7 +2413,7 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { if (i < 6) { BlobViewStruct view_struct(upstream_identifier, /*field_id=*/6, /*row_id=*/static_cast(i)); - auto serialized = view_struct.Serialize(GetDefaultPool()); + auto serialized = view_struct.Serialize(pool_); ASSERT_TRUE(view_builder .Append(reinterpret_cast(serialized->data()), serialized->size()) @@ -2539,4 +2542,463 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { } } +TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamExternalStorageBlob) { + auto file_format = GetParam(); + if (GetParam() == "lance") { + return; + } + // Upstream table has two blob descriptor fields: b0 (field_id=1, inline descriptor) and + // b1 (field_id=2, descriptor + external storage). The downstream view references cells from + // both b0 and b1. + const std::string upstream_db_name = "upstream_two_blob"; + const std::string upstream_table_name = "upstream_two_blob"; + arrow::FieldVector upstream_fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true)}; + auto upstream_schema = arrow::schema(upstream_fields); + std::map upstream_options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, + {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, + {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + + ::ArrowSchema upstream_c_schema; + ASSERT_TRUE(arrow::ExportSchema(*upstream_schema, &upstream_c_schema).ok()); + ASSERT_OK_AND_ASSIGN(auto upstream_catalog, + Catalog::Create(dir_->Str(), {{Options::FILE_SYSTEM, "local"}})); + ASSERT_OK(upstream_catalog->CreateDatabase(upstream_db_name, {}, /*ignore_if_exists=*/true)); + ASSERT_OK(upstream_catalog->CreateTable( + Identifier(upstream_db_name, upstream_table_name), &upstream_c_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, upstream_options, + /*ignore_if_exists=*/false)); + std::string upstream_table_path = + PathUtil::JoinPath(dir_->Str(), upstream_db_name + ".db/" + upstream_table_name); + + // Write 4 rows of b0/b1 data into the upstream table. + std::string upstream_raw_json = R"([ +[0, "b0_data_0", "b1_data_0"], +[1, "b0_data_1", "b1_data_1"], +[2, "b0_data_2", "b1_data_2"], +[3, "b0_data_3", "b1_data_3"] +])"; + auto upstream_raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(upstream_fields), + upstream_raw_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto upstream_desc_array, + ConvertRawBlobToDescriptor(upstream_raw_array, {"b0", "b1"})); + ASSERT_OK_AND_ASSIGN( + auto upstream_commit_msgs, + WriteArray(upstream_table_path, {}, upstream_schema->field_names(), {upstream_desc_array})); + ASSERT_OK(Commit(upstream_table_path, upstream_commit_msgs)); + + // Create the downstream blob-view table that references both b0 and b1 of the upstream table. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Build the view column mixing references to b0 (field_id=1) and b1 (field_id=2). + // - row 0: b0 row 0 -> "b0_data_0" + // - row 1: b1 row 1 -> "b1_data_1" + // - row 2: b0 row 2 -> "b0_data_2" + // - row 3: b1 row 3 -> "b1_data_3" + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + auto append_view = [&](int32_t field_id, int64_t row_id, arrow::LargeBinaryBuilder* builder) { + BlobViewStruct view_struct(upstream_identifier, field_id, row_id); + auto serialized = view_struct.Serialize(pool_); + ASSERT_TRUE( + builder + ->Append(reinterpret_cast(serialized->data()), serialized->size()) + .ok()); + }; + arrow::LargeBinaryBuilder view_builder; + append_view(/*field_id=*/1, /*row_id=*/0, &view_builder); + append_view(/*field_id=*/2, /*row_id=*/1, &view_builder); + append_view(/*field_id=*/1, /*row_id=*/2, &view_builder); + append_view(/*field_id=*/2, /*row_id=*/3, &view_builder); + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + + auto write_f0_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([100,101,102,103])") + .ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Scan/read the downstream table and verify the resolved view blobs. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 4); + ASSERT_OK_AND_ASSIGN(auto result_array, ConvertDescriptorToRawBlob(read_struct, {"view"})); + + std::string expected_json = R"([ +[100, "b0_data_0"], +[101, "b1_data_1"], +[102, "b0_data_2"], +[103, "b1_data_3"] +])"; + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + ASSERT_TRUE(result_array->Equals(expected_with_rk)) + << "result_array:" << result_array->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestBlobViewFieldWithMultipleUpstreamTables) { + auto file_format = GetParam(); + if (file_format != "orc" && file_format != "parquet") { + return; + } + + // Upstream table 1: append_table_with_multi_blob, with two blob fields f5 (field_id=5) and + // f6 (field_id=6). + const std::string multi_blob_db_name = "append_table_with_multi_blob"; + const std::string multi_blob_table_name = "append_table_with_multi_blob"; + { + std::string src_db_path = paimon::test::GetDataDir() + file_format + "/" + + multi_blob_db_name + ".db/" + multi_blob_table_name; + std::string dst_db_path = + PathUtil::JoinPath(dir_->Str(), multi_blob_db_name + ".db/" + multi_blob_table_name); + ASSERT_TRUE(TestUtil::CopyDirectory(src_db_path, dst_db_path)); + } + + // Upstream table 2: blob_append_table_alter_table_with_cast_with_data_evolution, with one blob + // field blob (field_id=8). + const std::string alter_db_name = "blob_append_table_alter_table_with_cast_with_data_evolution"; + const std::string alter_table_name = + "blob_append_table_alter_table_with_cast_with_data_evolution"; + { + std::string src_db_path = paimon::test::GetDataDir() + file_format + "/" + alter_db_name + + ".db/" + alter_table_name; + std::string dst_db_path = + PathUtil::JoinPath(dir_->Str(), alter_db_name + ".db/" + alter_table_name); + ASSERT_TRUE(TestUtil::CopyDirectory(src_db_path, dst_db_path)); + } + + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view1", true), + BlobUtils::ToArrowField("view2", true)}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view1,view2"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, dir_->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + Identifier multi_blob_identifier(multi_blob_db_name, multi_blob_table_name); + Identifier alter_identifier(alter_db_name, alter_table_name); + + auto append_view = [&](const Identifier& identifier, int32_t field_id, int64_t row_id, + arrow::LargeBinaryBuilder* builder) { + BlobViewStruct view_struct(identifier, field_id, row_id); + auto serialized = view_struct.Serialize(pool_); + ASSERT_TRUE( + builder + ->Append(reinterpret_cast(serialized->data()), serialized->size()) + .ok()); + }; + + // Build view1 column. References multi_blob.f5 (field_id=5) and f6 (field_id=6). + // Some upstream cells are referenced more than once on purpose. + // - row 0: f5 row 3 -> 'D' * 1024 + // - row 1: f6 row 1 -> 'b' * 2048 + // - row 2: f5 row 3 -> 'D' * 1024 (repeat of row 0) + // - row 3: f6 row 4 -> 'e' * 2048 + // - row 4: f5 row 3 -> 'D' * 1024 (repeat of row 0) + // - row 5: f6 row 1 -> 'b' * 2048 (repeat of row 1) + // - row 6: f5 row 5 -> 'F' * 1024 + // - row 7: f6 row 6 -> 'g' * 2048 + arrow::LargeBinaryBuilder view1_builder; + append_view(multi_blob_identifier, /*field_id=*/5, /*row_id=*/3, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/6, /*row_id=*/1, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/5, /*row_id=*/3, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/6, /*row_id=*/4, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/5, /*row_id=*/3, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/6, /*row_id=*/1, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/5, /*row_id=*/5, &view1_builder); + append_view(multi_blob_identifier, /*field_id=*/6, /*row_id=*/6, &view1_builder); + std::shared_ptr write_view1_array; + ASSERT_TRUE(view1_builder.Finish(&write_view1_array).ok()); + + // Build view2 column. References alter.blob (field_id=8). + // Some upstream cells are referenced more than once on purpose. + // - row 0: blob row 0 -> "Lily" + // - row 1: blob row 5 -> "Apple" + // - row 2: blob row 0 -> "Lily" (repeat of row 0) + // - row 3: blob row 2 -> "Bob" + // - row 4: blob row 5 -> "Apple" (repeat of row 1) + // - row 5: blob row 9 -> "Elderberry" + // - row 6: blob row 0 -> "Lily" (repeat of row 0) + // - row 7: blob row 3 -> "Cindy" + arrow::LargeBinaryBuilder view2_builder; + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/0, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/5, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/0, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/2, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/5, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/9, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/0, &view2_builder); + append_view(alter_identifier, /*field_id=*/8, /*row_id=*/3, &view2_builder); + std::shared_ptr write_view2_array; + ASSERT_TRUE(view2_builder.Finish(&write_view2_array).ok()); + + auto write_f0_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int32(), R"([100,101,102,103,104,105,106,107])") + .ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make( + arrow::ArrayVector({write_f0_array, write_view1_array, write_view2_array}), + std::vector({"f0", "view1", "view2"})) + .ValueOrDie()); + + // write & commit + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Expected blob contents per referenced upstream cell. + std::string blob_f5_row3(1024, 'D'); // multi_blob.f5 row 3 + std::string blob_f5_row5(1024, 'F'); // multi_blob.f5 row 5 + std::string blob_f6_row1(2048, 'b'); // multi_blob.f6 row 1 + std::string blob_f6_row4(2048, 'e'); // multi_blob.f6 row 4 + std::string blob_f6_row6(2048, 'g'); // multi_blob.f6 row 6 + + std::vector read_fields = {"view2", "view1", "f0"}; + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadTable(table_path, read_fields, plan, /*predicate=*/nullptr)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 8); + ASSERT_OK_AND_ASSIGN(auto result_array, + ConvertDescriptorToRawBlob(read_struct, {"view1", "view2"})); + + // Expected struct follows the requested (shuffled) column order: view2, view1, f0. + arrow::FieldVector expected_fields = {BlobUtils::ToArrowField("view2", true), + BlobUtils::ToArrowField("view1", true), + arrow::field("f0", arrow::int32())}; + // clang-format off + std::string expected_json = R"([ +["Lily", ")" + blob_f5_row3 + R"(", 100], +["Apple", ")" + blob_f6_row1 + R"(", 101], +["Lily", ")" + blob_f5_row3 + R"(", 102], +["Bob", ")" + blob_f6_row4 + R"(", 103], +["Apple", ")" + blob_f5_row3 + R"(", 104], +["Elderberry", ")" + blob_f6_row1 + R"(", 105], +["Lily", ")" + blob_f5_row5 + R"(", 106], +["Cindy", ")" + blob_f6_row6 + R"(", 107] +])"; + // clang-format on + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(expected_fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + + ASSERT_TRUE(result_array->Equals(expected_with_rk)) + << "result_array:" << result_array->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestBlobViewFailsWhenBothPathsAbsent) { + auto file_format = GetParam(); + if (GetParam() == "lance") { + return; + } + auto upstream_dir = UniqueTestDirectory::Create("local"); + const std::string upstream_db_name = "nonexistent_db"; + const std::string upstream_table_name = "nonexistent_table"; + + // Build downstream table that references the non-existent upstream table. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, upstream_dir->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Write a single row with a BlobViewStruct pointing to the non-existent upstream table. + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + BlobViewStruct view_struct(upstream_identifier, /*field_id=*/2, /*row_id=*/0); + auto serialized = view_struct.Serialize(pool_); + arrow::LargeBinaryBuilder view_builder; + ASSERT_TRUE( + view_builder + .Append(reinterpret_cast(serialized->data()), serialized->size()) + .ok()); + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + auto write_f0_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([100])").ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Reading should fail because both paths are absent. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_NOK_WITH_MSG(ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr), + "Ambiguous table path"); +} + +TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) { + auto file_format = GetParam(); + if (GetParam() == "lance") { + return; + } + const std::string upstream_db_name = "fallback_db"; + const std::string upstream_table_name = "fallback_table"; + arrow::FieldVector upstream_fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + auto upstream_schema = arrow::schema(upstream_fields); + std::map upstream_options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::FILE_SYSTEM, "local"}}; + + // Create the upstream table at the fallback path: /db/table (no .db). + auto upstream_dir = UniqueTestDirectory::Create("local"); + std::string fallback_table_path = + PathUtil::JoinPath(upstream_dir->Str(), upstream_db_name + "/" + upstream_table_name); + + // Manually create schema at fallback path so it can be read as a valid paimon table. + { + // Use a temporary warehouse with Catalog to build the table data, then copy to fallback. + auto temp_dir = UniqueTestDirectory::Create("local"); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*upstream_schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(auto catalog, + Catalog::Create(temp_dir->Str(), {{Options::FILE_SYSTEM, "local"}})); + ASSERT_OK(catalog->CreateDatabase(upstream_db_name, {}, /*ignore_if_exists=*/true)); + ASSERT_OK(catalog->CreateTable(Identifier(upstream_db_name, upstream_table_name), &c_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, upstream_options, + /*ignore_if_exists=*/false)); + std::string temp_table_path = + PathUtil::JoinPath(temp_dir->Str(), upstream_db_name + ".db/" + upstream_table_name); + + // Write data to the temp table. + std::string raw_json = R"([[0, "hello"], [1, "world"]])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(upstream_fields), raw_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + ASSERT_OK_AND_ASSIGN( + auto upstream_commit_msgs, + WriteArray(temp_table_path, {}, upstream_schema->field_names(), {desc_array})); + ASSERT_OK(Commit(temp_table_path, upstream_commit_msgs)); + + // Copy the temp table to the fallback path (without .db). + ASSERT_TRUE(TestUtil::CopyDirectory(temp_table_path, fallback_table_path)); + } + + // Build the downstream table. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, upstream_dir->Str()}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Write downstream rows referencing the upstream fallback table. + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + arrow::LargeBinaryBuilder view_builder; + for (int64_t row = 0; row < 2; ++row) { + BlobViewStruct view_struct(upstream_identifier, /*field_id=*/1, /*row_id=*/row); + auto serialized = view_struct.Serialize(pool_); + ASSERT_TRUE( + view_builder + .Append(reinterpret_cast(serialized->data()), serialized->size()) + .ok()); + } + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + auto write_f0_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), R"([100, 101])").ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Read and verify + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadTable(table_path, schema->field_names(), plan, /*predicate=*/nullptr)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_EQ(read_struct->length(), 2); + ASSERT_OK_AND_ASSIGN(auto result_array, ConvertDescriptorToRawBlob(read_struct, {"view"})); + + std::string expected_json = R"([[100, "hello"], [101, "world"]])"; + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + ASSERT_TRUE(result_array->Equals(expected_with_rk)) + << "result_array:" << result_array->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + } // namespace paimon::test diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 0d40181a..685159d0 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -564,7 +564,7 @@ TEST(SystemTableReadInteTest, TestReadOptionsSystemTable) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - std::string system_table_path = catalog->GetTableLocation(Identifier("db1", "tbl1$options")); + std::string system_table_path = PathUtil::JoinPath(dir->Str(), "warehouse/db1.db/tbl1$options"); ScanContextBuilder scan_context_builder(system_table_path); scan_context_builder.SetOptions(options); ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README index 9f3f85b1..3e17ccd4 100644 --- a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README +++ b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/README @@ -42,8 +42,7 @@ renameColumn("f3", "f0") renameColumn("f5", "f3") updateColumnPosition(SchemaChange.Move.first("f4")) addColumn("f6", INT()) -renameColumn("blob", "f5") -updateColumnPosition(SchemaChange.Move.after("f5", "f2")) +updateColumnPosition(SchemaChange.Move.after("blob", "f2")) f4:TIMESTAMP(9):6 key0:INT:0 @@ -51,7 +50,7 @@ key1:INT:1 f3:INT:2 f1:STRING:3 f2:DECIMAL(6, 3):4 -f5:BLOB NOT NULL:8 +blob:BLOB NOT NULL:8 f0:BOOLEAN:5 f6:INT:9 @@ -66,7 +65,7 @@ set first row id to 5 commit snapshot-2 NoCompact -write "f4", "key0", "key1", "f2", "f0", "f6", "f5" +write "f4", "key0", "key1", "f2", "f0", "f6", "blob" Add(Timestamp(1732603136054l, 154), 0, 1, "55.002", true, 56, "Apple") Add(Timestamp(1732603136064l, 164), 0, 1, "666.012", false, 66, "Banana") Add(Timestamp(1732603136074l, 174), 0, 1, "-77.022", true, 76, "Cherry") @@ -78,7 +77,7 @@ commit snapshot-3 NoCompact Recall with schema-1, with _ROW_ID and _SEQUENCE_NUMBER, result: -[f4, key0, key1, f3, f1, f2, f5, f0, f6, _ROW_ID, _SEQUENCE_NUMBER] +[f4, key0, key1, f3, f1, f2, blob, f0, f6, _ROW_ID, _SEQUENCE_NUMBER] [TIMESTAMP(9), INT, INT, INT, STRING, DECIMAL(6, 3), BLOB NOT NULL, BOOLEAN, INT, BIGINT, BIGINT NOT NULL] INSERT: 1970-01-05T00:00, 0, 1, 100, 2024-11-26 06:38:56.001000001, 0.020, Lily, true, NULL, 0, 1 INSERT: 1969-11-18T00:00, 0, 1, 110, 2024-11-26 06:38:56.011000011, 11.120, Alice, true, NULL, 1, 1 diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-17cd7b66-4ca1-42b3-a569-d7b0b889d314-0.orc b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2c772a51-21ba-4c17-b464-6f1a314d0950-0.orc similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-17cd7b66-4ca1-42b3-a569-d7b0b889d314-0.orc rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2c772a51-21ba-4c17-b464-6f1a314d0950-0.orc diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-0.orc b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-0.orc similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-0.orc rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-0.orc diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-1.blob b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-1.blob similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-1.blob rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-1.blob diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-2.blob b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-2.blob similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-330a942b-a3dd-408e-9042-293c03c6cba5-2.blob rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-a5e684ea-c245-4719-858d-eb6d7e3a8446-2.blob diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-0.orc b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-0.orc similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-0.orc rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-0.orc diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-1.blob b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-1.blob similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-1.blob rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-1.blob diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-2.blob b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-2.blob similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-2b2e00bc-77fd-428d-87d1-8bea1063991c-2.blob rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac7f689b-b9ec-4e8a-b0ae-4844ca3aa499-2.blob diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-35fe228b-1e40-44dd-9832-ff078fc60150-0.orc b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-d0ddcf17-3ea2-4c90-8995-3e0ee4d3b5a9-0.orc similarity index 100% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-35fe228b-1e40-44dd-9832-ff078fc60150-0.orc rename to test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-d0ddcf17-3ea2-4c90-8995-3e0ee4d3b5a9-0.orc diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-6ee90f75-0567-4692-b375-db3f5abd2b96-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-6ee90f75-0567-4692-b375-db3f5abd2b96-0 deleted file mode 100644 index ede0fd39294a810b891785fac66ea98180c5f96f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2285 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8ri$0UH*eWmP<2K@ z`dwGT^V`K#_}&qEfR(fbq_8*eJKYWLl|?-_hXyFEBVCN2|RE}UsJg`=&p*?hK^ zLPGPQWw{Ecl{v)}ggC+{YQ*+38YOoe=IV5n`Ph*xvrAT3X-UGJns~KmVutq`j?Pak oxy7RVN4fi5>hXffM|s6Jq+OV0|6s24od;~Q8;$OW>Y)cJ017q>A^-pY diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-70d9ec54-00dc-47b6-85d8-944c25110315-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-70d9ec54-00dc-47b6-85d8-944c25110315-0 new file mode 100644 index 0000000000000000000000000000000000000000..d597e233487dcb7df1f97c5aa19bc2c956d55fb8 GIT binary patch literal 2960 zcmbtUc{r478-L!JF=oc#HI_y=P3Tn0GGpvHrjdOcTc;_8LDtY^%~nIpM^T}iN@Wf; z$>;b)luFsMr4&&VC!r8&(`xzN#+-B2NnO`>eSggTzH>kK^85Yn`+1#Q%w+%|U`ZGp z!~(03S6KoY06-$*AQBdbfiYyv0)7AkGGKw_NW2U|42efFa5#vG!Wn))`kU$y43Uwp zUjQfsU`2)Y3?`V(PC2~yqOLsh? zI~sQc5%N3kAN2WZBx1+XfTVzfY+ZFY%%M92v3aAK-N0?MCC!h~#&qmq?b0BK`w5*j z|EE(359|?LdHXI1y!lK^4Co_P2S6YPnFs)e$WOnS8jy`pr0LM;npEWa>r<$D#xz3= z4A9b7>;z&$$o31=;)rTp1rnlK98oQs<)M_ws0eOYz-HtGfPd5yQe+{LqeSWp9Sp=W z0S~bZk-k6@fEb|h(UB<>Ox$z+OgO857-0pz(6Uqj@TVi=)%iSPe_ekn0?!ho&ZGpR zgr7j~43sAxs5-7*3a5oh0q)Q1fih&>BLl4Y*U(938E&x$_eH3-+{@TCo*^~}xV0;A z8|Wk~xmEG`!11xMsDDRZlfU|5Wij5Y@K~O|xO^sC*JQkm^IKmnt;|{D!Zd&V_CwK^ zK1U1Y|gxOIxA5&d3iJRhtjf`RFA>DKP`_YxvJ9}-Vh76t5j}! zEwAUfmv~z(NrC?$h|IIl=(_jFX5Ib~*4)#J&RVhil{Whqa`|g>T+aE!1qER;6c46X z&wa_AOOnGjDGXUgcfG=RZ%d|82H%_Idj?E~#wY)DIi;`KVwFbs92&kY{*rb#VFA;{)lxWSwX`HT_Qcm!=bu%~NTNwkN~U5^#Bel$FY&XsOIM+1DG!u{m@( zS@F9C8ic!o6ud%W>6KO&&qP)b}vb;Om7I6S{lxlVG;5j&&O zpm_8(MzNaud&a?6;G!|s8pSv|+eTfDSyQ#bsCi3`5@W2m@n}s~P*%o1TG$lZ>s{Wi znV4?BNMajye)aOP!;1k{i*HT|Ek=WJYt`G}$iILZ+8bt2T02vF?p@q~-pg(*N?8?~ zHVbohzd)6k24PDQ2}LgJKa8lBiKVRRstIp4>p6A^ItbE0Iyk*OM=*QL)M>@p1b%Qk zeFv-LA^oIjr#Nw(h(cq$Ml_NAJIA8;Lp2`WTI zEfyn)?b@b-VhyveuJu8LHpDfC+tTWE|xGpcSC8yX}VA?sv&M_7^o%e9>zvE?B zdZVaRJ*3^+7t@tWTqR}zonwdJ$sy%VK?Z1mOO_btjMmju7Y_}!bW zwKEN|a@{dgi(Vq7F`T3vPwGBvyoHvk6(yPl$+mO(Cqi@2t zH<$CU!c)8Dz4l{=mtLyRnfx=dX>{hmkgk$k($WoJmsz>l+|h}uU&_r|1N6!9UV&K+!n7tWA4<$Una+?XufQIsju$=fC<*}Vhj%2<1U=*|-PjggPJhl;D7 zJhNFV?qLcn$!;I=x_l>0gX8?9_wT&hPEOc(jODbbx)_@T{_L8abVxXNcu!N;)bg4B z{O5}gk6;MPGaqLBW_+UgbbU?czIvNxm7L(28i5UFy{^sHbPO*vSc!h&{F3-&+jiLi z%TbfC)q%M$x2G$$oC+?T^s%xCnr%M5_kQvq<-o5@^jyYS7DZ*;^H%A!iejz?ZWIfP Zk$F6vDhd!+)T4CpDR0ndCl_e zK~zvMploWDMSoPPLMxWC6$D%mT)vftF0SJb_F8G-Mv z;vu7shq_fxDCgdfc{Sc;Q{frNExoLMq#;#)YuaJd)??qF)IU=g{j&R*-b2HHtzoBQ zf##r29YoqB(&m5DorBw?60pT%INW7a*-bs7s2eBXAf9aW3LfA>tN? z5K!e|j2(s6SU!01r=2Pp6`}|03nM93o*)huO)=6>L`%R!{fpxgHWQ@Waw2{yf=Y%V z9_k}${*6zp`QHIz)SpU^qv5`&h8a>>QA}MHCNZRl7{8D(KPo0T-~gMpkq3}-h{A{r z@L4)V;T9-x13?NQzK{Q6@OTPPTGl)UMR|@mNr>X;i@_p;y0VnvN&^I}lc+Lfm7&>> za|%M=UiE3(cA|(rzdc$TqisjDB1Tj{3(PL@eEO>bL1$b3c-YX^OO~E497@N2`-%p$ z`+bzVXqT3Q>C!0m-tRwzOG4l}kQ1ckDM$j~gx1$*vE%?4MmI~0p&(5lizs1>uJ&gv zm1l7Q(sl%W5m-b>`ado#k}X^+l%fQIdl1qWEK-ckVu?C*2!$BY8`8QP?47v)#GiP! zupn9z{a~OXbf3rHh(iLJzb>Y7A1(Y0a= zIB3WgWrLhb(ZB_=0@;BEfrf!bfyT(_u|(sS1}$FOJ_XPO0nFT4rKWNuL<&A`Wt1G3 zlf!`^yR7JydMX&xq@Ek=Id!TuTWUQz^LBie(1ZjvCbx48S0<&d?cm<4ayfO;bZGKK ztwQyEp~3rlz1gm*5r*=4zsQ18j~Kys>XqtGwVz@2sq6yvl49fbw|HoC=Sr2`I26h_hRs4m>ep9cS_Bq z#k_DbopeRLsgiyp?0qHCJ|%eNzUWT#Bq#NdDlMhik>k(1GlsR#?-PbeZulvwumhF3 zY5BlJr0unln^NWvT08?ecQw_?^yvgbv{&fPh&ROzELr|=Jb`|<*%SM0sga4OPzm7X z!d>gbEs~O5Ur3VQSdZ&zb7JYRO8sGX7B|)25voxlNW;}PL-Z{aC*H>=4O~s>)tyV< zePUZwF3K2~00UrjMlatxzx}Iex3dxY=|NPFQdFA-y3#;C{`?RR) zj6FHX(xDiX+uPK$VQ0};3}EU22^dIXz&Ha?uz={gS|#&!Z%Rev!7AtF@S7^N{cNE| z-8}ZH9!0Y(U{uww1S@nl*62_E4ddTc3JQ>`>rB6PNM=l3<>}ayLJQNNqjQhsG}1ZW zwlk-<49hv}OSQYO-2?^w;9CabQ6{Z8pF?eWTP9YfCw-f5y?5Qs;Qn{N@HcpEoHU-^ zlOK7jwMR8^eNs;Ht$ZuHbrpZ?UiUcnY{`VF$2*O418AE$zHWuuUgKXbjL+Xvz4z*I zZd!(e@z9ES!+-=0Ic|?Ix$I>E@j5QDZY$H^=8o~SY=;??>M56Bv)-KlU|@I2yP~2* zi!fcNaYj(Qr!8w&-Xe zOVoIs`OpE2~ zU$ssAUvIu(#tFJ=WHPiunlP=H_;pTVP0;EmdX}bw-UgEAPCB<%(Tej-w}F0V%nO)& zcQF??qncPTW=)Z`FTTH*=(sY9YQ|XejdIr0xE1i0AS-r=nMN(9obMOju-p`qCIy%Tkn{`X$m7@=I zK0Lh3*ypxveOuzW)c?lJPEF*c&&UfN^zLBBT%8-dYShzSbOnMDSNx6(3&uJzKKe~BdhrWf zkEsc4qizRnYNANK=(J_OK&TXY-jNwXD=O4V28EMgSumDfN(&FM^1_VgVIfz9_ Mc!fqgZ(gtPC&IVszyJUM literal 0 HcmV?d00001 diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-fbe3fd00-750f-446d-81c3-bff220caed22-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-fbe3fd00-750f-446d-81c3-bff220caed22-0 new file mode 100644 index 0000000000000000000000000000000000000000..57e6edf4ba088108838c9b0638653fd5fb6bb353 GIT binary patch literal 3104 zcmb_ecT`hp7XRK$;SmEo0>Yz86A%Q0A`olnZGnWzGKd06C@LZ{Dn&(%R8bf@N)Z@r zfb60OQe=iUidX>|x(chC%=U(5MK3Mn;Gz5u<*_4p4vr7FdD%(ML1_w4nd(N`Pt1NZQ;e3%7bkgX?>Ylm0ifR`>1l*9=l zND%snianD23}zxq!DLxL2ak~cKoP+y!ie3MtRM0CJ7Gq63AqJOQoiC}fe^08I`dIG|X2>0n*wcC{Cx5BgWMw=j9}g*B(2NDwH-tuTQ8d={XSP@p1TGYp4$2m_Qw zQNjbtUkpuN3xi6+-&`*v(Q+-U7z{2%-Ey?cgq8r{46mZoafy1tWG|2gg7A*pz~`A7 zi?y_@0p4C5qhup21VQ~jUWk|lEf(4q0)bw<02$=qrx{+6*f=>4N=sIz*I3Hh?>0dheVqM6HrMEJ>0)EW(c;^9vrrH!iLwCM5hm zFN|=8DHK$co`>T4@i)fxKo@Glf3-i|-$bYp;UgpLxfT<^Hjq^uEc!v^Cc$^b0eo+z zrdmRv*`^rgeHQ#%7YaCkc;&j}R4Ik73`@gX@ zyL&W2RpNnq^T`WZoezJ1F6Qm~Xm02Pw7r;@XWrd;NnuDsbC@ix#y;QzZXBFR zs%0Op${^A+qAG%K4Sc`nnnGE^RQbM$fgXjxw&|{`&w};q+ME>Tn@#)b9wtut2h?~E zw9gISd&6X=cXL$yw(=^c(!^WW_h*;n7)2Ktbq?8$?U{(&Ij%IkCh(=5op!!6yzj<{ zUfp6BW1-U&aK2-cG3%U(3caI(q49vv@w-%7nzY>LM#Jn4*|zl1ZEkK!wHHPUSIosu z2r}cYUc2vMvuSO=?`=9*0}OzY$;AXuj4qK=b~ZM{&Ye2qDRVjc9xJP46$bg>VvjLW6xxI-swCjXLg0Uf#`%}NuZ=k zRe!|L01=i_`b3i$M{>Ys6!lt0QwlFz$>}FY`X^$|Ud3G@AHUt3Okw1UlQzSKBN zB*_}R`zm%oG3bhEOnHk}_8nd+T1JPqkt{)gw%Y#myGC!w3o0!mXDR`0`UOjrU55V+ z(L~QOr7=yGnJZL?FnQlG4c>BF#X4Uum$y+INjUEC57Hae9rK#1+j{C}SFSj)QbwQ1 zu=O$D)a$a-ihL5Jf>QvKw|_rHn&`Nl+O_P=xe~4Cat{8jmu}GWbF{KlT9(K@l^}{y zWg7`*kbl&C^3u$Vnr*2cCt zIg~EjLF-?e7$B9pqD!Y{Wzy)eoF^|Z)yy-tCx`eg%A)R5$b~z5b2r8eK90zLZ5o!2m+k1NY}XGrF1o;Q z7*(txEDh7S-r1$w(U|@pV$=r3O=>0DOH=*B1v_uf zSZgU)u~o}+Ev8I-8(eGa{wZF*ez@%JyRrVB=DLZkKTI%HoyDT4%z`%W3bDCO`9$R| z@4e6F(-_^}uWa8s&P3EpWr*t}UwI~7XgHHSctCAbrZKv9@E##@SxS71d0R~P@Z9b$ ziC@Ws;b}70ciC>yupoGawI0y4Sm*vf<0p@Oc^<#t-%x%Cs@~oEty$H@YSy*LrKQQ9 z;SV{cK0XAhoRMq&kO^g`_R@=-G4He!7q33w+Y{N*@;e1=lg%Ppw_>*MGRA%3Z`r;p0YYB z&9vU3ouZq@X@Lrhj7ne8ZW4^5xTu6UUAtK~CF96bEP~#fo7ri1ZbR}>F0nH&e!P8) z{l0V_x;2hrr#I48UvC9D3*t}{r0DI7fKxwayw@*}a^$_+5!-Q}AdpDb4thyZPfGLq zx!fRTa+lzJ)=zeJgtx9bD5Jfv@>+OpL6-liQmNI$D-w@uHPQ8$jjkiy{&P?0(wx#D z>3tifNs-L-zF8&HLo$Vhlc1_nIDco;EsCtv9I;%tDp#McNt`QIL5u ZO2PyJL<0s?-65DR9yLuMxVTy>`~jBat6l&A literal 0 HcmV?d00001 diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-3d1147dc-1860-48ab-a450-fce85ac5e9da-0 new file mode 100644 index 0000000000000000000000000000000000000000..7964d271bf293e33ccdf1f5ccda69d756ee1a8b8 GIT binary patch literal 1543 zcmeYdau#G@;9?VE;b074&;~MvxtJLk7=(B@7=_q4SOi!kSOOSUFfeFr)&Hwd8q35W z!opR|#mXR*#lfhQ!lj>^n3tKBT3n)=Y?x}6YGz=lo0e>xq-&CxWTM7r?;8AQ8%_yMWaYXq_0uI)0#aoE%?_1ilzad@;h}5)J_l2@Wjc z3<3-i3;_&Co=^vxDhPBT2nca-Fgh`DSOhR&7e#eJ07DWdM+1jI1BXNd2TE8l0mA}n z6N)$}j0T%HG^E%W7~B{f;u#o7j3J;tKL&kH28K8W$r7+34M5U?fx!|SqD){CiC~1X z!TMx~QUp}$!~iyxkpW^hLDG$(L5&fd2onkzd|4QhSOrBILM@Fqm@)GC&4|BxwrFC& z2DK#S*Afepe;oB{l37{9v);9Fag&4mv#F{-emveB;WcwXrtc*Vle*8M1SOy+5U(@TI2X$38LoO@BqfTXw1xsHpUcGA7s$YBsJ@SdI9TyID z-Om%4w7d7qw!$9Qij?$eDmQoEn#S1ouIB$;i$mLgUEBS2vDL20wu%?#p6_M0&VPUV z`+e3D^lOoTn-Jr`k|v_30GlH#-92I_R9}P-aM!Oxz{|i z$$9lDz25|EBj&b#eKHD7Hq&e$%6e zws0#QR-2-EK&^n3tKBT3n)QZjfS`nrv#KYhaL)tZQPPWTtCjnqr}AX=0LW zWNK(=U~FirYak>M7r?;8AQ8%_yMWaYXq_0uI)0#aoE!`E1QzH?EYQQ^5+(sA2_`J! z3<3-i3;_&Co=^vxDhPBT2nca-Fgh`DSOhR&7e#eJ07DWdM+1jI1BXNd2TE8l0mA}n z6N)$}j0m_4Lp?5YNEPQ2;F93N5+XtZLJ~p&4D1XHZVV3b3=Aa35Kx~VgFYt%LmY#+ z0+8dx&;TSI7#J+UA<6_MkqAa88>~-;C`AZU89~m3+W@f#jdWvJ02J|K5K?0VC&GjR z245D2BvwI@hEPl64Q7mdely~)o-LXfut6<}`L)Et#_@08oxD)-N%7~W4)JN?-=b(_fDv}bvxIhKLP%-8gKw?SQ%&2W~L z!Kh0`eZtn<|NsB<-~Fmrsd(atVbr0-_P!|9h137O-^#`JDl)Q0SoP+8Bi6G;wg2yY zE`PpSIR9mIk*?_{lhdJFoi@Z|u5ZbDo__rG^k}Bjr8Dbh-Z3)$6r$~?8-8>@hk9`b zk1_9rDSS)HgE}gtI2TGQPZ92#BQ32VAYdrP(CvQky~EbSv!9-tXL;@IO20X)vz}!i zVCG{u5YTWugrkkmp^a6`pn=1^TK4)(&Ihl~a2Aw`Ozzc?TB-4g84|V+nQHkN)TEQF z6f##xM+iJphzr+Ln1T>GokZcYw^p}kS3 zYRE=u-P^jU!3-cMIAzoAm|629zg-lXwXpK%i8^1sB7x^ZPZWN9FUd){XKhI8He)ncWLl40a*&QFZHmzfR)xvPqNi=rHMe9#458dQ%{5u|{ zeO54eVM*YTp1pxZJwH-c#JlYH$XcRuc**gq>+Rbs));ahnsYAnVZ$Qjj}4o6bVnY?hXeXz8J14abr9DFSJ71_q9&n=E7+%#uwM{rssR#HWQXj z_g}*ype9kGrJ}(R(YNmYf!TguUQ?Jd>m;2?*p0W|JAQMuRn{d!}XS@~bM8aS<2BTJ<4qQqK1-y5BCCRZ=GyjhiFPNB7( z(6M)^@A;n_?EG@#!V{kxFCTCm|8R#TWxdTCb%XS^u?=#2R-HC`(*0DONrdAMvjD3^ YLxX@469bRNF-8NpF literal 0 HcmV?d00001 diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-0 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-7cf0d548-c508-4619-823c-10b0a842e6c2-0 new file mode 100644 index 0000000000000000000000000000000000000000..04ebc0214c1ecf684c2afc0b57092671682865d3 GIT binary patch literal 392 zcmV;30eAjSQbW=L04TLD{a_IS{R#l~5kfFfskQ+yGcz;MRaKP^!6rJ9uHW<5B+M5# zqG45L;P4ow(AsQQCJ^=UzI-0vO+*GW05kwJ0B>!J?Ix$*?6%eCzWmm-3s+mN=C}AF ze4M$vx??@>6Tpf<9+hLEpe8AL$jYM$I$4tk5Dp=Rh(?TPlS}BQDs-JVvGrISKb3pO z+%{qJX+B@>9do;1Im_ATk5liu_2=&HyF1kj0vhF$W-*e{11NU6OIo0J{<_U${N=B- z{c!a(oH5DD4JakUD3|n#AR5CZ#Rw$X5Sly$*(5W;6rkt>Ia%TW1U={rNkgG8H0Z$r zNkVrGdeA8iUf6|EE}4BEn$asZ?>eDGfv0XgrR%c{h>KjE^n3tKBT3n)=Y?x}6YGz=lo0e>xq-&CxWTM7r?;8AQ8%_yMWaYXq_0uI)0#aoE%?_1ilzad@;h}5)J_l2@Wjc z3<3-i3;_&Co=^vxDhPBT2nca-Fgh`DSOhR&7e#eJ07DWdM+1jI1BXNd2TE8l0mA}n z6N)$}j0T%HG^E%W7~B{f;u#o7j3J;tKL&kH28K8W$r7+34M5U?fx!|SqD){CiC~1X z!TMx~QUp}$!~iyxkpW^hLDG$(L5&fd2onkzd|4QhSOrBILM@Fqm@)GC&4|BxwrFC& z2DK#S*Afepe;oB{l37{9v);9Fag&4mv#F{-emveB;WcwXrtc*Vle*8M1SOy+5U(@TI2X$38LoO@BqfTXw1xsHpUcGA7s$YBsJ@SdI9TyID z-Om%4w7d7qw!$9Qij?$eDmQoEn#S1ouIB$;i$mLgUEBS2vDL20wu%?#p6_M0&VPUV z`+e3D^lOoTn-Jr`k|v_30GlH#-92I_R9}P-aM!Oxz{|i z$$9lDz25|EBj&b#eKHD7Hq&e$%6e zws0#QR-2-EK&KKXiUPh9 zJ*9DI0Du%8MoDdiF$jhTQ-BbFtxbB%s&RH`kYX>dT&JUi@1jg)y~Sh^=TsiD~eh2UQmyZVQ=OfFZ5DHZhH=z5uOFJDb3o*R=HD(I(=oN0IJjd{HH zqSx%^QmvZRTuZ;xwIVhY>Vb*XL;Bg zg;s|lP>{YIjRoKkq^BnifFGnlmsedSOA+0c5U^jPG#dpVQ`7+pK&xl95P(L2l6=bp z)anLB#cvuG?YD;5qGqAb?yRBDv?9WFZ+i&0N&!n5LaRTU76lj>kisR%v72_S1iQ(S z=MP_r2$?&0L6h~cs0b4)F5JARo@M2UQ##CV=SY^&VhN4Gy^>p|UX+QI>MBM=@SZDZ?1RIDVq`$n$hS2Y@c z)L^xFKL1OgHll9YbeA{2;z5ohxfot39-%@f%a})~tgZO8fp`gH@t22DqdS2Qb@E5jHujplTNEB(8hLNp)_gw#AKWZ^3H~kk&?zHsKK|9DC^Q8bGsssUi->|(D zaxGfl$f(hiXp}l`rOCWLbBEXy_(H!ywsWFG?v|jG7!W|nZWEiAGG;-&Chc^aG+QZ7 zQtx1>ZDM*Y* za%B_Ld34$zrSa@Z+(?2jBmQrX3#~}<9>tt=88(~lv&+c} z$w{~R;Jk1ghkVqlAia(O`23ae&ThhV@Q&q=X0)Wt;&-tnGg}BhBtqHI5i|8wW$;)+ z_)DL+SOXRffu39{oJYy;O|^@|?%d+d_^r#rDels%tM<8q&X5;0uaMr>1=_93LdjEM zFDn?$S+RpAcCSl(wKF0nq6715FUpdNYWNEG`gw7)j&b(|?||BqELEC?`!bwsHT-7V zj3bZpUPDoQ&dF!gJ90*$c7Qrm-s4`~Go<$5AKw;r%T& z>?^{3TxYG+N`D4fJH+g5Q(1eJOxp-GY0h@|zI5{K?NeE4rBMv*ugprdoyo%!Q--`p z%FWd!LDd+N>iKeiS4pDuhTuKKcEM3{P=>yZB5%I%{XhDAkL14R(nH?1j>m4A>-tOY z(p;=N#B-pSJ)K=T;CZ`XRHZLDW`^b8OCBHUAoQG{O5^zjjhAu;FULXq5`MZK;UIs{ fnYI;XV~`C*CKFLd0sPiykUlnL0gZO1+9>=BP&r4# literal 0 HcmV?d00001 diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-1 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8046c4af-62fd-466e-8199-29d90455f97d-1 new file mode 100644 index 0000000000000000000000000000000000000000..0091f8e77dae1c88b14cee6e3a5606f6bc17b0cb GIT binary patch literal 1569 zcmeYdau#G@;9?VE;b074&;~MvxtJLk7=(B@7=_q4SOi!kSOOSUFfeFr)&Hwd8q35W z!opR|#mXR*#lfhQ!lj>^n3tKBT3n)=mXvComSSL_Yi??grfXtimZEE6m~5<@l$K^> zWRRSgnqp+6Yak>M7r?;8AQ8%_yMWaYXq_0uI)0#aoE!%X1P&NT95BG*5;g%g2{tU^ z3<3-i3;_&Co=^vxDhPBT2nca-Fgh`DSOhR&7e#eJ07DWdM+1jI1BXNd2TE8l0mA}n z6N)$}jF<$NB$#l!3`0FGb4V5E65x{H!eXzGfRKbx00TP%gBycGJOcxXF$C1-$Dq&2 zz!1kE;Q%%y0Z2M9Fj#^^lnG2C5sXkaSf310ihxR;7#cvPGJ>25Wix=-4j7~x!vdhV zAA^t@BRCNz6fpR*FeI@GiZp~;8gDRTeVE(vW929 zYvbZ32l;1HRe$_=yg9;ig7Vah&pKEce;EAumppB%-i%aMRRsg)opL`CI60=(q-{Pk z{ie<*)8pGV?wC0n)>gTHCdKf6n%e2lzNynf+{vYoU>~pUTO1 z(=>Ioa_;TgSiHIDs@L0t+&`W6UXbGL=DMuZzM>#fbi%Z&Tc=uPhjnn-f9n6W*fc7t zW%`PM1Gjg7e{$I3|GhcA-gAyTdg2s#r-rrR_jKdRSuX|n=2<2j5K)Mbx+G-K%{6^} z-07%W&7#X<*9ckgF&wzyAmXez!;)cLkRgkU#jV#n&iOEer~5UmUOXem<)Vix<1c1N z*gj;M%g<02oor>`d6ZXxEw-V+z~I789i0n?x390QFJ)_wOIWx}LF&gFZI^39C;?2F=)XC4Bn63ed*{YVF@md1{MKUiG~INB_;+Qjbn@koK0Vtnf-&DB?0;N*I57n literal 0 HcmV?d00001 diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-2 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-2 deleted file mode 100644 index 120279720cb30f402ad7552fb288fccc89f678e3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1034 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxbu6*jf>A+E30;Nlg~2DlSRPOGzwBVKBGgm{71@Dst1_&9379N2jy2 zF==eo|I6TUm6>5tyQKU9RayHF|NqM~FeLQe4H7!{pLx^vJJpkq-mJMQG&zcE-}~SD z6AH^#%wi7CwMutexGChy#>LNaUN*haYk&XyU$69o+hKe>BIz?!7Oh;68pNwQxlN@) pN72eTGj>z=G>11@G9t_eIoKF1Pbe9P9enB#%Wbenwt)%VRREJ~UGD$@ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-3 b/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-3 deleted file mode 100644 index 6e740fbd91b7c0933e8a45f7d5f2e56d9cbc4f45..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 996 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxbu6*jf>A+E30;Nlg~2DlSRPOGzwBVKC1A%aU`vGBm_VMe_NBJT|6X zj2c_@|1x+iWn#E=L{R2{s_g&&|LqMJ5@v2)dZ+tJ*VDM_$#*WNr+jw_+V}qV_V?4H zKjbcFPikmi#eVse>4cq=HS<@xw11hAeX1+AO`RjhnL*3ai_rrc8|?0((92R%6zg`)TE&}3 z5fP6bya*mV=&j!B#gm9f5B&h*(SxT7YJ8J*o82`P#5IIT-g}dm-~8vj2^TKckfVu* z7Ewc(0`OnWeJ}ANkX{Hlg~w*l!Xg@Icn(+Y-YCEyu*jW$IdKs{Bg10M0=dmR)Z74! zqU-Q$(VZ&V!V-^|*kW>Cnt#gY&JY)%1~e4u0k97zQ=#8DmDk~FRGgVbjvUShe3-K~ z!1+*@I~tK@IYu((lFnneNV4rIdiX1VlB*GKMk-THt*BjBuT`Cf<)}-G5a?%Op8{WN z*Bw@+aLPQn+}Bpu@0|y^Yj|k?Qde6$kupDL;ysg(4f&$rWeIW-eLOpaIQ_JT_#sJ> zZ327tIZ2+pOvN4Ia=7w3T*Z|XnYAD2yNOI*ZS7|jyF%MMc*7i>A|V~&KLNIL%T)jX literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-1.blob b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-1.blob similarity index 100% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-1.blob rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-1.blob diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-2.blob b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-2.blob similarity index 100% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-cea41455-da2f-4e8b-a026-f001aace3d58-2.blob rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-50989127-b7b2-4950-aaca-f77216b0a46b-2.blob diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-5bb7b1e9-7186-463f-82f5-8d1b004386f2-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-5bb7b1e9-7186-463f-82f5-8d1b004386f2-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..307553da4a2a15a2e8cc93d536b3a4da20add153 GIT binary patch literal 1160 zcmb7EO=uHQ5T5tj*j9c>L6}%RC{QQ7Quj0cDd> z;+2J^R2HG7tdJXKRN)z?rKk#g>J_j-*<$^8m9W6H*pBq)c0sZ@nlcMQqa^0 z;jFl&_eTot6Xp8o_}I0n>UDRtj0RQXa#X$P-gSziB}jNt28>W5*PJ+XtCl4IG_Sym zn%^kbQgLyv4AiZXh}6Rj(=tmoSj5(?=_+X#*cf338)1u*-EZrb?~H$Cknnu+GceB* zkcz}EPNfWag8UiAPWDE_Bpk-&2e>2sr#vD@zCRDFCb#*;jobYh9hOnqcHmT^+mbPso`>0D|!bs_7dUEafUIbN>1m6}_v Y=6J=eW%+5dM5}N8LV?cY0R0F32BMR+$N&HU literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-aaf02df8-9f32-4c97-9b43-de9ef946a1ed-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-aaf02df8-9f32-4c97-9b43-de9ef946a1ed-0.parquet deleted file mode 100644 index 9fdb6073505ec0efaccc0bba2e9094a45a87de27..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1295 zcmb7^PiPZC6vp5FX_};|Rq7kEu;3o-nu4v_WH(Jq4)r2cM2Z^mP>>{UY_Vx;nu_)$ zh*0UJh*VJzr3X=w5?o+5~Xh$!@+QhV}H=%o~UX!T7tG0~KQ%g*l1zM0?m_M5lc zd-$Y*GG4(Pewur_;f|pfWq`noLz~c(6TlSbH?|sHBf$5`7U-V3<9n+I8&AJ_^X*h( zSNX97GrsPLX4@kGARxnb%-@(?Yf?E3UFa?Y&jN@5AnlW+i6g*g%jiZ8S6AOZmSZSZ zq&1pGhXdS=0sOkS@Yz!X>@EXAs?dHe+8S$LYb zGt{0?drobEnv?aifVTu(|8;k@zGnG~0Q)7!ie_363Wm%!!)OadG{fo&MZ3a=X?1o5 zjRp!HJ&@5xbJ@hu1zUn^Z05__{S&01>=6||Z58#L8;S}8g?SYnyrn3AX^QgOHc_5B zMR~@x(dKP(Qv3~7$^5-WWwQeIF^*JOLBunq?vnJUJ1^sS>1nw-Qj=x#MaIUJ-al#yV(97Tw=mOKPD zd|fO8f$B=}Z&yrJd@6!+-2(gJL5)3fV?VIGi+Oc7TXF5BAIPzhME25{oomZvwaa#P zbYOT$GujQSJ){MaV*`WdwW!r;CsIblN(Q6p&QM1>!M~zlDjZIxthCt?F@sh*8Hu(k ReN-RSCtRm*v6p^@zX3&z*Sr7# diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..35224b8efd5b804191d1491a20a61ac086f8051d GIT binary patch literal 1297 zcmb7^O=uHQ5P)ajZsRtl*1vgVWuc+aH3f~kX-q2QP!53%#Ld&285q9Fw*1@+3ur^i)Qx6{cLMP3txmWL++f#>% zY8fVdO{2%`N(le~Ifn4r-LZ{oUBJ+TY7uw^fB}G9Bg>-~Kv*lNVgT3IKR!|7$V#CD zWJW(1co>8Db$R}CIhg+TcS~?+~lV4IFPXAA}z*vQ;9l$i}!#WYIGBEy$(azT}ZgB2OPECsQx!s2pyZ4;!;eJW3qV zalazkPyM5M>AakJ#|2=y2mn#E127#xoVOb@6>%C4zXX(yv+e}U2*5oRYfH5`farDx zBc8ZBNq%RnL{&+rMlobNVQ&QNu}bCexeD$zRKDRkP8Z0A$#z2cYolf-9OOP z+Y_)tRy1S=g6W~ITxTF+hBFO~mYHdYg*(INqGp3_HCnNlmCD%ZtewfkthAktTJ>5^ P2iWz83A&lu=ra5V-Kg7) literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-1.blob b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-1.blob similarity index 100% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-1.blob rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-1.blob diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-2.blob b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-2.blob similarity index 100% rename from test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-2.blob rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-ac3de758-1ff4-45c3-94f6-27a4e7c47172-2.blob diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-c8aab979-5e51-4112-92f2-ff3036b332c6-0.parquet deleted file mode 100644 index e855b6c3ff4d61cf3fd9cfa21bad66b16ee69df2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1374 zcmb7^%S%*o6vxkfI9+d}_Buc2#s!DK+Xyw?8E3{3?!sJzgbN8LS_sXf8Is0FKBASS_ji7e^ZA~0euvwA z_Cg3{yosau0x!<|h0aj{BI5Bs+Xvquz{XS)%-(+^s<90FG2q3@yRdk}RM*}B+uFi@s=Xlw<&N<0t-o1ahEV)irWXJ_N=;@;8NwHW= z(n!wQVNvtZeUar1C~6EerfLSLrCpvm?ebxdT^=v(@)X>=0yzqTdtK%skXO>h)Y*)H zy^KrhY+1yMC38)xmKc$ zBtfP~rhF}X%S`ZuN|R7RbS7dh+@#V?c~Q}O`N%^A+uKa=9Bng}I0958UCSGwqeqJ> zvPa_r=w&Kf7g#|sW&}ggW0c4#>QWhr&)Zc5tGQHOVAHk8fD^vCrMv7Zy0ZF$gg352QwYm)I$A_0E6&YKHVXZs1PaDl}GI|RV4LD#u!--YmnebRn&-v^IO%Mjs))a?bH(MvVlMEc{ zL5AlYchG~Nf_NEY;P5b1c2M-7|G)%y5%J>c`zC2SR}uVDzRCOX{d_*}eP1(v?!1Ev zs+hvv-FJ@_4`oyU(l5^Zf}s=yO9DW^W0=M-pVmJdGEw6b2~-Li+(Mn%;E^X zeYv#U(T20#C_v!6uh;8qd}8%my}t6Kp?3-=@Z;^huZgz9-vOS6<;TH*Q_utbj3SoO z2{FCc%UH&7+0X?zL=Th=0)sTe0xgOP!5kHj_NZhVx?CA+4=6$=9}`O3qiS4m@-lR_ zq3X{plSc6Zi42o=q~_~}jLQiAmC^Oe>N#YN=+}(Qqe72mK8#Fi!IrD#)y29%b;lqh15;bzmpGiK7Px;hh%z zj%2M$OdxI}+yz0)`WL}N6^ZU9B7!9A2iDJ(2kU9qGSV<0xekEF)zG#Js1KZP)tE+?L%#q{#q#e(NLx|mAL`teTfz& zYfJH3v!XW#h|!x9)bjq7<+F-9J@3E~pf!awOQp?tCx0u3=knES^ZxaonQH2WU#(5g zR#Hx{md1`L?)_;X%w8Hg(3*5f*Fg`P2N@z`3@SSmq^%9seYy|PokSES zyLlS2!wz0{&`r>rj9!F+Fuf>4n4-st42H-K3ci1m?h^$=$jkr#Uf%EH|JweZ0UKph zF@f9L>q~MTMU(+jFVFvm-URrU0DyoeF@>8OtM8)*YV?JIYNM;^c1H*n|e@@5MMhJ(PpLI^BTU)!?$U15wPT|tZx8>$M z7ts(fC4eLLT=<~AMu0DKNqDm{8ZC@9X`dsDwT0Y?#D3* z9dcx_;fN&7UJMmGcp1OXzWR7v#=Ax}cX5zdJtJ$KdjOB0`gZB3L_JR10AWB;XP`4p zH$j6S%blex5680-d85ZA+p&)4WDZJSa;(EygZ1GR#=^A&Znw`PKaJW^sB&r z3g(V%Zb(L!PVDcJe-WcSwE>xoPo~Uf%=HMT;!jB2AD07U_Fa1vB~iOv6kCWe%~V-k zX{BG-rSa0l&FSjYxsi#)t?I;NZEQ4Qr)@W#Nu!B0M_-nqW}N^ diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-fcdc65ab-9916-4a9e-a13a-5e5298933a18-0.parquet b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/key0=0/key1=1/bucket-0/data-fcdc65ab-9916-4a9e-a13a-5e5298933a18-0.parquet new file mode 100644 index 0000000000000000000000000000000000000000..04132a79e2ec19b9e01148979c75209c71801678 GIT binary patch literal 1232 zcmb7^O=uHQ5P)ZYnkH##l{$|sEVu`|MzC>Te%q2my+{?2qDDjtnxq>oHf>E)(Vjdg zRC*DSD%C^jO_35od$Xqqq98pe^q^9EF7{H2Jy@NWO-!)481i;!W@o;cH}iJ;kDcMD z;!P~zr-c{W?gV;K1qi%8`V0CB5=7+1?VXml2=INn1A6a2^1U~$)^l&(eLI`lQ#C6v z=j)wnw>=U70x}F@@$U3yn=WAJLU$E-0YC--<$$709S7m9q8km|*!cKNO`u#$ZIB!N z9N=CI;MbL<&z@bt-YO838tv!Oy_aqO(&+L3(f}iP3O^Kot*9a~)bC`uns|n|bF`k* zdPQrAmQ(ebgbyU#`u%vLc~|(l1P2wT73~xdvO4y7S$5KPN(UD8znX7g# zg?m?JoKyjQWq;lQQJ1b$9n7og+JFf65Eo@dC7F^F{d zqMC4K>sMz)Z-^--?PxSI+B-F~y!TlFTdu-GT z7&Ne=2MxB#qIu1vojD%3q(b3!S;Val_7!4~>o(Jji1~%Lt?=hK2`? zh!x6&ySSAJ$3ugm?wA#}c^8kzc`9S4vvwvE=V?0|R0~N zEbv&J+^29~f6DG{<jd!O~zDxAQoTsWci^CYKM zdpkQVHGUzHA2Tj3KJ@>tbE0w}vvT7Dg@i-X4ld@&Sej+k9l1H)jG?KrA-+>0J}xfq z$^n-rG7?I=0}iNM&YgAaR@I%!w_hyFeyy@(t?D7WfWsFnc3!lax5?<=8 zf7;po$9!Is@rm!;sqIgMB%fz}eDC&q_M7Y{871bw-+YetoW!)=_Vhf}&IxUkTs#)P zb?p3f{`}k&A2owdpXc~b`R~rgE|IAu!^^_SDBhw?XWz?-1Nfx`~8=PzptPh#9 z{PN2dHl0}-jDn5)tX?yBJXpN9SKnxtN$Ya<{Dqp)-&RJqt!xcQWS$uscJ6*H(_~u? JRRMIz0|1T?X%zqf diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-26f82168-b0c0-43fc-86af-1a15e18be25b-0 similarity index 84% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-1 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-26f82168-b0c0-43fc-86af-1a15e18be25b-0 index 2407d6d0344796580a4a5f04f95345608480d56b..b7c6ea666796f171ddf5f73a8e7f748483c98862 100644 GIT binary patch delta 364 zcmV-y0h9i?5$h4Ki3fl6H}V0|?(%|1o+Kh&AiW5H1t_&I{QyY42>`YbIW%y^Nn`x$ z^k2tnd#C3SFdUpQaOW`u&L+e|8;EgfqqNFZp&?`wB5hyg>6=@~611HqNdYbZBmgV` z%t1h5sT>#@IHd^!Ik!(TwP?brO^5_FX?n3y)!VomHEDm5lIiTM|MUu%YVtUA z{q+@rKmbgI0R2^89SYzAa{m(l&b^(V2mXh3Hk%!rO@R!E|92-8U{->pTj{|VXOjm~ zHWyJ)gB!O+_^63;!@1sEZmxH!CXWkmg|d%oBcqwkX0sK6TcPZuy1rTr(AnB>CbmM^ z$BRUBL32Senuuf(VW4m*AYg-nrfqQGV-g}Bal|97a1|M)&{!a2qJ_T z5iFZ=?S#g0EjE5-F2M&6;KxIxl-^v%BQx-#g_FJkAH-mT8LSP=a0j{7^*8bX(eCns KN1h}iT_C+?5v6kg delta 313 zcmaDYxI=J5Cwsl{?Mc>Fysr}zj+}0J)Ev$>i$!Cr{$B!-MKBfPPhK;^L4W-RrU1#Hp!aRUM-k)fylk@wULZ9 zKFdzMSbenh)$e`pH;JgUI+)E%NjEZ=obx7h8RuaTHjG@{^RNDwmHG+}EywujPq$v| zXyMFgsLBZ8v%f zTFo+Ho;~Og&0$*76;{PDLkJy7^wn9 zIvkHE9Amf}y*99DhNq#0SgZD?0-h`3LRGAq4XOzV8@NxfOYrcpu}K>Ur0ighxr!D- E09kK^EC2ui diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-3d16fcf1-d81b-4d69-b118-f581c55e2278-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-3d16fcf1-d81b-4d69-b118-f581c55e2278-0 deleted file mode 100644 index f65b6ddd1729928893ca4d1535d90bee850f7ff6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2278 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8f|jg5pA(TPDj9Hh zQM3B}AdYV=8e8@MGI&hoWLS2=+@QfSljn|rq|#>oGzCVdp7Zy9XawXq+UT@TQ@OP5 zK(d4T62V)!_ZKgSV%CYh9WhBnk3pG1hvAIigMyU;JbWn!51VrNEfMjYI5%m@TamNh z;(Bs#o!ENfpv20|*?T;)=Fa^(J&tik*C&gTzh6sr4YZ6TzJ%A7zBuAs@}K+vxxI7e zGyZESHJdFv`)q5=q5r$tl~g*KLWDKv7vENTkup0-^tO|yDMy$7sVt*AU(0sJ@&^aX ziTit}+O6XCO*5N)b{dNxi1S*z_1v|#%Lckre^v-(gx_Jkn4%;g$=JlKp^(trv@Eya zL9&pNT;UPrwFRP*sRy(l7zNo>G)zCzMWhcvzivpTU e&8L4n!Lujh;gb9V?1ncI*s__8?>OqAhb91z4+9qf diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-90a8ba7d-9c17-4a24-a2e6-0b67534a6961-0 similarity index 86% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-1cee2cdf-03d4-48ab-8683-047e8b613dd1-0 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-90a8ba7d-9c17-4a24-a2e6-0b67534a6961-0 index c34df04b60ef44e90d2ebdb794c759632cc50835..7a696089dd447d9eb9ffe482de7026dd27c0c4cb 100644 GIT binary patch delta 308 zcmew@ut{)32YbD;{bPss`X7%UI2E(g&L~fdt&c@xtNvdGkFV?un|yTi8!R(<=6cO& zXPJ25;#PO-o2pB%H&v|I<*-}#CX?|2#an5&g=X2Ntq3w{nGo$ciRrK>t88dP_tV<@ z<)R)1L5}WUFNU{s)n0E)Y?#btq;!IRr@~H^or}DtSOlng%~88v?(m6Lq=!it zuP>{N)p#P9@k8<1oZ=vlH7{q+4~)3~y>@;ByO)oJLj@a8ho@dqZ_!3u^^NU0%xfGi zrLu$Ea*B3HWCrxxp6d2UOZ&S#T!EqB!D2;57ak=x!DbJI7+0w&4$9Y3csv*%Fa|S8 zRK+yB+j>pIa>fTGuJ37uIy{~lpM~_#C!c9vxYYeX4j&I28*{U$1c%WbMm_Yf0RXuE Bfm;9o delta 252 zcmdla_*-B@2YdZL?@8RpK1X?*gdB>WrJ1V9w18P-tNvdG4_h{dOWqnf3)C}t{vEu( zZ4ZNiG~+$v-X(u;nmcj4%!sJ1yPdM#cN?qyzcoQeZk>LsQ_l5%%Z$vmE>#+SO12^v zy32g(Yv(Gc$*8FP+I3C)@_$wKLoF^MV(iRpd-q)v3pXTQ7Xi(r}S$8mXQQMUM>RBsHm6FY#Rc?6OlFMkUYWis$1g(y8#<>bF72ifrJDDgBuNv3WsGMFroVq0Mfs4C;$Ke diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-f7c12f92-163b-45c2-9160-76375875e57b-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-f7c12f92-163b-45c2-9160-76375875e57b-1 deleted file mode 100644 index 480e49b90518a9a70d0095211debc5e47c99687c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2222 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8Oulca4*TQX2a_bCi8AUUl`Z%QbCE^=5S68O6YTB*$#FZnOB)thsfPT2{MxA3Mm31n`MRp8EQv zYG=&SSqAgGOcQO2OkaI@Z>}#ce%g7z@8?{TM;tDHgba^K=xSX1a`wE*otrsk#Y}1% zSDl)eTncP_C)sc;QhCv6bo(@W=J}oO#et%xCm33q3MQF&G_ssLbEm$FRe_=4!6HRQ z7ak=x!DbJI7*nYh2jy!iJRXb>7=xK4s$v@6ZM~*pIpc#8*Y~tS9#4(WLi*>E&onPw Y>V6=HkB5zoxmi?#!{`p99(t$%00xcX<^TWy diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-0 deleted file mode 100644 index 1179a001311c379997f06f7e2a6c757e3e3501f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 995 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxbu6*jf>A+E30;Nlg~2DlSRPOGzwBVF;8^i*nRjv(=D`Z_~;CwdPDa z7&W%)|7Gx)%f!&cA}DoWIrIPj@7oW|QvH7K(0wh>O}B3q_2pmx<`T5;{cm^k2e-r6 z4;e3z(zIO~wJPe0PuuY#)=eSpJ9ERWnC3oPdq{=5iHU{5L1KX(gTy^P2S#-300v!4 A#sB~S diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-1 deleted file mode 100644 index 48b19be2184f181034958a6e134d107ed6682167..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 996 zcmeZI%3@>@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxbu6*jf>A+E30;Nlg~2DlSRPOGzwBVYq+5^V!Lu`w9=f|KD7Yyitc~ z7o)~j{l5$zOPLrh9TAk-u+)<0umR6z2F{4WO5+5P$y1}NCZAVa!{MRJy6^q(?eC{s zf5_F2=a5KQ?HB%X(Uy!LJsV&46b;pRU;Vb#a(LV^p72P)i@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxbu6*jf>A+E30;Nlg~2DlSRPOGzwBVfeqO*QfZ4K!Q&4f~m!4GObve zm^8NP|7Gwv%FM8cT~cm?x-8EI6;B}+5f4_+6J}mo`@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxbu6*jf>A+E30;Nlg~2DlSRPOGzwBVVJj;H|gu8c{PXT$?R@BSH#S; zhf!mz{$B=q%^VUVP=$qv*yP&<(u9*A9y6UNTxH9iJ8G6mcd|;Yy%Uzg#dBk BQ33z} diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-0 similarity index 68% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-1 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-0 index 51b3950cb87297f438e1b16920725a8ff28f3089..1404cbe4cd098ec445543426580354eee63cb2b4 100644 GIT binary patch delta 236 zcmaFDexF0wKPiimMJ%zbC||EQIU_YU@fF|X07m(ZMpDd^XEE`}d-}x(`G?1Qx+v)= zl~fj_Dp^G<<(1~-0J%B&dFhk?Gbzg8Rba)eRUaANyOdq{qI&L6ePf@8?_M+QVbs{F z|ChmIEfd3`7D4F)so>zvMPlY(%r<{t`x~OPd;d^t#=`)Oza@iL3En;D2aEN6v*dyD( Jgk~S(AprJ5NVfn0 diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-1 similarity index 68% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-8c48d688-c706-4a24-b5bd-f930c1043d9e-0 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-1 index cc1b1ddd8f87f97bb1d5171230da1eca139f25e8..ee129c72818198951004b66abf5810678714cc18 100644 GIT binary patch delta 238 zcmeyu_K-u^KPiimMJ%zbC||EQIU_YU@fF|X07m(ZMpDd^XEE`}d-}x(`G?1Qx+v)= zl~fj_Dp^G<<(1~-0J%B&dFhk?Gbzg8Rba)eRiB*yTI-6y)3r8%lVsL@4CH4zz^JiR z|1X2bUM2>iRw3C9OD%a08}J-$6mUFpNo|hjC)-IXf|gGf2(c`g#?;Dry)<@rY3!Ya zd9$+?CwzGM>CfJkDH(lJ9-OyY<-&7uYV}(-PUi0nO`;B7%nS~(3!-WC7U; E04?%bBme*a delta 53 zcmV-50LuU22=oRMPhx5T1Qua-a&InkV`ybImI*?A5d^e~8cG;{3c58jq5xO@7kM+Mbcn)}}W zHaCB8TTO=Jv}0`dtxYwV(#yQMGhQ}$eNI?aKCQUR>(RW%4T}vJSr{B77U(fZ+~ad# HM7ItACTc}{ diff --git a/test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-1 similarity index 68% rename from test/test_data/orc/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-b54f1ecf-7c13-4dba-a1c5-19f988924166-0 rename to test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-1 index 14070357cab8480f2d05196373fc2fcdbf9e9912..41dd0a5aa08f930b828c6e7f2075cdcce8f2ad0d 100644 GIT binary patch delta 236 zcmaFNexF0wKPiimMJ%zbC||EQIU_YU@fF|X07m(ZMpDd^XEE`}d-}x(`G?1Qx+v)= zl~fj_Dp^G<<(1~-0J%B&dFhk?Gbzg8Rba)eRX>H{13z~&cR{(fzJueBc5$XXj2c_@ z|1x;2WnwtgA}D=8Rrde?|J)3_{@kk357L(Wrj}BCE>5)EQ-$@}zW2ZXotFKv?W!Bc z1KpBamvS^h@2y&Lyi9>la^2-A?&fouRdRVl z=H?G>tI2Sjc8u-5wW%gkdYM;u#>)n;&k3u_rxka3J(}0JVX*-t3xk8i0zC$adwdRz H=+*%M_EtcP diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/manifest/manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-0 new file mode 100644 index 0000000000000000000000000000000000000000..e03501c362775e427f3e6e77c95df175f0ec742c GIT binary patch literal 1155 zcmeZI%3@>@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhQN+($r2ZyZ7nsHxtIF8=O;@Glg3v4zYHE{nHfI0Nyu+dm#r}1Ih?@G8sxa`Zofp| zbmu(JCkx+uEK21#p~l?AeTP@7;_lX!Yuh%zno+Pz?b3YHS8nICnw5&m=P@ODrqO*DFrWNX<=r#aFFVQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxc@~pshjv;mAP(_bnt$(L)r?yQFE3trY?1y5#(n)MTNm;*!L?l*FPG zhUf{2PA|guy$d-lw{c&h>kOu2j2c_@|1x-7WnvI=6_(pzD$8@&famDLhX=MSk)12E z{rjnVDfTQ(nrgFyy%~?R|K9ih{=WWq2e!Y}5ow&_HDAxTZ<)c;V_Q#N+mxeKxpGTu c-evCt!s5qnvNeEN8A{*C`pAe7aMZHB~x6}2N93M3^aSQFD2SV9oK3*MRDO32?g>-2{^Sdq7JlC z3CWa<#ITNXt(yuePJdt=&!*5ckx8|tTk5Sung}DZ2KEwmh((GKs7oLzRsRT&(mHWP zJhtV~F2r87-TqYZ?A!DJVkPp}-R-mbwxt~{;<+jd{4XI`R60RQU#A{t3vIH$%^zv&KmCv9D-I!bWV$?-= pGu!`A{rq75_k%R#!Tx=wm@ODrqO*DFrWNXij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3)jdHT4<`#Ji$B9s)D6lLb6W2y@Fj6zbClaHpxFVr{Q(Z$8pB_3pEFv2{n zDqUQCTtkRZ8{ile;u!+;LVR$DV+g`j0)ZDFkEsz$@c4TA#fLfigd#gPsj?)s7{&L# zjuAwt@(cCxiFfu7^+R%Hv{GJaPL2{VyrIFLSX7i)2@IOdlGI#KOhM(9z+n_y8>^#~ zlA4xSnp2`=1=3j?TZVXi*pSxbu6*jf>A+E30;Nlg~2DlSRPOGzwBVd(vv_mR!CW{unh+jpruw1k*; zFlubo|I6Spmx-Z?MNsO%a_0a4-?tx_rTYEeq5E2%n{MAK>dU|W%_V5x``_;74{nFC zA2MDbrD?k~YE{$~pSI>eZmGcjks$G0lCp_K*s96B7%AgTw+o28nxo4vgs50RRfV BPig=F diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 index 5f36c97d..42062ec2 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-0 @@ -42,11 +42,12 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { + "bucket" : "-1", + "blob.target-file-size" : "50", + "row-tracking.enabled" : "true", "data-evolution.enabled" : "true", "manifest.format" : "avro", - "blob.target-file-size" : "50", - "file.format" : "parquet", - "row-tracking.enabled" : "true" + "file.format" : "parquet" }, - "timeMillis" : 1762240078799 + "timeMillis" : 1780568705752 } \ No newline at end of file diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 index e2f4b655..35f77e5c 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/schema/schema-1 @@ -27,7 +27,7 @@ "type" : "DECIMAL(6, 3)" }, { "id" : 8, - "name" : "f5", + "name" : "blob", "type" : "BLOB NOT NULL" }, { "id" : 5, @@ -42,11 +42,12 @@ "partitionKeys" : [ "key0", "key1" ], "primaryKeys" : [ ], "options" : { + "bucket" : "-1", + "blob.target-file-size" : "50", + "row-tracking.enabled" : "true", "data-evolution.enabled" : "true", "manifest.format" : "avro", - "blob.target-file-size" : "50", - "file.format" : "parquet", - "row-tracking.enabled" : "true" + "file.format" : "parquet" }, - "timeMillis" : 1762240081035 + "timeMillis" : 1780568707875 } \ No newline at end of file diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 index 02a8cd25..4a38b005 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-1 @@ -2,18 +2,15 @@ "version" : 3, "id" : 1, "schemaId" : 0, - "baseManifestList" : "manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-0", - "baseManifestListSize" : 884, - "deltaManifestList" : "manifest-list-f96bc893-0979-4689-a3d0-cd9cd2f1a203-1", - "deltaManifestListSize" : 995, - "changelogManifestList" : null, - "commitUser" : "ec4202d2-8ff3-4c75-9e6a-23281dad6562", - "commitIdentifier" : 1, + "baseManifestList" : "manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-ba29e330-ee07-4bd0-b5ee-02f21c647791-1", + "deltaManifestListSize" : 1119, + "commitUser" : "7aec5357-eb8d-42b4-bb9e-1753ec6e10ae", + "commitIdentifier" : 0, "commitKind" : "APPEND", - "timeMillis" : 1762240080710, - "logOffsets" : { }, + "timeMillis" : 1780568707810, "totalRecordCount" : 15, "deltaRecordCount" : 15, - "changelogRecordCount" : 0, "nextRowId" : 0 } \ No newline at end of file diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 index 76fdce1b..3ea1d014 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-2 @@ -2,18 +2,15 @@ "version" : 3, "id" : 2, "schemaId" : 1, - "baseManifestList" : "manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-0", - "baseManifestListSize" : 995, - "deltaManifestList" : "manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-1", - "deltaManifestListSize" : 996, - "changelogManifestList" : null, - "commitUser" : "9a49e1a1-9141-41a4-bfd1-403f0679d72c", - "commitIdentifier" : 2, + "baseManifestList" : "manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-0", + "baseManifestListSize" : 1119, + "deltaManifestList" : "manifest-list-6d497f15-c843-4136-b18c-25c34fa32b31-1", + "deltaManifestListSize" : 1121, + "commitUser" : "732dc057-d23c-456e-96a7-b888244afe95", + "commitIdentifier" : 1, "commitKind" : "APPEND", - "timeMillis" : 1762240081198, - "logOffsets" : { }, + "timeMillis" : 1780568707986, "totalRecordCount" : 20, "deltaRecordCount" : 5, - "changelogRecordCount" : 0, "nextRowId" : 0 } \ No newline at end of file diff --git a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 index cf9eb13b..3c972724 100644 --- a/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 +++ b/test/test_data/parquet/blob_append_table_alter_table_with_cast_with_data_evolution.db/blob_append_table_alter_table_with_cast_with_data_evolution/snapshot/snapshot-3 @@ -2,18 +2,15 @@ "version" : 3, "id" : 3, "schemaId" : 1, - "baseManifestList" : "manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-2", - "baseManifestListSize" : 1032, - "deltaManifestList" : "manifest-list-084c1aa1-da4f-4232-8e28-9d935ec4cea6-3", - "deltaManifestListSize" : 997, - "changelogManifestList" : null, - "commitUser" : "9a49e1a1-9141-41a4-bfd1-403f0679d72c", - "commitIdentifier" : 3, + "baseManifestList" : "manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-0", + "baseManifestListSize" : 1155, + "deltaManifestList" : "manifest-list-c802eb01-61a7-4f42-98fe-4f674b3c4966-1", + "deltaManifestListSize" : 1124, + "commitUser" : "6ae1e966-c41a-4699-b01f-0caf0b76c924", + "commitIdentifier" : 2, "commitKind" : "APPEND", - "timeMillis" : 1762240081236, - "logOffsets" : { }, + "timeMillis" : 1780568708049, "totalRecordCount" : 30, "deltaRecordCount" : 10, - "changelogRecordCount" : 0, "nextRowId" : 0 } \ No newline at end of file From 4ef48e9c9b641a36b1aec5a0970e720b5a973278 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:10:40 +0800 Subject: [PATCH 048/138] test: add blob compatible test --- test/inte/blob_table_inte_test.cpp | 90 +++++++++++++++++- .../blob_desc_field_with_external_path/README | 29 ++++++ ...aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc | Bin 0 -> 763 bytes ...eee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob | Bin 0 -> 77 bytes ...eee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob | Bin 0 -> 54 bytes ...eee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob | Bin 0 -> 29 bytes ...est-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 | Bin 0 -> 2932 bytes ...ist-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 | Bin 0 -> 392 bytes ...ist-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 | Bin 0 -> 1510 bytes .../raw_blob/b0-row-0.bin | 1 + .../raw_blob/b0-row-1.bin | 1 + .../raw_blob/b0-row-2.bin | 1 + .../schema/schema-0 | 41 ++++++++ .../snapshot/EARLIEST | 1 + .../snapshot/LATEST | 1 + .../snapshot/snapshot-1 | 16 ++++ .../blob_desc_field_with_external_path/README | 29 ++++++ ...888c-c975-46af-9a7d-36ca13c32455-1.parquet | Bin 0 -> 2379 bytes ...749888c-c975-46af-9a7d-36ca13c32455-2.blob | Bin 0 -> 77 bytes ...749888c-c975-46af-9a7d-36ca13c32455-3.blob | Bin 0 -> 54 bytes ...749888c-c975-46af-9a7d-36ca13c32455-0.blob | Bin 0 -> 29 bytes ...est-de59f444-0069-4836-8dcd-8a3a81158e02-0 | Bin 0 -> 2150 bytes ...ist-7395f790-699b-4a38-8747-10f23ceba1d6-0 | Bin 0 -> 1006 bytes ...ist-7395f790-699b-4a38-8747-10f23ceba1d6-1 | Bin 0 -> 1115 bytes .../raw_blob/b0-row-0.bin | 1 + .../raw_blob/b0-row-1.bin | 1 + .../raw_blob/b0-row-2.bin | 1 + .../schema/schema-0 | 41 ++++++++ .../snapshot/EARLIEST | 1 + .../snapshot/LATEST | 1 + .../snapshot/snapshot-1 | 16 ++++ 31 files changed, 267 insertions(+), 5 deletions(-) create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-0.bin create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-1.bin create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-2.bin create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/EARLIEST create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/LATEST create mode 100644 test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/snapshot-1 create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-2.blob create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-0.bin create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-1.bin create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-2.bin create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/EARLIEST create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/LATEST create mode 100644 test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/snapshot-1 diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 00433dcd..9cf19492 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -18,11 +18,11 @@ #include #include -#include -#include +#include #include #include #include +#include #include #include #include @@ -42,6 +42,7 @@ #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_view_struct.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" @@ -347,19 +348,58 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter }); } + struct BlobDescriptorPathRewrite { + std::string table_path; + std::vector table_relative_blob_dirs; + }; + + static std::optional TryRewriteDescriptorUri( + const std::string& descriptor_uri, const BlobDescriptorPathRewrite& rewrite, + const std::shared_ptr& fs) { + if (rewrite.table_path.empty()) { + return std::nullopt; + } + + for (const auto& blob_dir : rewrite.table_relative_blob_dirs) { + const std::string marker = "/" + blob_dir + "/"; + auto marker_pos = descriptor_uri.find(marker); + if (marker_pos != std::string::npos) { + std::string relative_blob_path = descriptor_uri.substr(marker_pos + 1); + return PathUtil::JoinPath(rewrite.table_path, relative_blob_path); + } + } + return std::nullopt; + } + /// Convert a StructArray with serialized BlobDescriptor bytes back to a StructArray /// with raw blob bytes. Only blob fields are resolved; other columns (including /// _VALUE_KIND) are kept as-is. Result> ConvertDescriptorToRawBlob( const std::shared_ptr& desc_array, - const std::set& blob_fields) const { + const std::set& blob_fields, + const BlobDescriptorPathRewrite& rewrite = {}) const { auto fs = std::make_shared(); return TransformBlobFields( desc_array, blob_fields, [&](const std::string_view& descriptor_bytes, arrow::LargeBinaryBuilder* builder) -> Status { - PAIMON_ASSIGN_OR_RAISE(auto blob, Blob::FromDescriptor(descriptor_bytes.data(), - descriptor_bytes.size())); + PAIMON_ASSIGN_OR_RAISE( + auto descriptor, + BlobDescriptor::Deserialize(descriptor_bytes.data(), descriptor_bytes.size())); + std::string descriptor_uri = descriptor->Uri(); + auto rewritten_uri = TryRewriteDescriptorUri(descriptor_uri, rewrite, fs); + if (rewritten_uri.has_value()) { + descriptor_uri = rewritten_uri.value(); + } + + PAIMON_ASSIGN_OR_RAISE( + auto rewritten_descriptor, + BlobDescriptor::Create(descriptor->Version(), descriptor_uri, + descriptor->Offset(), descriptor->Length())); + auto rewritten_descriptor_bytes = rewritten_descriptor->Serialize(pool_); + PAIMON_ASSIGN_OR_RAISE(auto blob, + Blob::FromDescriptor(rewritten_descriptor_bytes->data(), + rewritten_descriptor_bytes->size())); PAIMON_ASSIGN_OR_RAISE(auto data, blob->ToData(fs, pool_)); PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append(data->data(), data->size())); return Status::OK(); @@ -3001,4 +3041,44 @@ TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) { << "expected:" << expected_with_rk->ToString(); } +TEST_P(BlobTableInteTest, TestReadBlobDescriptorFieldFromJava) { + auto file_format = GetParam(); + if (file_format != "orc" && file_format != "parquet") { + return; + } + std::string table_path = + GetDataDir() + "/" + file_format + + "/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path"; + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), + BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), + BlobUtils::ToArrowField("b3", true)}; + auto schema = arrow::schema(fields); + // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls + std::string raw_json = R"([ + [1, "img_0", null, "raw_2_0", "raw_3_0"], + [2, "img_1", "vid_1", "raw_2_1", null ], + [3, "img_2", null, "raw_2_2", "raw_3_2" ] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + + // After read, b0 and b1 are both descriptor-stored; resolve all back to raw bytes. + // Java-generated descriptors may contain absolute paths from the generation machine. + // Rewrite them to the portable blob directories inside the copied table path. + BlobDescriptorPathRewrite rewrite{table_path, {"raw_blob", "external_blob"}}; + ASSERT_OK_AND_ASSIGN(auto resolved, + ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"}, rewrite)); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)); +} + } // namespace paimon::test diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README b/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README new file mode 100644 index 00000000..79f90708 --- /dev/null +++ b/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README @@ -0,0 +1,29 @@ +f0:int b0:blob b1:blob b2:blob b3:blob (all can be null) +bucket count: -1 +target-file-size: 700 +row-tracking.enabled: true +data-evolution.enabled: true +blob-descriptor-field: b0,b1 +blob-external-storage-field: b1 +blob-external-storage-path:
/external_blob (absolute path at generation time) + +b0: descriptor field, inline in main file, source .bin files in raw_blob/ +b1: descriptor field, repacked to external storage in external_blob/ +b2: regular blob, written to .blob files +b3: regular blob, written to .blob files + +Note: b0 is passed as descriptor via Blob.fromLocal(); b1/b2/b3 are raw bytes. +Paimon auto-converts b1 to descriptor internally. + +Msgs: +snapshot-1 +write field: "f0", "b0", "b1", "b2", "b3" +Add: 1, "img_0", null, "raw_2_0", "raw_3_0" +Add: 2, "img_1", "vid_1", "raw_2_1", null +Add: 3, "img_2", null, "raw_2_2", "raw_3_2" +NoCompact + +C++ read note: + Descriptor URIs contain absolute paths from the Java generation machine. + ConvertDescriptorToRawBlob uses BlobDescriptorPathRewrite{"raw_blob", "external_blob"} + to redirect them to
/raw_blob/ and
/external_blob/ at read time. diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc b/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc new file mode 100644 index 0000000000000000000000000000000000000000..179e5412d33ed52918613839e35eebf297f4413b GIT binary patch literal 763 zcmeYdau#G@;9?VE;b0D6&;~MvxtJLk7=(B@n1$Flm;~4)cmfzSf#O13tUz&~3?~P( z6Envj_5cP$ps*MhCs-IN#R!yI#vH&16m(-!Nw8vM(AcW~H{l<{To#5R4>{Ehrm{ST z4S4jf`p7A(>QA&i=4O6aTay|0^Y6SniL*w@g9_WyGJ>Xd3|=rA*V z?vVI&na@X)(S*~r*g=PP3G&h^Ao1TJ3gS|Q z%c`slm%LPzCzxCQ|6lLYd^uOeGp~D|itnK{)xVQPUgTxGSM@%2ch3J-p7Qc?^#aD% z+58-rIj-DT?4ccC)P3%d!d#Cw@i}K44lUn(^@aB1Ur{N$cGML3Z^+y=Y3+hD`=182 zezmunzAAl}%K9)*qyMfO-LlVp-MV6Z&;4J)h3%JECo*|<9{vze zp|+UJ_ZpL@Fq-r~yJ%y$e)$2=J;uyX+u1vKg-mNEi)u-J~+&8Pznzooe>3?>{L%mR~G zq_}}psKj$7g7iMPvAZG~xqao_M literal 0 HcmV?d00001 diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob b/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob new file mode 100644 index 0000000000000000000000000000000000000000..9cbdf5db3f6cbff7fe742fd48217db6041dc1b0b GIT binary patch literal 77 zcmX>v=oe9xSRQW_Zy?S90YPesb?4#2hA?44BdJV;un|o73g7eqJq8A5pk77*&lwOg literal 0 HcmV?d00001 diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob b/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob new file mode 100644 index 0000000000000000000000000000000000000000..8e8a3cdb65e6a85b6405bce3eb21ef879fa8667d GIT binary patch literal 54 rcmX>v=oe9xSRQX2Zy?S90h4~d-+vx1Yy=Y)xhOeNPv3wUsE82&v6BsK literal 0 HcmV?d00001 diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob b/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob new file mode 100644 index 0000000000000000000000000000000000000000..fa69a509e514bf6257b924fc9169a2b79945f90a GIT binary patch literal 29 ecmX>v=oe9znG$a($^Zd7gKy7d)X`=J@)!YgJqAkv literal 0 HcmV?d00001 diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 b/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 new file mode 100644 index 0000000000000000000000000000000000000000..5a29f6c000e1382894076007c9b955193e69f4c5 GIT binary patch literal 2932 zcmb_cdpJ~iAOFoc<2c4lGlrPa#V`t^VVH3%i`+#_5nZrma*2pg=z>L-ZW2jKVJRZ@ zy6z^qy-QOoq$jJST;H}-lBi9!$BcKMo;US8`#kSI-*e9Q`uTh>$7Q{h8~_A_ibr8S z(8gZn2`~VFNIj2Fg|xE}A3)#~D_*e^(#7yDl<-@C@k7yI%9Jc^ zhBOamgo4nhUt>5w@6UM5?=Zhs4+Etr0Y#o2f=~vHM=9|rEgofJCbGnYqJRp`_Wavb zqUVK-X}?kBi@5-7>iBtj0#P6u@(=WLTQX06@%DTJk_twgBq5HVKnNE2m75o{ON8QHYQG zY%pKqlg#$-014`^rvCuLEu8dO-K6^I9ULkAzcm!TS9M0Y6|3l z;Mq#ZSHb)r4PsurIZCES0Ic^1+MIsS}CwIl=$ygpJA)YQ%QJ76KNt zxy9WR3vrhv0!vBLCOG7X3Cq{Zx68MSabV)p-)^6Luw)jv+3z`}IQiX$0Qhr+9HE|2 zU&zIb5abJq|8G7=AR8HQyq%@WNp+G&54PfIIIoMww645$htrN*+nUt`TorDQDh$!` zOG_RC`2(7iZKhNtTl3-4JomM8qAXx71y!`L!&&^3vdhxAU#5C#u8OXJm-( zyI0uu|6A_ZMB@9QU1#jS8x1jtKAkMcl4?$PQ?HHppcckFSnlyG;>w=ZjXs2DCpWC? zy%4RnW-)HbQVqnkHqxpEthhq?K=<4!1Jysjyt|`mByf5$F`#&LVN+Ia`?wnOnHv==%i|T zuOf4Ja_oZPqd&V$qcXzm5^g@S&Lob8uij}P^3hRTq_f0x?=YDO_Hki%DZ_F{B*>gX z@m5h>geUjH=3F8URqR3X9{ZM`6-C9sAl92SFj9P14s1Tv5XNs2Od-)b!+6BJKRjA@ zWwVl4PQasy66;J(+EHPG+(aZRScE)O2Fc|G)(n?2I3H!8j7E}(L9$y$vqsUy>B+b= z(zoGlnbARH=A~sJ0}iG-w*wsy-gnfU zW(`ip?4_|Xc)ly+!#Wugj3$uswzj=p-?DZx=;(@@QwRNH+BAm$klSCdY)i${?A=Zo z0li)9{RJZz!_z+gS;2Bz>v7w=xs_C`puAB^$s~W^!nT7lq&Rs}o@JBCd+9B3toNIk zsPv94Exqe6b!z0S=-t3(MjjmXX{$OtR1#h@QeWU9{~_|wsER>S`KcWyFZ0!$ z{kmsHmTT16MDRsZ^6cL9(sSNj&x$M9FHaf%B5m}(NK}!=xczF=s@+|z>3GY4|D4PC zQFHuFv(@O6E^d7Vvu?NSwV$>=N1tAX^AIj%zMxh6fHE=F>W+`0nf>=AZd8Fa!&+VX z_qNtYO@qrrlso$~hqTDCW8ofk?t8lrmsUN!e7HfaUmKHQO5H{bQvJ#k;2OKyGM@OcTi=d1HP5 zIYYlOIJoRQedS)muI=BS&v;eVb=Ui71U{*+f3kU(irWiYh2)TDuU+nI74_>EO>1@R z58fWSau?k*kI`hAWQ!{46c27J#LedU7%ZGec-Pf?$^_*j7U4XZ>@Xx z@sLXM{*)gC*w_i7!-sSgR( zEJ+bere3?V+_PA2HBQc+Y0ab-73wTLR-ngvO?@J-X9-kI)f#!KM=TxQ*&W60v^yAnL#mw7*D;9Z+s=gU$Tr))EKCO&#h zb!f-FhslIb2b{SZf3qANv8XyVXx|svUeZl5nQ>f3G> kMkS{!J?Ix$*?6%eCzWmm-3s+mN=C}AF ze4M$vx??@>6Tpf<9+hLEpe8AL$jYM$I$4tk5Dp=Rh(?TPlS}BQDs-JVvGrISKb3pO z+%{qJX+B@>9do;1Im_ATk5liu_2=&HyF1kj0vhF$W-*e{11NU6OIo0J{<_U${N=B- z{c!a(oH5DD4JakUD3|n#AR5CZ#Rw$X5Sly$*(5W;6rkt>Ia%TW1U={rNkgG8H0Z$r zNkVrGdeA8iUf6|EE}4BEn$asZ?>eDGfv0XgrR%c{h>KjE^n3tKBT3n)QY-w(hmXv0xYi?#_tZR~LoT{5-mYS+-kYZ|@ zVqk7;Zl0E?Yak>M7r?;8AQ8%_yMWaYXq_0uI)0#aoE$H71zzY%ywJts5;g%g2{tU^ z3<3-i3;_&Co=^vxDhPBT2nca-Fgh`DNCYro7e#eJ07DWdM+1jI1BXNdhceI@eu(2Z zI2eSOK(=6n57hQSBMyx(b_NDF28VbC1`=ZjsLziA=7s2@Xv- z0hT&|RS=?{ND}Bs9Y%1{N+@7RWnoyPfA|+(TjF3+a=Di%Eb!Lsf|^gKH;##>9qzAmLCz=VPNrp*RHm_b?aDF6$F?a zt!p-jv3T1VM?Nv$Y0C6!ZThxZCkUM^to~e8-9j@jOe`Z!(JAbHm7` zRkqN5&0=ZYzc*C1R35aI7~e1EublM6G0N*`=F24xyBVIE8qeKnb8M?q(;QZgvy2Dq zI!l^k0~aP8y%O$izvfWM)}x2mxEYL_o7y>K?i~28ee8A5xid`P)B9RiFP@R(a?!(; z@ewm5SZ_1g@-wu}O0W*_T*GW29DDejjuLl+VpgDJ)`pBd{he&LO5(SDK3((JS1M^^ z`N3SrZM&YU7PiKTC{ASg5N}l0e8fvKfFX#%pW$xWrZ--jx~H!H-yb5vxmZqAi6D@(m~ z&qDH?%k2$+4;*Z}JoA7D&jzX3dPeoRB|R%%xamLq#oyQWRe4=lSgPy&1=~BPIreW| za%!9R?35WPOb1-&e@)fi_I|ti9x+GAk?J zEK*HNp/external_blob (absolute path at generation time) + +b0: descriptor field, inline in main file, source .bin files in raw_blob/ +b1: descriptor field, repacked to external storage in external_blob/ +b2: regular blob, written to .blob files +b3: regular blob, written to .blob files + +Note: b0 is passed as descriptor via Blob.fromLocal(); b1/b2/b3 are raw bytes. +Paimon auto-converts b1 to descriptor internally. + +Msgs: +snapshot-1 +write field: "f0", "b0", "b1", "b2", "b3" +Add: 1, "img_0", null, "raw_2_0", "raw_3_0" +Add: 2, "img_1", "vid_1", "raw_2_1", null +Add: 3, "img_2", null, "raw_2_2", "raw_3_2" +NoCompact + +C++ read note: + Descriptor URIs contain absolute paths from the Java generation machine. + ConvertDescriptorToRawBlob uses BlobDescriptorPathRewrite{"raw_blob", "external_blob"} + to redirect them to
/raw_blob/ and
/external_blob/ at read time. diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..42bd60e03504b93a325a19ee7db1800ee40f49fa GIT binary patch literal 2379 zcmWG=3^EjD5LFSi5j{8U zP}Kqs(MwFCuMf0;=f%+WhJ_)?Q&DZfa?Ah!|DO-_nEQXJ3a5t`-;sZQnZoXi#J10G zxcySiiSvJ&*`(m^O9eii^sw+Rl2fLU>e=v#uKRIC|8-@>BSQ(UhG}I?7 zfBFA*EEBpVGWE_P-=Yx5piL&arfrE*DL0p_)swWJ77%IFo=|4=Dk&qk$?4sWc}@kp zH@<#(IbiYAdv@oV?p$P!j@d3@Kke45O^?gU?!*Zg%k>87a=HZSn8fKdi+-{{eCR=f zV?cvJ!Jm8!zW@Eq3;|PBmKiNO)g=e?l%yb&qy!5nu7DIXkP>B*6oU#oFgXXix;Xj! zIQ0T0^fU5vQ}uHyDviuc49$)83lcMP^Yfrod`W6?i9VP}Ni0d!Ps+(picgW0pwx&| ztj^g+xpT-f0_+?X21yw)4$&-NFo9S_qAkpz5EJ{u4&r`e7F`A5FJlG;rjG|NB&6bt zlQU9t6Ghp8qN+?1Olbz8z|iIaF_R2JnP5!AP!i$_#MYyrEd9s|fM9++845VJ0E zh_x`Q71@|EXslwSG^$gJlZ!G7O7e^1(=tYpRfQ(|GTw;1^ zd|rNhVsU&5T)lo;e!gBxl73QRk$zEPc|0iP=qDNI7Uh@g8tElv<^e+)41nbV7)T6K z#~NUBEK+=c(g%asJRVUwv2*OA^O(@$#1I_6Zn&`i%jK_jU&GbcsC#K0)o*uu~t+1T7D#mLgkz&O>= o!qD8@FflnbDJ?ZQ+1xNGHOv=oe9xSRQW_Zy?S90YPesb?4#2hA?44BdJV;un|o73g7eqJq8A5pk77*&lwOg literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob new file mode 100644 index 0000000000000000000000000000000000000000..8e8a3cdb65e6a85b6405bce3eb21ef879fa8667d GIT binary patch literal 54 rcmX>v=oe9xSRQX2Zy?S90h4~d-+vx1Yy=Y)xhOeNPv3wUsE82&v6BsK literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob new file mode 100644 index 0000000000000000000000000000000000000000..fa69a509e514bf6257b924fc9169a2b79945f90a GIT binary patch literal 29 ecmX>v=oe9znG$a($^Zd7gKy7d)X`=J@)!YgJqAkv literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 new file mode 100644 index 0000000000000000000000000000000000000000..e8c82cc3303d1d345f5a383309a7dedb2139b9ac GIT binary patch literal 2150 zcmeZI%3@>@ODrqO*DFrWNX<>$CtIylQdy9yWTjM;nw(#hqNJmgmzayeFD^(-1_|aD zrRyaE*%_&N1&Nut`FVO^!_rgpQi~ExQbF3&GE;L>ij}OQt6?U^hq(p?d;0qUC82g@ z=9MVb>L3(&d-}QHmJM(W3h@j-~-QA$ZoODxSP zQL+N*tc|TjPK;nj!2JU;5^gm1Tmnf6@ep6awGRED0L5IeYhb9WpEEEuhx$6X1|iZI zW-0_GG2&GRJA1hLI>vhnLR!|}ip(=`GfG;mWO86TfZAZ3n!Xpl3a z)PZXUg)lzNNO=R83w;p54l@BBX1IbMpXtE(bPe(Y7SjQaAs(21gBGCpOmOoI3Jw8> zGE%yNxs7lThX;8ABZ*2L1C}bWwIbm9F*!daHCd<%m|XHw5{pt8emczX;9hxw>yYT3 z#l7O5E7`s=X>8U1%iuAUl_AJeL4CneNxuKhp`uSz>+Umk?32x&xcXY}o4Na|gq4(> zyr(?o$*(xA<{9MN$fY-F>-Xxb5)(SSl)4jLH@R$TNm~2;{r7}`0tbN$`{#bWcP;tx zw~ylQ^0wbLJ(TLp(B@!n;llv_S8cFkBx@vn(**Ke3 zm|;T4)iW|%e9V-k&2G>1*>tO+sJN(M#;dez3&fW2F^DvGHyk{Wkg$P?xu}5y-8TTr Cp3DXS literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 new file mode 100644 index 0000000000000000000000000000000000000000..402ee39dd34b01665b1e14415c681e4c3215ed14 GIT binary patch literal 1006 zcmbVLJ5Iwe7zT-%0U=nJydhPQSi70VhzKV^P0}K&$Z`|64a%c(T%=NR4=%vY6*vVW zI|FCHE~N=g#iLsse-D4(Z`=Kw?VSU9%QGzxMl7LkyE7oBV+NXlvmoOkXn>R^wRJwG zf%=*8p+@ERh-qUQC0VM?#fB_poC>DsgNVgp4w}8$mlElqwqw|q++27?sQ^Wz1e_Td z5eu5Agye!p>9CG+jOz+2&Q>s%XA-FENGD3uDfQMOb%c@o2ka$myM`+x5XXgXsrpNK zgw%;^>M;$AbRhPs$L&uA%fF>J;ng2dmWMHE*`8H7We=p2I0mxsMg5d>I+fK#f+cEF zi+wQf$S?c`2-$#6VgZ_pS@=bDSWfF4nUEOz59O*zr(Wv> zi%|#R&1^TNO4qjCtyZ#3``PJgqbqU$NA`>T`Kt@EkOg~pxu9uCdARld937t=zCJy? K+&^{`eDnc5#YBGq literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 new file mode 100644 index 0000000000000000000000000000000000000000..48db56792018c3a24280f8900ad385988a35c51b GIT binary patch literal 1115 zcmbVLzi!h&9Co9K5hPk6R&HX5QJ^>QvG$baephV}v_I-cu`+j!!=)*?!3I52^dghO4jISS_06rO0 z&;pG5DI=f;5*)8QXJhOaFH<(GV{t#C^)U|PG^tO;hBToJb6U`YkVYf}t;3VwN~jNe zj?uNn=FBThILK=Yz$?Rq$kr^lfMlG7$#50r81G7`C|$zXo`p1BhdL@WorT_=NFAEc zy$AM<)^iP4L_m(KxeL{Ql!wqN@s4^d!$y70^vcIQnsA!^k>1kY-UDTOrit2J&n}(v z5G2P@1VrDn`Y~g8BB}{Fjf+Xm_QABHK|ZJfLIZppanLH5`5-S3%V~F@r6EKAOSwg) zmtOldbmySCOELTvi_*tX1YMged`;Mg=6bg4QiW@;dr+=qp7yrWAOYrUUSJ-Kcl$%HW&CX@zeB=4W_D`vER*_Dt)%P(@!hmL6CUkc{ zXf~U&qU`OFP&A+FCaI1r5Vr A3IG5A literal 0 HcmV?d00001 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-0.bin b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-0.bin new file mode 100644 index 00000000..1207296c --- /dev/null +++ b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-0.bin @@ -0,0 +1 @@ +img_0 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-1.bin b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-1.bin new file mode 100644 index 00000000..a3f12b37 --- /dev/null +++ b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-1.bin @@ -0,0 +1 @@ +img_1 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-2.bin b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-2.bin new file mode 100644 index 00000000..c4c94eef --- /dev/null +++ b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-2.bin @@ -0,0 +1 @@ +img_2 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 new file mode 100644 index 00000000..174973a9 --- /dev/null +++ b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 @@ -0,0 +1,41 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "f0", + "type" : "INT" + }, { + "id" : 1, + "name" : "b0", + "type" : "BLOB" + }, { + "id" : 2, + "name" : "b1", + "type" : "BLOB" + }, { + "id" : 3, + "name" : "b2", + "type" : "BLOB" + }, { + "id" : 4, + "name" : "b3", + "type" : "BLOB" + } ], + "highestFieldId" : 4, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "row-tracking.enabled" : "true", + "blob-external-storage-path" : "external_blob", + "target-file-size" : "700", + "blob-external-storage-field" : "b1", + "data-evolution.enabled" : "true", + "file-system" : "local", + "manifest.format" : "avro", + "file.format" : "parquet", + "blob-descriptor-field" : "b0,b1" + }, + "timeMillis" : 1781088620905 +} \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/EARLIEST b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/EARLIEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/LATEST b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/LATEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/snapshot-1 b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/snapshot-1 new file mode 100644 index 00000000..99789afc --- /dev/null +++ b/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/snapshot-1 @@ -0,0 +1,16 @@ +{ + "version" : 3, + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1", + "deltaManifestListSize" : 1115, + "commitUser" : "d30e72bf-d9bb-4de4-a178-f11f2d2f4fa5", + "commitIdentifier" : 0, + "commitKind" : "APPEND", + "timeMillis" : 1781088622173, + "totalRecordCount" : 9, + "deltaRecordCount" : 9, + "nextRowId" : 3 +} \ No newline at end of file From 9ec410cb7d5428f9817839c0c36562c8af513737 Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Thu, 11 Jun 2026 19:06:06 +0800 Subject: [PATCH 049/138] refractor(parquet): simplify FileReaderWrapper and PageFilteredRowGroupReader with unified TargetRowGroup structure --- .../format/parquet/file_reader_wrapper.cpp | 530 ++++++++---------- .../format/parquet/file_reader_wrapper.h | 67 +-- .../parquet/file_reader_wrapper_test.cpp | 132 +++-- .../page_filtered_row_group_reader.cpp | 219 ++++---- .../parquet/page_filtered_row_group_reader.h | 54 +- .../page_filtered_row_group_reader_test.cpp | 29 +- .../parquet/parquet_file_batch_reader.cpp | 42 +- .../parquet/parquet_file_batch_reader.h | 11 +- src/paimon/format/parquet/row_ranges.h | 17 + 9 files changed, 567 insertions(+), 534 deletions(-) diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 8829a829..c3b395d2 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -80,8 +80,8 @@ std::vector<::arrow::io::ReadRange> MergeOverlappingRanges( } // namespace Result> FileReaderWrapper::Create( - std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, ::arrow::MemoryPool* pool, - int64_t batch_size) { + std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, int64_t batch_size, + std::shared_ptr<::arrow::MemoryPool> pool) { try { if (file_reader == nullptr) { return Status::Invalid("file reader wrapper create failed. file reader is nullptr"); @@ -101,15 +101,17 @@ Result> FileReaderWrapper::Create( return Status::Invalid(fmt::format( "unexpected error. row group ranges not match with num rows {}", num_rows)); } - std::vector row_groups_indices = - arrow::internal::Iota(file_reader->num_row_groups()); std::vector columns_indices = arrow::internal::Iota(file_reader->parquet_reader()->metadata()->num_columns()); auto file_reader_wrapper = std::unique_ptr(new FileReaderWrapper( - std::move(file_reader), all_row_group_ranges, num_rows, pool, batch_size)); - PAIMON_RETURN_NOT_OK(file_reader_wrapper->PrepareForReadingLazy( - std::set(row_groups_indices.begin(), row_groups_indices.end()), - columns_indices)); + std::move(file_reader), all_row_group_ranges, num_rows, batch_size, pool)); + std::vector all_target_row_groups; + for (int32_t i = 0; i < file_reader_wrapper->GetNumberOfRowGroups(); i++) { + all_target_row_groups.emplace_back(/*rg_index=*/i, /*page_filtered=*/false, + /*ranges=*/RowRanges()); + } + PAIMON_RETURN_NOT_OK( + file_reader_wrapper->PrepareForReadingLazy(all_target_row_groups, columns_indices)); return file_reader_wrapper; } PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::Create") @@ -141,7 +143,7 @@ Status FileReaderWrapper::Close() { FileReaderWrapper::FileReaderWrapper( std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, const std::vector>& all_row_group_ranges, uint64_t num_rows, - ::arrow::MemoryPool* pool, int64_t batch_size) + int64_t batch_size, std::shared_ptr<::arrow::MemoryPool> pool) : file_reader_(std::move(file_reader)), all_row_group_ranges_(all_row_group_ranges), pool_(pool), @@ -160,39 +162,54 @@ void FileReaderWrapper::WaitForPendingPreBuffer() { } } +void FileReaderWrapper::AdvanceToNextRowGroup() { + current_row_group_idx_++; + // Skip row groups excluded by read range. + while (current_row_group_idx_ < target_row_groups_.size() && + target_row_groups_[current_row_group_idx_].excluded_by_read_range) { + current_row_group_idx_++; + } + if (current_row_group_idx_ >= target_row_groups_.size()) { + next_row_to_read_ = num_rows_; + } else { + next_row_to_read_ = + all_row_group_ranges_[target_row_groups_[current_row_group_idx_].row_group_index].first; + } +} + Status FileReaderWrapper::SeekToRow(uint64_t row_number) { try { - // Reset any in-progress page-filtered streaming current_page_filtered_reader_.reset(); filtered_global_offset_ = 0; for (uint64_t i = 0; i < target_row_groups_.size(); i++) { - if (row_number > target_row_groups_[i].first && - row_number < target_row_groups_[i].second) { + if (target_row_groups_[i].excluded_by_read_range) { + continue; + } + uint32_t rg_id = target_row_groups_[i].row_group_index; + uint64_t rg_start = all_row_group_ranges_[rg_id].first; + uint64_t rg_end = all_row_group_ranges_[rg_id].second; + if (row_number > rg_start && row_number < rg_end) { return Status::Invalid( fmt::format("seek to row failed. row number {} should not be in the middle of " "readable range", row_number)); } - if (target_row_groups_[i].first >= row_number) { + if (rg_start >= row_number) { current_row_group_idx_ = i; - next_row_to_read_ = target_row_groups_[i].first; + next_row_to_read_ = rg_start; - // Rebuild batch_reader_ only for non-page-filtered row groups at/after seek - // position. Page-filtered RGs need no seek-side bookkeeping: their per-RG - // reader is constructed on demand in Next() from row_group_row_ranges_ each - // time, so backward seek "just works". - std::vector target_row_group_indices; + // Rebuild batch_reader_ for non-page-filtered RGs at/after seek position. + std::vector fully_matched_indices; for (uint64_t j = i; j < target_row_groups_.size(); j++) { - if (page_filtered_indices_.count(j) == 0) { - PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, - GetRowGroupId(target_row_groups_[j])); - target_row_group_indices.push_back(row_group_id); + if (!target_row_groups_[j].excluded_by_read_range && + !target_row_groups_[j].is_partially_matched) { + fully_matched_indices.push_back(target_row_groups_[j].row_group_index); } } - if (!target_row_group_indices.empty()) { + if (!fully_matched_indices.empty()) { PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( - target_row_group_indices, target_column_indices_, &batch_reader_)); + fully_matched_indices, target_column_indices_, &batch_reader_)); } else { batch_reader_.reset(); } @@ -206,126 +223,97 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::SeekToRow") } +Result> FileReaderWrapper::NextPageFiltered() { + int32_t rg_id = target_row_groups_[current_row_group_idx_].row_group_index; + + // Construct the per-RG streaming reader on demand. + if (!current_page_filtered_reader_) { + const auto& target_rg = target_row_groups_[current_row_group_idx_]; + auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( + file_reader_->parquet_reader(), target_rg, target_column_indices_); + bool pre_buffered = !prebuffered_ranges_.empty(); + int64_t max_chunksize = batch_size_ > 0 ? batch_size_ : std::numeric_limits::max(); + PAIMON_ASSIGN_OR_RAISE( + current_page_filtered_reader_, + PageFilteredRowGroupReader::ReadFilteredRowGroup( + file_reader_->parquet_reader(), target_rg, target_column_indices_, + page_filtered_read_schema_, file_reader_->properties().cache_options(), + pre_buffered, page_ranges, max_chunksize, pool_)); + current_filtered_row_ranges_ = target_rg.row_ranges; + current_filtered_rg_start_ = all_row_group_ranges_[rg_id].first; + filtered_global_offset_ = 0; + } + + std::shared_ptr record_batch; + PAIMON_RETURN_NOT_OK_FROM_ARROW(current_page_filtered_reader_->ReadNext(&record_batch)); + + if (record_batch) { + auto original_row = + current_filtered_row_ranges_.MapFilteredIndexToOriginalRow(filtered_global_offset_); + previous_first_row_ = original_row.has_value() ? current_filtered_rg_start_ + + static_cast(*original_row) + : current_filtered_rg_start_; + filtered_global_offset_ += record_batch->num_rows(); + return record_batch; + } + + // RG exhausted — reset and advance. + current_page_filtered_reader_.reset(); + filtered_global_offset_ = 0; + AdvanceToNextRowGroup(); + return std::shared_ptr(); +} + +Result> FileReaderWrapper::NextFullyMatched() { + if (!batch_reader_) { + return std::shared_ptr(); + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr record_batch, + batch_reader_->Next()); + if (!record_batch) { + return std::shared_ptr(); + } + + int32_t rg_id = target_row_groups_[current_row_group_idx_].row_group_index; + uint64_t rg_end = all_row_group_ranges_[rg_id].second; + int64_t num_rows = record_batch->num_rows(); + + previous_first_row_ = next_row_to_read_; + if (next_row_to_read_ + num_rows < rg_end) { + next_row_to_read_ += num_rows; + } else if (next_row_to_read_ + num_rows == rg_end) { + AdvanceToNextRowGroup(); + } else { + return Status::Invalid( + fmt::format("Next failed. next_row_to_read {} + num_rows {} exceeds row group end {}", + next_row_to_read_, num_rows, rg_end)); + } + return record_batch; +} + Result> FileReaderWrapper::Next() { try { if (PAIMON_UNLIKELY(!reader_initialized_)) { - PAIMON_RETURN_NOT_OK( - PrepareForReading(target_row_group_indices_, target_column_indices_)); + PAIMON_RETURN_NOT_OK(PrepareForReading(target_row_groups_, target_column_indices_)); } - // Loop until we produce a batch or exhaust all row groups. A null from the active - // per-RG reader means that RG is done; we advance and try the next RG without - // surfacing a spurious null to the caller. while (current_row_group_idx_ < target_row_groups_.size()) { - std::shared_ptr record_batch; - bool is_page_filtered = page_filtered_indices_.count(current_row_group_idx_) > 0; - - if (is_page_filtered) { - // Construct the per-RG streaming reader on demand. Inputs are recomputed each - // time from existing wrapper fields (no per-RG meta cached on the wrapper), - // mirroring how the fully-matched path delegates to Arrow's stateless - // GetRecordBatchReader. This makes both forward and backward seeks work - // uniformly: SeekToRow only resets current_page_filtered_reader_, and the - // next Next() rebuilds from authoritative state. - if (!current_page_filtered_reader_) { - PAIMON_ASSIGN_OR_RAISE( - int32_t rg_index, - GetRowGroupId(target_row_groups_[current_row_group_idx_])); - auto range_it = row_group_row_ranges_.find(rg_index); - if (range_it == row_group_row_ranges_.end()) { - return Status::Invalid( - fmt::format("page-filtered row group {} missing row ranges in " - "row_group_row_ranges_", - rg_index)); - } - const RowRanges& row_ranges = range_it->second; - auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( - file_reader_->parquet_reader(), rg_index, row_ranges, - target_column_indices_); - bool pre_buffered = !prebuffered_ranges_.empty(); - // batch_size_ == 0 means "no per-batch row cap" in the wrapper's contract, - // but TableBatchReader::set_chunksize(0) would loop forever emitting empty - // batches. Translate to int64_max so the reader produces one batch per - // underlying chunk boundary instead. - int64_t max_chunksize = - batch_size_ > 0 ? batch_size_ : std::numeric_limits::max(); - PAIMON_ASSIGN_OR_RAISE(current_page_filtered_reader_, - PageFilteredRowGroupReader::ReadFilteredRowGroup( - file_reader_->parquet_reader(), rg_index, row_ranges, - target_column_indices_, page_filtered_read_schema_, - pool_, file_reader_->properties().cache_options(), - pre_buffered, page_ranges, max_chunksize)); - current_filtered_row_ranges_ = row_ranges; - current_filtered_rg_start_ = target_row_groups_[current_row_group_idx_].first; - filtered_global_offset_ = 0; - } - PAIMON_RETURN_NOT_OK_FROM_ARROW( - current_page_filtered_reader_->ReadNext(&record_batch)); - } else if (batch_reader_) { - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(record_batch, batch_reader_->Next()); - } - - if (record_batch) { - int64_t num_rows = record_batch->num_rows(); - if (is_page_filtered) { - // Map the cumulative filtered-row offset back to the original row index - // within this row group. Must be evaluated BEFORE incrementing the offset. - auto original_row = current_filtered_row_ranges_.MapFilteredIndexToOriginalRow( - filtered_global_offset_); - previous_first_row_ = - original_row.has_value() - ? current_filtered_rg_start_ + static_cast(*original_row) - : current_filtered_rg_start_; - filtered_global_offset_ += num_rows; - // Stay on this RG; the next ReadNext will either return more data or null. - } else { - previous_first_row_ = next_row_to_read_; - if (next_row_to_read_ + num_rows < - target_row_groups_[current_row_group_idx_].second) { - next_row_to_read_ += num_rows; - } else if (next_row_to_read_ + num_rows == - target_row_groups_[current_row_group_idx_].second) { - if (current_row_group_idx_ == target_row_groups_.size() - 1) { - next_row_to_read_ = num_rows_; - } else { - current_row_group_idx_++; - next_row_to_read_ = target_row_groups_[current_row_group_idx_].first; - } - } else { - return Status::Invalid(fmt::format( - "Next failed. Unexpected error, next row to read {} + num rows just " - "read {} should always be within current row group range or exactly " - "equals to current row group end {}", - next_row_to_read_, num_rows, - target_row_groups_[current_row_group_idx_].second)); - } - } - return record_batch; - } - - // Null batch: current row group is exhausted (or fully-matched RGs hit a degenerate - // EOF). Advance to the next row group and continue the loop. - if (is_page_filtered) { - current_page_filtered_reader_.reset(); - filtered_global_offset_ = 0; - if (current_row_group_idx_ == target_row_groups_.size() - 1) { - next_row_to_read_ = num_rows_; - current_row_group_idx_ = target_row_groups_.size(); - } else { - current_row_group_idx_++; - next_row_to_read_ = target_row_groups_[current_row_group_idx_].first; - } - } else { - // Fully-matched path: batch_reader_ is exhausted with no more RBs to align on - // row counts. Stop here — remaining RGs (if any) should be page-filtered and - // will be handled by re-entering the loop, but if we got here without advancing - // first, treat as terminal to avoid an infinite loop. + bool is_page_filtered = target_row_groups_[current_row_group_idx_].is_partially_matched; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, + is_page_filtered ? NextPageFiltered() : NextFullyMatched()); + if (batch) { + return batch; + } else if (!is_page_filtered) { + // Null from fully-matched path means batch_reader_ is globally exhausted. break; } + // current_row_group_idx_ has been advanced in NextPageFiltered() or NextFullyMatched(), + // loop to try next RG. } previous_first_row_ = next_row_to_read_; - return std::shared_ptr(); // EOF + return std::shared_ptr(); } PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::Next") } @@ -343,191 +331,167 @@ Result>> FileReaderWrapper::GetRowGrou return row_group_ranges; } -Status FileReaderWrapper::PrepareForReadingLazy(const std::set& target_row_group_indices, - const std::vector& column_indices) { - target_row_group_indices_ = target_row_group_indices; +Status FileReaderWrapper::PrepareForReadingLazy( + const std::vector& target_row_groups, + const std::vector& column_indices) { + target_row_groups_ = target_row_groups; target_column_indices_ = column_indices; reader_initialized_ = false; return Status::OK(); } -Status FileReaderWrapper::PrepareForReading(const std::set& target_row_group_indices, - const std::vector& column_indices) { - try { - std::vector> target_row_groups; - PAIMON_ASSIGN_OR_RAISE(target_row_groups, GetRowGroupRanges(target_row_group_indices)); - - // Build position map: rg_index -> position in target_row_groups (O(1) lookup) - std::map rg_idx_to_position; - { - uint64_t pos = 0; - for (int32_t rg_idx : target_row_group_indices) { - rg_idx_to_position[rg_idx] = pos++; - } +Status FileReaderWrapper::BuildPageFilteredSchema(const std::vector& column_indices) { + if (page_filtered_read_schema_) { + return Status::OK(); + } + std::shared_ptr schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetSchema(&schema)); + auto parquet_schema = file_reader_->parquet_reader()->metadata()->schema(); + std::vector> fields; + for (int32_t col_idx : column_indices) { + const std::string& col_name = parquet_schema->Column(col_idx)->name(); + auto field = schema->GetFieldByName(col_name); + if (!field) { + return Status::Invalid(fmt::format( + "PrepareForReading: Parquet column {} ('{}') has no matching Arrow field in " + "file schema", + col_idx, col_name)); } + fields.push_back(field); + } + page_filtered_read_schema_ = arrow::schema(fields); + return Status::OK(); +} - // Separate row groups into fully matched (Arrow's standard reader) and partially - // matched (page-filtered, per-RG reader constructed on demand in Next()). - // Per-RG metadata for the page-filtered path is NOT cached on the wrapper — it's - // recomputed on demand in Next() from row_group_row_ranges_ + target_column_indices_, - // mirroring how the fully-matched path lets Arrow's FileReader own all metadata. - std::vector fully_matched_row_groups; - page_filtered_indices_.clear(); - page_filtered_read_schema_.reset(); +std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges( + const std::vector& column_indices) { + std::vector<::arrow::io::ReadRange> ranges; + auto file_metadata = file_reader_->parquet_reader()->metadata(); - // Page-level byte ranges collected here only for the bulk PreBuffer call below; - // discarded once PreBuffer is dispatched. - std::vector<::arrow::io::ReadRange> page_filtered_byte_ranges; - - for (int32_t rg_idx : target_row_group_indices) { - auto range_it = row_group_row_ranges_.find(rg_idx); - if (range_it != row_group_row_ranges_.end()) { - uint64_t pos = rg_idx_to_position[rg_idx]; - page_filtered_indices_.insert(pos); - - // Build the page-filter read_schema once on first encounter — it's identical - // across all page-filtered RGs in this session. - if (!page_filtered_read_schema_) { - std::shared_ptr schema; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetSchema(&schema)); - std::vector> fields; - auto parquet_schema = file_reader_->parquet_reader()->metadata()->schema(); - for (int32_t col_idx : column_indices) { - const std::string& col_name = parquet_schema->Column(col_idx)->name(); - auto field = schema->GetFieldByName(col_name); - if (!field) { - return Status::Invalid(fmt::format( - "PrepareForReading: Parquet column {} ('{}') has no matching Arrow " - "field in file schema", - col_idx, col_name)); - } - fields.push_back(field); - } - page_filtered_read_schema_ = arrow::schema(fields); + for (const auto& trg : target_row_groups_) { + if (trg.excluded_by_read_range) continue; + + if (trg.is_partially_matched) { + // Page-filtered RGs: only matching page byte ranges. + auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( + file_reader_->parquet_reader(), trg, column_indices); + ranges.insert(ranges.end(), std::make_move_iterator(page_ranges.begin()), + std::make_move_iterator(page_ranges.end())); + } else { + // Fully-matched RGs: entire column chunk ranges. + auto rg_metadata = file_metadata->RowGroup(trg.row_group_index); + for (int32_t col_idx : column_indices) { + auto col_chunk = rg_metadata->ColumnChunk(col_idx); + int64_t offset = col_chunk->data_page_offset(); + if (col_chunk->has_dictionary_page() && col_chunk->dictionary_page_offset() > 0 && + offset > col_chunk->dictionary_page_offset()) { + offset = col_chunk->dictionary_page_offset(); } + ranges.push_back({offset, col_chunk->total_compressed_size()}); + } + } + } + return ranges; +} - auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( - file_reader_->parquet_reader(), rg_idx, range_it->second, column_indices); - page_filtered_byte_ranges.insert(page_filtered_byte_ranges.end(), - std::make_move_iterator(page_ranges.begin()), - std::make_move_iterator(page_ranges.end())); - } else { - fully_matched_row_groups.push_back(rg_idx); +void FileReaderWrapper::DispatchPreBuffer(std::vector<::arrow::io::ReadRange> ranges) { + const auto& cache_opts = file_reader_->properties().cache_options(); + ::arrow::io::IOContext io_ctx(pool_.get()); + auto merged_ranges = MergeOverlappingRanges(std::move(ranges)); + try { + file_reader_->parquet_reader()->PreBufferRanges(merged_ranges, io_ctx, cache_opts); + prebuffered_ranges_ = std::move(merged_ranges); + } catch (const std::exception&) { + prebuffered_ranges_.clear(); + } +} + +Status FileReaderWrapper::PrepareForReading(const std::vector& target_row_groups, + const std::vector& column_indices) { + try { + target_row_groups_ = target_row_groups; + target_column_indices_ = column_indices; + page_filtered_read_schema_.reset(); + + // Partition into fully-matched and page-filtered row groups, skipping excluded ones. + std::vector fully_matched_row_groups; + uint64_t active_count = 0; + for (const auto& trg : target_row_groups_) { + if (trg.excluded_by_read_range) { + continue; + } + active_count++; + if (!trg.is_partially_matched) { + fully_matched_row_groups.push_back(trg.row_group_index); } } - // Wait for any previously pre-buffered data before starting new pre-buffer. + bool has_page_filtered = fully_matched_row_groups.size() != active_count; + if (has_page_filtered) { + PAIMON_RETURN_NOT_OK(BuildPageFilteredSchema(column_indices)); + } + WaitForPendingPreBuffer(); - // Create standard reader for fully matched row groups FIRST. - // GetRecordBatchReader internally calls PreBuffer, but we'll override it below - // with a single PreBuffer covering ALL row groups (page-filtered + fully-matched) - // so that async I/O for all files starts in parallel. - std::unique_ptr batch_reader; + // Create standard reader for fully-matched row groups. if (!fully_matched_row_groups.empty()) { PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( - fully_matched_row_groups, column_indices, &batch_reader)); + fully_matched_row_groups, column_indices, &batch_reader_)); + } else { + batch_reader_.reset(); } - // Collect all byte ranges for a single PreBufferRanges call. - // Page-filtered RGs: only matching page ranges (from ComputePageRanges). - // Fully-matched RGs: entire column chunk ranges. - // - // When there are no page-filtered RGs, skip the manual PreBufferRanges entirely: - // GetRecordBatchReader has already issued PreBuffer internally (driven by - // ArrowReaderProperties::pre_buffer=true), and a second PreBufferRanges call here - // would tear down and rebuild cached_source_, redundantly re-issuing the same IO - // on remote filesystems. The manual path is only needed to merge page-level ranges - // with column-chunk ranges into a single PreBuffer covering both kinds of RGs. - if (!page_filtered_indices_.empty()) { - std::vector<::arrow::io::ReadRange> all_ranges = std::move(page_filtered_byte_ranges); - - // Fully-matched row groups: add entire column chunk ranges - // The correct calculation follows Arrow's ColumnChunkMetaData::file_range(): - // - col_start = data_page_offset (or dictionary_page_offset if present and lower) - // - col_length = total_compressed_size (includes all pages: dictionary + data) - auto file_metadata = file_reader_->parquet_reader()->metadata(); - for (int32_t rg_idx : fully_matched_row_groups) { - auto rg_metadata = file_metadata->RowGroup(rg_idx); - for (int32_t col_idx : column_indices) { - auto col_chunk = rg_metadata->ColumnChunk(col_idx); - int64_t offset = col_chunk->data_page_offset(); - if (col_chunk->has_dictionary_page() && - col_chunk->dictionary_page_offset() > 0 && - offset > col_chunk->dictionary_page_offset()) { - offset = col_chunk->dictionary_page_offset(); - } - int64_t size = col_chunk->total_compressed_size(); - all_ranges.push_back({offset, size}); - } - } + // When page-filtered RGs exist, issue a single PreBuffer covering both kinds. + // Otherwise GetRecordBatchReader already issued PreBuffer internally. + if (has_page_filtered) { + auto all_ranges = CollectPreBufferRanges(column_indices); + DispatchPreBuffer(std::move(all_ranges)); + } - const auto& cache_opts = file_reader_->properties().cache_options(); - ::arrow::io::IOContext io_ctx(pool_); - // Merge overlapping ranges before calling PreBufferRanges, which rejects overlapping - // ranges. - auto merged_ranges = MergeOverlappingRanges(std::move(all_ranges)); - // PreBuffer is an optimization - if it fails (e.g., IO error during testing), - // continue without pre-buffering. Subsequent reads will fetch data on-demand. - try { - file_reader_->parquet_reader()->PreBufferRanges(merged_ranges, io_ctx, cache_opts); - // Track for cleanup on destruction - prebuffered_ranges_ = std::move(merged_ranges); - } catch (const std::exception& e) { - // Pre-buffering failed, clear ranges to indicate no pre-buffered data available. - // Reading will fall back to on-demand I/O. - prebuffered_ranges_.clear(); - } + // Reset read state. Find the first non-excluded row group. + uint64_t first_active_idx = 0; + while (first_active_idx < target_row_groups_.size() && + target_row_groups_[first_active_idx].excluded_by_read_range) { + first_active_idx++; } - target_row_groups_ = target_row_groups; - target_column_indices_ = column_indices; - batch_reader_ = std::move(batch_reader); - if (target_row_groups_.empty()) { + if (first_active_idx >= target_row_groups_.size()) { next_row_to_read_ = num_rows_; } else { - next_row_to_read_ = target_row_groups_[0].first; + next_row_to_read_ = + all_row_group_ranges_[target_row_groups_[first_active_idx].row_group_index].first; } previous_first_row_ = std::numeric_limits::max(); - current_row_group_idx_ = 0; + current_row_group_idx_ = first_active_idx; reader_initialized_ = true; return Status::OK(); } PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("FileReaderWrapper::PrepareForReading") } -Result> FileReaderWrapper::FilterRowGroupsByReadRanges( - const std::vector>& read_ranges, - const std::vector& src_row_groups) const { - std::set target_row_groups; - PAIMON_ASSIGN_OR_RAISE(std::set row_groups_to_read, - ReadRangesToRowGroupIds(read_ranges)); - for (const auto& row_group_id : src_row_groups) { - if (row_groups_to_read.find(row_group_id) != row_groups_to_read.end()) { - target_row_groups.emplace(row_group_id); +Status FileReaderWrapper::ApplyReadRanges( + const std::vector>& read_ranges) { + if (read_ranges.empty()) { + for (auto& trg : target_row_groups_) { + trg.excluded_by_read_range = true; } + reader_initialized_ = false; + return Status::OK(); } - return target_row_groups; -} - -Result> FileReaderWrapper::ReadRangesToRowGroupIds( - const std::vector>& read_ranges) const { - std::set selected_row_group_ids; + // Build a set of row group indices whose range matches one of the read ranges. + std::set matching_rg_indices; for (const auto& read_range : read_ranges) { - PAIMON_ASSIGN_OR_RAISE(int32_t row_group_id, GetRowGroupId(read_range)); - selected_row_group_ids.emplace(row_group_id); - } - return selected_row_group_ids; -} - -Result FileReaderWrapper::GetRowGroupId(std::pair target_range) const { - for (size_t i = 0; i < all_row_group_ranges_.size(); i++) { - if (all_row_group_ranges_[i] == target_range) { - return i; + for (size_t i = 0; i < all_row_group_ranges_.size(); i++) { + if (all_row_group_ranges_[i] == read_range) { + matching_rg_indices.insert(static_cast(i)); + } } } - return Status::Invalid(fmt::format( - "not expected failure. target range bound '{},{}' not match with row group range bound", - target_range.first, target_range.second)); + // Mark each target row group as excluded or not based on the matching set. + for (auto& trg : target_row_groups_) { + trg.excluded_by_read_range = matching_rg_indices.count(trg.row_group_index) == 0; + } + reader_initialized_ = false; + return Status::OK(); } std::shared_ptr<::parquet::PageIndexReader> FileReaderWrapper::GetPageIndexReader() { diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h index 3d02164e..373d0159 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.h +++ b/src/paimon/format/parquet/file_reader_wrapper.h @@ -57,8 +57,8 @@ class FileReaderWrapper { ~FileReaderWrapper(); static Result> Create( - std::unique_ptr<::parquet::arrow::FileReader>&& reader, ::arrow::MemoryPool* pool, - int64_t batch_size); + std::unique_ptr<::parquet::arrow::FileReader>&& reader, int64_t batch_size, + std::shared_ptr pool); /// Seek to the specified row number. /// @param row_number The row to seek to (must be at a row group boundary). @@ -89,7 +89,7 @@ class FileReaderWrapper { } /// Get the underlying Parquet file reader. - ::parquet::arrow::FileReader* GetFileReader() const { + ::parquet::arrow::FileReader* GetFileReader() { return file_reader_.get(); } @@ -111,24 +111,18 @@ class FileReaderWrapper { /// Prepare for lazy reading of the specified row groups and columns. /// Actual reader initialization is deferred until the first Next() call. - Status PrepareForReadingLazy(const std::set& row_group_indices, + Status PrepareForReadingLazy(const std::vector& target_row_groups, const std::vector& column_indices); /// Prepare for immediate reading of the specified row groups and columns. /// Initializes the reader and starts pre-buffering I/O. - Status PrepareForReading(const std::set& row_group_indices, + Status PrepareForReading(const std::vector& target_row_groups, const std::vector& column_indices); - /// Filter row groups by read ranges, returning only those that overlap. - Result> FilterRowGroupsByReadRanges( - const std::vector>& read_ranges, - const std::vector& src_row_groups) const; - - /// Set per-row-group RowRanges for page-level filtering. - /// Only partially matched row groups should have entries. - void SetRowGroupRowRanges(const std::map& ranges) { - row_group_row_ranges_ = ranges; - } + /// Apply read ranges to the current target_row_groups_, keeping only those + /// whose row-group range is equal to one of the given read ranges. + /// Resets reader state so that the next Next() call will re-initialize. + Status ApplyReadRanges(const std::vector>& read_ranges); /// Get the page index reader for the file. /// Returns nullptr if page index is not available. @@ -146,21 +140,38 @@ class FileReaderWrapper { private: FileReaderWrapper(std::unique_ptr<::parquet::arrow::FileReader>&& file_reader, const std::vector>& all_row_group_ranges, - uint64_t num_rows, ::arrow::MemoryPool* pool, int64_t batch_size); + uint64_t num_rows, int64_t batch_size, + std::shared_ptr<::arrow::MemoryPool> pool); + + /// Wait for all pending PreBuffer operations to complete. + void WaitForPendingPreBuffer(); + + /// Advance current_row_group_idx_ to the next row group and update next_row_to_read_. + void AdvanceToNextRowGroup(); + + /// Read next batch from a page-filtered row group. Returns nullptr when the RG is exhausted. + Result> NextPageFiltered(); - Result> ReadRangesToRowGroupIds( - const std::vector>& read_ranges) const; - Result GetRowGroupId(std::pair target_range) const; + /// Read next batch from the fully-matched batch_reader_. Returns nullptr when exhausted. + Result> NextFullyMatched(); + + /// Build page_filtered_read_schema_ from the given column indices. No-op if already built. + Status BuildPageFilteredSchema(const std::vector& column_indices); + + /// Collect all byte ranges that need pre-buffering (page-filtered + fully-matched). + std::vector<::arrow::io::ReadRange> CollectPreBufferRanges( + const std::vector& column_indices); + + /// Dispatch a single PreBufferRanges call with merged ranges. + void DispatchPreBuffer(std::vector<::arrow::io::ReadRange> ranges); std::unique_ptr<::parquet::arrow::FileReader> file_reader_; std::unique_ptr batch_reader_; std::vector> all_row_group_ranges_; - std::set target_row_group_indices_; - std::vector> target_row_groups_; std::vector target_column_indices_; - ::arrow::MemoryPool* pool_; + std::shared_ptr<::arrow::MemoryPool> pool_; int64_t batch_size_; // 0 means no limit const uint64_t num_rows_; @@ -177,13 +188,8 @@ class FileReaderWrapper { RowRanges current_filtered_row_ranges_; // RowRanges for the active page-filtered RG uint64_t current_filtered_rg_start_ = 0; // Absolute row-group start row number - // Page-level filtering state. Externally injected via SetRowGroupRowRanges and - // looked up by row group index when entering a page-filtered RG. - std::map row_group_row_ranges_; - - // Set of target_row_groups_ positional indices that use page-filtered reading. - // Built in PrepareForReading from row_group_row_ranges_. - std::set page_filtered_indices_; + // Target row groups with row ranges for none page-level filtering and page-level filtering + std::vector target_row_groups_; // Arrow schema covering target_column_indices_, used when constructing the per-RG // page-filtered reader. Cached in PrepareForReading because it's identical across @@ -192,9 +198,6 @@ class FileReaderWrapper { // Track pre-buffered ranges so we can wait on destruction std::vector<::arrow::io::ReadRange> prebuffered_ranges_; - - /// Wait for all pending PreBuffer operations to complete. - void WaitForPendingPreBuffer(); }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/file_reader_wrapper_test.cpp b/src/paimon/format/parquet/file_reader_wrapper_test.cpp index b18d0277..59d71b3f 100644 --- a/src/paimon/format/parquet/file_reader_wrapper_test.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper_test.cpp @@ -137,8 +137,7 @@ class FileReaderWrapperTest : public ::testing::Test { PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.memory_pool(arrow_pool_.get()) ->properties(arrow_reader_props) ->Build(&file_reader)); - return FileReaderWrapper::Create(std::move(file_reader), ::arrow::default_memory_pool(), - wrapper_batch_size); + return FileReaderWrapper::Create(std::move(file_reader), wrapper_batch_size, arrow_pool_); } void PrepareParquetFile(const std::string& file_path, int32_t row_count, @@ -198,8 +197,9 @@ TEST_F(FileReaderWrapperTest, EmptyFile) { } TEST_F(FileReaderWrapperTest, NullFileReader) { - ASSERT_NOK_WITH_MSG(FileReaderWrapper::Create(nullptr, ::arrow::default_memory_pool(), - /*batch_size=*/0), + ASSERT_NOK_WITH_MSG(FileReaderWrapper::Create(nullptr, + /*batch_size=*/0, + /*pool=*/arrow_pool_), "file reader wrapper create failed. file reader is nullptr"); } @@ -263,11 +263,11 @@ TEST_F(FileReaderWrapperTest, PageFilteredZeroBatchSizeDoesNotHang) { // contiguous ranges keep the test honest about RowRanges semantics; the actual // numbers don't matter as long as their total falls inside the row group. RowRanges rr({RowRanges::Range(0, 49), RowRanges::Range(100, 149)}); - reader_wrapper->SetRowGroupRowRanges({{0, rr}}); std::vector all_columns = {0, 1, 2}; - ASSERT_OK(reader_wrapper->PrepareForReading({0}, all_columns)); - + ASSERT_OK(reader_wrapper->PrepareForReading( + {TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/rr)}, + all_columns)); int64_t total = 0; int64_t batch_count = 0; while (true) { @@ -297,10 +297,14 @@ TEST_F(FileReaderWrapperTest, SeekBackToConsumedPageFilteredRowGroup) { std::map row_ranges_map; row_ranges_map[0] = RowRanges(RowRanges::Range(10, 49)); row_ranges_map[1] = RowRanges(RowRanges::Range(100, 149)); - reader_wrapper->SetRowGroupRowRanges(row_ranges_map); std::vector all_columns = {0, 1, 2}; - ASSERT_OK(reader_wrapper->PrepareForReading({0, 1}, all_columns)); + ASSERT_OK(reader_wrapper->PrepareForReading( + {TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, + /*ranges=*/row_ranges_map[0]), + TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/true, + /*ranges=*/row_ranges_map[1])}, + all_columns)); auto count_all_rows = [&](int64_t* out_total) { int64_t total = 0; @@ -350,8 +354,9 @@ TEST_F(FileReaderWrapperTest, PageFilteredRespectsBatchSize) { for (int64_t batch_size : {int64_t{1}, int64_t{2}, int64_t{3}, int64_t{5}, int64_t{10}}) { SCOPED_TRACE("batch_size=" + std::to_string(batch_size)); ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path, batch_size)); - reader_wrapper->SetRowGroupRowRanges({{0, rr}}); - ASSERT_OK(reader_wrapper->PrepareForReading({0}, {0, 1, 2})); + ASSERT_OK(reader_wrapper->PrepareForReading( + {TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/rr)}, + {0, 1, 2})); int64_t total = 0; int64_t batch_count = 0; @@ -382,45 +387,93 @@ TEST_F(FileReaderWrapperTest, GetRowGroupRanges) { ASSERT_TRUE(ranges.empty()); } -TEST_F(FileReaderWrapperTest, ReadRangesToRowGroupIds) { +TEST_F(FileReaderWrapperTest, ApplyReadRanges) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); PrepareParquetFile(file_path, /*row_count=*/5500); ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); - std::set expected_row_group_ids = {0, 3, 5}; + + // Prepare with a subset of row groups: {0, 1, 2, 4, 5} + std::vector initial_targets = { + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/2, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/4, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/5, /*is_partially_matched=*/false, + /*ranges=*/RowRanges())}; + std::vector all_columns = {0, 1, 2}; + ASSERT_OK(reader_wrapper->PrepareForReadingLazy(initial_targets, all_columns)); + + // Apply read ranges that match RG 0, 3, 5. Only 0 and 5 are in initial targets. std::vector> read_ranges = { {0, 1000}, {3000, 4000}, {5000, 5500}}; - ASSERT_OK_AND_ASSIGN(auto row_group_ids, reader_wrapper->ReadRangesToRowGroupIds(read_ranges)); - ASSERT_EQ(expected_row_group_ids, row_group_ids); - std::vector> invalid_ranges = { - {0, 1000}, {3000, 4000}, {5000, 5600}}; - ASSERT_NOK_WITH_MSG(reader_wrapper->ReadRangesToRowGroupIds(invalid_ranges), - "not match with row group range bound"); - ASSERT_OK_AND_ASSIGN(row_group_ids, reader_wrapper->ReadRangesToRowGroupIds({})); - ASSERT_TRUE(row_group_ids.empty()); + ASSERT_OK(reader_wrapper->ApplyReadRanges(read_ranges)); + + // Verify: reading should only produce rows from RG 0 (1000 rows) and RG 5 (500 rows). + int64_t total_rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + if (!batch) { + break; + } + total_rows += batch->num_rows(); + } + ASSERT_EQ(1500, total_rows); + + // Apply empty read ranges should result in no data. + ASSERT_OK(reader_wrapper->PrepareForReadingLazy(initial_targets, all_columns)); + ASSERT_OK(reader_wrapper->ApplyReadRanges({})); + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + ASSERT_FALSE(batch); } -TEST_F(FileReaderWrapperTest, FilterRowGroupsByReadRanges) { +TEST_F(FileReaderWrapperTest, ApplyReadRangesWiderSecondCall) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); PrepareParquetFile(file_path, /*row_count=*/5500); ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); - std::set expected_row_group_ids = {0, 5}; - std::vector> read_ranges = { - {0, 1000}, {3000, 4000}, {5000, 5500}}; - ASSERT_OK_AND_ASSIGN(auto row_group_ids, - reader_wrapper->FilterRowGroupsByReadRanges(read_ranges, {0, 1, 2, 4, 5})); - ASSERT_EQ(expected_row_group_ids, row_group_ids); - ASSERT_OK_AND_ASSIGN(row_group_ids, - reader_wrapper->FilterRowGroupsByReadRanges(read_ranges, {})); - ASSERT_TRUE(row_group_ids.empty()); + // Prepare with row groups: {0, 1, 2, 4, 5} + std::vector initial_targets = { + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/2, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/4, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/5, /*is_partially_matched=*/false, + /*ranges=*/RowRanges())}; + std::vector all_columns = {0, 1, 2}; + ASSERT_OK(reader_wrapper->PrepareForReadingLazy(initial_targets, all_columns)); + + // First ApplyReadRanges: narrow to RG 0 only. + ASSERT_OK(reader_wrapper->ApplyReadRanges({{0, 1000}})); + + // Second ApplyReadRanges: widen to RG 0, 1, 2. Previously excluded RG 1, 2 should restore. + ASSERT_OK(reader_wrapper->ApplyReadRanges({{0, 1000}, {1000, 2000}, {2000, 3000}})); + + // Verify: reading should produce rows from RG 0 + 1 + 2 = 3000 rows. + int64_t total_rows = 0; + while (true) { + ASSERT_OK_AND_ASSIGN(auto batch, reader_wrapper->Next()); + if (!batch) break; + total_rows += batch->num_rows(); + } + ASSERT_EQ(3000, total_rows); } TEST_F(FileReaderWrapperTest, PrepareForReading) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "test.parquet"); PrepareParquetFile(file_path, /*row_count=*/5500); ASSERT_OK_AND_ASSIGN(auto reader_wrapper, PrepareReaderWrapper(file_path)); - ASSERT_OK(reader_wrapper->PrepareForReading(/*row_group_indices=*/{1}, - /*column_indices=*/{0})); + ASSERT_OK(reader_wrapper->PrepareForReading( + /*target_row_groups=*/{TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, + /*ranges=*/RowRanges())}, + /*column_indices=*/{0})); // seek before actual read range ASSERT_OK(reader_wrapper->SeekToRow(0)); ASSERT_EQ(1000, reader_wrapper->GetNextRowToRead()); @@ -440,8 +493,12 @@ TEST_F(FileReaderWrapperTest, PrepareForReading) { ASSERT_FALSE(record_batch); // empty column indices - ASSERT_OK(reader_wrapper->PrepareForReading(/*row_group_indices=*/{0, 1}, - /*column_indices=*/{})); + ASSERT_OK(reader_wrapper->PrepareForReading( + /*target_row_groups=*/{TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, + /*ranges=*/RowRanges()), + TargetRowGroup(/*rg_index=*/1, /*is_partially_matched=*/false, + /*ranges=*/RowRanges())}, + /*column_indices=*/{})); ASSERT_EQ(0, reader_wrapper->GetNextRowToRead()); ASSERT_EQ(std::numeric_limits::max(), reader_wrapper->GetPreviousBatchFirstRowNumber().value()); @@ -450,8 +507,9 @@ TEST_F(FileReaderWrapperTest, PrepareForReading) { ASSERT_EQ(0, record_batch->num_columns()); // empty row group indices - ASSERT_OK(reader_wrapper->PrepareForReading(/*row_group_indices=*/{}, - /*column_indices=*/{0})); + ASSERT_OK(reader_wrapper->PrepareForReading( + /*target_row_groups=*/{}, + /*column_indices=*/{0})); ASSERT_EQ(5500, reader_wrapper->GetNextRowToRead()); ASSERT_EQ(std::numeric_limits::max(), reader_wrapper->GetPreviousBatchFirstRowNumber().value()); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index f729d729..af073b13 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -63,34 +63,30 @@ class TableRecordBatchReader : public arrow::RecordBatchReader { } // namespace +std::pair PageFilteredRowGroupReader::GetPageRowRange( + const std::vector<::parquet::PageLocation>& page_locations, int32_t page_idx, + int64_t row_group_row_count) { + int64_t first_row = page_locations[page_idx].first_row_index; + int64_t last_row = (page_idx + 1 < static_cast(page_locations.size())) + ? page_locations[page_idx + 1].first_row_index - 1 + : row_group_row_count - 1; + return {first_row, last_row}; +} + std::function PageFilteredRowGroupReader::MakePageFilter( const RowRanges& row_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count) { - // Shared counter tracks the current page index as the callback is invoked - // in order for each data page. auto page_counter = std::make_shared(0); - const auto& page_locations = offset_index->page_locations(); auto num_pages = static_cast(page_locations.size()); return [row_ranges, page_locations, num_pages, row_group_row_count, page_counter](const ::parquet::DataPageStats& /*stats*/) -> bool { int32_t page_idx = (*page_counter)++; - if (page_idx >= num_pages) { - // Safety: if more pages than expected, don't skip return false; } - - int64_t first_row = page_locations[page_idx].first_row_index; - int64_t last_row; - if (page_idx + 1 < num_pages) { - last_row = page_locations[page_idx + 1].first_row_index - 1; - } else { - last_row = row_group_row_count - 1; - } - - // Return true to skip this page if it has no overlap with RowRanges + auto [first_row, last_row] = GetPageRowRange(page_locations, page_idx, row_group_row_count); return !row_ranges.IsOverlapping(first_row, last_row); }; } @@ -106,10 +102,7 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa int64_t compressed_offset = 0; for (int32_t page_idx = 0; page_idx < num_pages; ++page_idx) { - int64_t page_from = page_locations[page_idx].first_row_index; - int64_t page_to = (page_idx + 1 < num_pages) - ? page_locations[page_idx + 1].first_row_index - 1 - : row_group_row_count - 1; + auto [page_from, page_to] = GetPageRowRange(page_locations, page_idx, row_group_row_count); int64_t page_size = page_to - page_from + 1; if (!original_ranges.IsOverlapping(page_from, page_to)) { @@ -117,19 +110,13 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa continue; } - // Page is kept. Map overlapping original ranges to compressed row space. for (const auto& range : ranges) { - if (range.to < page_from) { - continue; - } - if (range.from > page_to) { - break; // Ranges are sorted - } + if (range.to < page_from) continue; + if (range.from > page_to) break; int64_t overlap_from = std::max(range.from, page_from); int64_t overlap_to = std::min(range.to, page_to); - int64_t c_from = compressed_offset + (overlap_from - page_from); - int64_t c_to = compressed_offset + (overlap_to - page_from); - compressed.Add(RowRanges::Range(c_from, c_to)); + compressed.Add(RowRanges::Range(compressed_offset + (overlap_from - page_from), + compressed_offset + (overlap_to - page_from))); } compressed_offset += page_size; @@ -138,13 +125,45 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa return {compressed, compressed_offset}; } +Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( + std::shared_ptr<::parquet::internal::RecordReader> record_reader, const RowRanges& ranges, + int64_t total_row_count, int32_t row_group_index, int32_t column_index) { + int64_t current_row = 0; + for (const auto& range : ranges.GetRanges()) { + if (range.from > current_row) { + int64_t to_skip = range.from - current_row; + int64_t skipped = record_reader->SkipRecords(to_skip); + if (skipped != to_skip) { + return Status::Invalid(fmt::format( + "PageFilteredRowGroupReader: expected to skip {} records but skipped {} " + "(row_group={}, column={})", + to_skip, skipped, row_group_index, column_index)); + } + current_row = range.from; + } + int64_t to_read = range.Count(); + int64_t read = record_reader->ReadRecords(to_read); + if (read != to_read) { + return Status::Invalid( + fmt::format("PageFilteredRowGroupReader: expected to read {} records but read {} " + "(row_group={}, column={}, range=[{},{}])", + to_read, read, row_group_index, column_index, range.from, range.to)); + } + current_row += to_read; + } + if (current_row < total_row_count) { + record_reader->SkipRecords(total_row_count - current_row); + } + return Status::OK(); +} + Result> PageFilteredRowGroupReader::ReadFilteredColumn( const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader, ::parquet::ParquetFileReader* parquet_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, int32_t row_group_index, int32_t column_index, const RowRanges& row_ranges, const std::shared_ptr& field, int64_t row_group_row_count, - ::arrow::MemoryPool* pool) { + std::shared_ptr<::arrow::MemoryPool> pool) { auto file_metadata = parquet_reader->metadata(); const auto* col_descriptor = file_metadata->schema()->Column(column_index); @@ -173,102 +192,72 @@ Result> PageFilteredRowGroupReader::ReadFil // Create RecordReader ::parquet::internal::LevelInfo leaf_info = ::parquet::internal::LevelInfo::ComputeLevelInfo(col_descriptor); - auto record_reader = ::parquet::internal::RecordReader::Make(col_descriptor, leaf_info, pool); + auto record_reader = + ::parquet::internal::RecordReader::Make(col_descriptor, leaf_info, pool.get()); record_reader->SetPageReader(std::move(page_reader)); - // Execute skip/read pattern based on effective RowRanges - const auto& ranges = effective_ranges.GetRanges(); - int64_t current_row = 0; - - for (const auto& range : ranges) { - // Skip rows before this range - if (range.from > current_row) { - int64_t to_skip = range.from - current_row; - int64_t skipped = record_reader->SkipRecords(to_skip); - if (skipped != to_skip) { - return Status::Invalid(fmt::format( - "PageFilteredRowGroupReader: expected to skip {} records but skipped {} " - "(row_group={}, column={})", - to_skip, skipped, row_group_index, column_index)); - } - current_row = range.from; - } - - // Read rows in this range - int64_t to_read = range.Count(); - int64_t read = record_reader->ReadRecords(to_read); - if (read != to_read) { - return Status::Invalid( - fmt::format("PageFilteredRowGroupReader: expected to read {} records but read {} " - "(row_group={}, column={}, range=[{},{}])", - to_read, read, row_group_index, column_index, range.from, range.to)); - } - current_row += to_read; - } + PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern( + record_reader, effective_ranges, effective_row_count, row_group_index, column_index)); - // Skip remaining rows after the last range to properly finalize the reader - if (current_row < effective_row_count) { - record_reader->SkipRecords(effective_row_count - current_row); - } - - // Transfer to Arrow ChunkedArray std::shared_ptr chunked_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(::parquet::arrow::TransferColumnData( - record_reader.get(), field, col_descriptor, pool, &chunked_array)); + record_reader.get(), field, col_descriptor, pool.get(), &chunked_array)); return chunked_array; } -Result> PageFilteredRowGroupReader::ReadFilteredRowGroup( +Status PageFilteredRowGroupReader::WaitForPreBuffer( ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, - const RowRanges& row_ranges, const std::vector& column_indices, - const std::shared_ptr& arrow_schema, ::arrow::MemoryPool* pool, + const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options, + bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, + std::shared_ptr<::arrow::MemoryPool> pool) { + std::vector rg_vec = {row_group_index}; + std::vector col_vec(column_indices.begin(), column_indices.end()); + if (!pre_buffered) { + ::arrow::io::IOContext io_ctx(pool.get()); + parquet_reader->PreBuffer(rg_vec, col_vec, io_ctx, cache_options); + } + if (!page_ranges.empty()) { + auto status = parquet_reader->WhenBufferedRanges(page_ranges).status(); + if (!status.ok()) { + ::arrow::io::IOContext io_ctx(pool.get()); + parquet_reader->PreBuffer(rg_vec, col_vec, io_ctx, cache_options); + PAIMON_RETURN_NOT_OK_FROM_ARROW(parquet_reader->WhenBuffered(rg_vec, col_vec).status()); + } + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(parquet_reader->WhenBuffered(rg_vec, col_vec).status()); + } + return Status::OK(); +} + +Result> PageFilteredRowGroupReader::ReadFilteredRowGroup( + ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, + const std::vector& column_indices, const std::shared_ptr& arrow_schema, const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, - const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize) { + const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, + std::shared_ptr<::arrow::MemoryPool> pool) { + const auto& row_ranges = target_row_group.row_ranges; + int32_t row_group_index = target_row_group.row_group_index; + if (row_ranges.IsEmpty()) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr empty_table, - arrow::Table::MakeEmpty(arrow_schema, pool)); + arrow::Table::MakeEmpty(arrow_schema, pool.get())); return std::make_unique(std::move(empty_table), max_chunksize); } int64_t expected_rows = row_ranges.RowCount(); - // Wait for pre-buffered data to be ready. - // When pre_buffered=true, PreBuffer was already called in PrepareForReading() covering - // all row groups in parallel. We only need to wait. Calling PreBuffer again would create - // a new cached_source_, discarding the parallel I/O already in progress. - { - std::vector rg_vec = {row_group_index}; - std::vector col_vec(column_indices.begin(), column_indices.end()); - if (!pre_buffered) { - ::arrow::io::IOContext io_ctx(pool); - parquet_reader->PreBuffer(rg_vec, col_vec, io_ctx, cache_options); - } - if (!page_ranges.empty()) { - // Page-level PreBuffer: wait on specific page byte ranges - // If pre-buffering failed (e.g., IO error during testing), fall back to on-demand read - auto status = parquet_reader->WhenBufferedRanges(page_ranges).status(); - if (!status.ok()) { - // Pre-buffering failed, fall back to row-group level PreBuffer - ::arrow::io::IOContext io_ctx(pool); - parquet_reader->PreBuffer(rg_vec, col_vec, io_ctx, cache_options); - PAIMON_RETURN_NOT_OK_FROM_ARROW( - parquet_reader->WhenBuffered(rg_vec, col_vec).status()); - } - } else { - PAIMON_RETURN_NOT_OK_FROM_ARROW(parquet_reader->WhenBuffered(rg_vec, col_vec).status()); - } - } + PAIMON_RETURN_NOT_OK(WaitForPreBuffer(parquet_reader, row_group_index, column_indices, + cache_options, pre_buffered, page_ranges, pool)); - // Open row group and page index once, share across all columns auto row_group_reader = parquet_reader->RowGroup(row_group_index); auto rg_metadata = parquet_reader->metadata()->RowGroup(row_group_index); int64_t row_group_row_count = rg_metadata->num_rows(); - auto page_index_reader = parquet_reader->GetPageIndexReader(); // reuse RowGroupPageIndexReader for multiple columns in the same row group to avoid redundant // metadata reads std::shared_ptr<::parquet::RowGroupPageIndexReader> rg_page_index_reader; + auto page_index_reader = parquet_reader->GetPageIndexReader(); if (page_index_reader) { rg_page_index_reader = page_index_reader->RowGroup(row_group_index); } @@ -295,18 +284,16 @@ Result> PageFilteredRowGroupReader::Re columns.push_back(std::move(chunked_array)); } - // Wrap columns in a Table and stream zero-copy-sliced batches via TableBatchReader. - // For multi-chunk variable-length columns this avoids the deep copy of CombineChunks: - // each emitted batch contains at most max_chunksize rows (capped further by the - // smallest remaining chunk across columns), and every column's Array is a zero-copy - // Slice of its underlying chunk. auto table = arrow::Table::Make(arrow_schema, std::move(columns), expected_rows); return std::make_unique(std::move(table), max_chunksize); } std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRanges( - ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, - const RowRanges& row_ranges, const std::vector& column_indices) { + ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, + const std::vector& column_indices) { + int32_t row_group_index = target_row_group.row_group_index; + const auto& row_ranges = target_row_group.row_ranges; + std::vector<::arrow::io::ReadRange> ranges; auto file_metadata = parquet_reader->metadata(); auto rg_metadata = file_metadata->RowGroup(row_group_index); @@ -351,23 +338,17 @@ std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRange auto num_pages = static_cast(page_locations.size()); for (int32_t page_idx = 0; page_idx < num_pages; ++page_idx) { - int64_t first_row = page_locations[page_idx].first_row_index; - int64_t last_row = (page_idx + 1 < num_pages) - ? page_locations[page_idx + 1].first_row_index - 1 - : row_group_row_count - 1; + auto [first_row, last_row] = + GetPageRowRange(page_locations, page_idx, row_group_row_count); if (!row_ranges.IsOverlapping(first_row, last_row)) { - continue; // Page doesn't overlap with target rows + continue; } - // Compute page byte range int64_t page_offset = page_locations[page_idx].offset; - int64_t page_size; - if (page_idx + 1 < num_pages) { - page_size = page_locations[page_idx + 1].offset - page_offset; - } else { - page_size = chunk_end - page_offset; - } + int64_t page_size = (page_idx + 1 < num_pages) + ? page_locations[page_idx + 1].offset - page_offset + : chunk_end - page_offset; ranges.push_back({page_offset, page_size}); } } diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index f2a06c50..30c9746a 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -48,8 +48,7 @@ class PageFilteredRowGroupReader { /// Read a row group with page-level filtering. /// @param parquet_reader The underlying ParquetFileReader - /// @param row_group_index Row group to read - /// @param row_ranges Matching row ranges within this row group + /// @param target_row_group Target row group with index and row ranges /// @param column_indices Leaf column indices to read /// @param arrow_schema The target Arrow schema for output columns /// @param pool Memory pool @@ -57,50 +56,59 @@ class PageFilteredRowGroupReader { /// @param pre_buffered If true, assumes PreBuffer was already called externally /// and only waits via WhenBuffered (no redundant PreBuffer). /// @param page_ranges If non-empty, wait via WhenBufferedRanges instead of WhenBuffered - /// @param max_chunksize Per-batch row cap for the returned reader, mirroring Arrow's - /// TableBatchReader::set_chunksize. Each batch yields at most this many rows; - /// actual size may be smaller when an underlying ChunkedArray's chunk boundary - /// is reached first (zero-copy slice). - /// @return A RecordBatchReader streaming the filtered rows. Multi-chunk variable-length - /// columns are emitted as multiple zero-copy-sliced batches along chunk boundaries - /// instead of being concatenated, avoiding the deep copy of CombineChunks. + /// @param max_chunksize Per-batch row cap for the returned reader. + /// @return A RecordBatchReader streaming the filtered rows. static Result> ReadFilteredRowGroup( - ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, - const RowRanges& row_ranges, const std::vector& column_indices, - const std::shared_ptr& arrow_schema, ::arrow::MemoryPool* pool, - const ::arrow::io::CacheOptions& cache_options = ::arrow::io::CacheOptions::Defaults(), - bool pre_buffered = false, const std::vector<::arrow::io::ReadRange>& page_ranges = {}, - int64_t max_chunksize = std::numeric_limits::max()); + ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, + const std::vector& column_indices, + const std::shared_ptr& arrow_schema, + const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, + std::shared_ptr<::arrow::MemoryPool> pool); /// Compute the byte ranges of pages that overlap with the given RowRanges. /// Uses OffsetIndex to determine per-page file offsets and sizes. /// Includes dictionary pages unconditionally. /// Falls back to entire column chunk range if OffsetIndex is unavailable. static std::vector<::arrow::io::ReadRange> ComputePageRanges( - ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index, - const RowRanges& row_ranges, const std::vector& column_indices); + ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, + const std::vector& column_indices); private: + /// Get the [first_row, last_row] range of a page given page locations. + static std::pair GetPageRowRange( + const std::vector<::parquet::PageLocation>& page_locations, int32_t page_idx, + int64_t row_group_row_count); + + /// Wait for pre-buffered data to become available before reading. + static Status WaitForPreBuffer(::parquet::ParquetFileReader* parquet_reader, + int32_t row_group_index, + const std::vector& column_indices, + const ::arrow::io::CacheOptions& cache_options, + bool pre_buffered, + const std::vector<::arrow::io::ReadRange>& page_ranges, + std::shared_ptr<::arrow::MemoryPool> pool); + + /// Execute the skip/read pattern on a RecordReader based on RowRanges. + static Status ExecuteSkipReadPattern( + std::shared_ptr<::parquet::internal::RecordReader> record_reader, const RowRanges& ranges, + int64_t total_row_count, int32_t row_group_index, int32_t column_index); + /// Create a data_page_filter callback for a column based on RowRanges + OffsetIndex. - /// Returns true (skip) if the page's row range has no overlap with RowRanges. static std::function MakePageFilter( const RowRanges& row_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); /// Read a single column using skip/read pattern driven by RowRanges. - /// When OffsetIndex is available, uses data_page_filter for I/O-level page skipping - /// and compressed RowRanges for decode-level row skipping. static Result> ReadFilteredColumn( const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader, ::parquet::ParquetFileReader* parquet_reader, const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader, int32_t row_group_index, int32_t column_index, const RowRanges& row_ranges, const std::shared_ptr& field, int64_t row_group_row_count, - ::arrow::MemoryPool* pool); + std::shared_ptr<::arrow::MemoryPool> pool); /// Compute compressed RowRanges after data_page_filter skips non-matching pages. - /// Maps original RowRanges to the compressed row space where skipped pages are removed. - /// @return pair of (compressed RowRanges, compressed total row count) static std::pair ComputeCompressedRowRanges( const RowRanges& original_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 041839ba..e9aff95e 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -529,7 +529,9 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesPartialMatch) { row_ranges.Add(RowRanges::Range(50, 59)); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - parquet_reader.get(), /*row_group_index=*/0, row_ranges, /*column_indices=*/{0}); + parquet_reader.get(), + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), + /*column_indices=*/{0}); // Should have exactly 1 range (page 5 of column 0, no dictionary since disabled) ASSERT_EQ(1, ranges.size()); @@ -552,8 +554,9 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesAllMatch) { RowRanges row_ranges; row_ranges.Add(RowRanges::Range(0, 99)); - auto ranges = - PageFilteredRowGroupReader::ComputePageRanges(parquet_reader.get(), 0, row_ranges, {0}); + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + parquet_reader.get(), + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}); // 10 pages, all matching ASSERT_EQ(10, ranges.size()); @@ -576,8 +579,9 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesNoMatch) { RowRanges row_ranges; // empty - auto ranges = - PageFilteredRowGroupReader::ComputePageRanges(parquet_reader.get(), 0, row_ranges, {0}); + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + parquet_reader.get(), + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}); ASSERT_EQ(0, ranges.size()); } @@ -597,8 +601,10 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiColumn) { RowRanges row_ranges; row_ranges.Add(RowRanges::Range(50, 59)); - auto ranges = - PageFilteredRowGroupReader::ComputePageRanges(parquet_reader.get(), 0, row_ranges, {0, 1}); + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + parquet_reader.get(), + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), + {0, 1}); // 1 matching page per column = 2 ranges total ASSERT_EQ(2, ranges.size()); @@ -623,8 +629,9 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) { row_ranges.Add(RowRanges::Range(20, 29)); row_ranges.Add(RowRanges::Range(70, 79)); - auto ranges = - PageFilteredRowGroupReader::ComputePageRanges(parquet_reader.get(), 0, row_ranges, {0}); + auto ranges = PageFilteredRowGroupReader::ComputePageRanges( + parquet_reader.get(), + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0}); // 2 matching pages for 1 column ASSERT_EQ(2, ranges.size()); @@ -792,7 +799,9 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding) row_ranges.Add(RowRanges::Range(0, 99)); auto ranges = PageFilteredRowGroupReader::ComputePageRanges( - parquet_reader.get(), /*row_group_index=*/0, row_ranges, /*column_indices=*/{0}); + parquet_reader.get(), + TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, /*ranges=*/row_ranges), + /*column_indices=*/{0}); ASSERT_FALSE(ranges.empty()); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index e64f481a..cd14b837 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -67,7 +67,6 @@ ParquetFileBatchReader::ParquetFileBatchReader( arrow_pool_(arrow_pool), input_stream_(std::move(input_stream)), reader_(std::move(reader)), - read_ranges_(reader_->GetAllRowGroupRanges()), metrics_(std::make_shared()), logger_(Logger::GetLogger("ParquetFileBatchReader")) {} @@ -91,8 +90,8 @@ Result> ParquetFileBatchReader::Create( ->properties(arrow_reader_properties) ->Build(&file_reader)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, - FileReaderWrapper::Create(std::move(file_reader), pool.get(), - static_cast(batch_size))); + FileReaderWrapper::Create(std::move(file_reader), + static_cast(batch_size), pool)); auto parquet_file_batch_reader = std::unique_ptr( new ParquetFileBatchReader(std::move(input_stream), std::move(reader), options, pool)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, @@ -161,6 +160,9 @@ Status ParquetFileBatchReader::SetReadSchema( } // Apply page-level filtering after bitmap pruning so we don't read page index // pages for row groups that the bitmap already excluded. + // If no predicate is provided, skip page-level filtering, row_group_row_ranges will be + // empty + std::map row_group_row_ranges; if (predicate && !row_groups.empty()) { PAIMON_ASSIGN_OR_RAISE( bool enable_page_index_filter, @@ -179,38 +181,38 @@ Status ParquetFileBatchReader::SetReadSchema( } } + std::pair, std::map> page_filter_result; PAIMON_ASSIGN_OR_RAISE( - auto page_filter_result, + page_filter_result, FilterRowGroupsByPageIndex(predicate, column_name_to_index, row_groups)); row_groups = std::move(page_filter_result.first); - reader_->SetRowGroupRowRanges(page_filter_result.second); + row_group_row_ranges = std::move(page_filter_result.second); } } read_data_type_ = arrow::struct_(read_schema->fields()); - read_row_groups_ = row_groups; - read_column_indices_ = column_indices; metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_TOTAL, reader_->GetNumberOfRowGroups()); metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER, row_groups.size()); - PAIMON_ASSIGN_OR_RAISE( - std::set ordered_row_groups, - reader_->FilterRowGroupsByReadRanges(read_ranges_, read_row_groups_)); - - // When predicate or selection is applied, prepare eagerly so PreBuffer I/O - // starts immediately. All file readers are created before consumption begins, - // so eager preparation allows I/O for multiple files to overlap. - Status ret; - if (predicate || selection_bitmap) { - ret = reader_->PrepareForReading(ordered_row_groups, read_column_indices_); - } else { - ret = reader_->PrepareForReadingLazy(ordered_row_groups, read_column_indices_); + // Build TargetRowGroup list with page-filter info in one shot. + std::vector target_row_groups; + for (int32_t rg_id : row_groups) { + auto it = row_group_row_ranges.find(rg_id); + if (it != row_group_row_ranges.end()) { + target_row_groups.emplace_back(/*rg_index=*/rg_id, /*page_filtered=*/true, + /*ranges=*/it->second); + } else { + target_row_groups.emplace_back(/*rg_index=*/rg_id, + /*page_filtered=*/false, + /*ranges=*/RowRanges()); + } } - return ret; + PAIMON_RETURN_NOT_OK(reader_->PrepareForReadingLazy(target_row_groups, column_indices)); } PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::SetReadSchema") + return Status::OK(); } Result> ParquetFileBatchReader::FilterRowGroupsByPredicate( diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index d9dfe91a..393bc385 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -104,11 +104,7 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { } Status SetReadRanges(const std::vector>& read_ranges) override { - read_ranges_ = read_ranges; - PAIMON_ASSIGN_OR_RAISE( - std::set ordered_row_groups, - reader_->FilterRowGroupsByReadRanges(read_ranges_, read_row_groups_)); - return reader_->PrepareForReadingLazy(ordered_row_groups, read_column_indices_); + return reader_->ApplyReadRanges(read_ranges); } std::shared_ptr GetReaderMetrics() const override { @@ -181,17 +177,12 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { std::unique_ptr reader_; std::shared_ptr read_data_type_; - std::vector> read_ranges_; std::shared_ptr metrics_; std::unique_ptr logger_; uint64_t read_rows_ = 0; uint64_t read_batch_count_ = 0; - - // last time set read schema - std::vector read_row_groups_; - std::vector read_column_indices_; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/row_ranges.h b/src/paimon/format/parquet/row_ranges.h index 2f49f4f4..05edec20 100644 --- a/src/paimon/format/parquet/row_ranges.h +++ b/src/paimon/format/parquet/row_ranges.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "paimon/utils/range.h" @@ -108,4 +109,20 @@ class RowRanges { std::vector ranges_; }; +struct TargetRowGroup { + int32_t row_group_index{-1}; + bool is_partially_matched{false}; + // page-filtered row ranges, only valid if is_partially_matched is true. + RowRanges row_ranges; + // Whether this row group has been excluded by ApplyReadRanges. + // When true, this row group is logically skipped during iteration + // but retained so that a subsequent wider ApplyReadRanges can restore it. + bool excluded_by_read_range{false}; + + TargetRowGroup() = default; + TargetRowGroup(int32_t rg_index, bool is_partially_matched, RowRanges ranges) + : row_group_index(rg_index), + is_partially_matched(is_partially_matched), + row_ranges(std::move(ranges)) {} +}; } // namespace paimon::parquet From fb3aa41466db7411ae95571fc98aa83ed6c07a63 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Fri, 12 Jun 2026 08:50:48 +0800 Subject: [PATCH 050/138] fix(scan): fix dangling reference and unsigned overflow in FileStoreScan --- src/paimon/core/operation/file_store_scan.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index b5275eed..6cdffe36 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -171,8 +171,9 @@ Result> FileStoreScan::CreatePlan() cons metrics_->SetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID, snapshot.has_value() ? snapshot.value().Id() : int64_t{0}); metrics_->SetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS, filtered_manifest_file_metas.size()); - metrics_->SetCounter(ScanMetrics::LAST_SCAN_SKIPPED_TABLE_FILES, - all_data_files - manifest_entries.size()); + metrics_->SetCounter( + ScanMetrics::LAST_SCAN_SKIPPED_TABLE_FILES, + std::max(int64_t{0}, all_data_files - static_cast(manifest_entries.size()))); metrics_->SetCounter(ScanMetrics::LAST_SCAN_RESULTED_TABLE_FILES, manifest_entries.size()); return std::make_shared(scan_mode_, snapshot, std::move(manifest_entries)); @@ -221,7 +222,7 @@ Status FileStoreScan::ReadFileEntries(const std::vector& manif std::vector* manifest_entries) const { std::vector>>> futures; for (const auto& meta : manifest_metas) { - auto read_meta_task = [this, &meta]() -> Result> { + auto read_meta_task = [this, meta]() -> Result> { std::vector tmp_entries; PAIMON_RETURN_NOT_OK(ReadManifestFileMeta(meta, &tmp_entries)); return tmp_entries; From 50768c1cbacb671d180bea422e5982e3eb28ae44 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Fri, 12 Jun 2026 13:01:05 +0800 Subject: [PATCH 051/138] feat(shredding): Add infrastructure for shared-shredding MAP storage layout --- include/paimon/defs.h | 15 + src/paimon/CMakeLists.txt | 2 + .../shredding/map_shared_shredding_utils.cpp | 394 ++++++++++++++++++ .../shredding/map_shared_shredding_utils.h | 115 +++++ .../map_shared_shredding_utils_test.cpp | 368 ++++++++++++++++ .../data/shredding/map_shredding_defs.h | 97 +++++ src/paimon/common/defs.cpp | 2 + src/paimon/core/core_options.cpp | 27 ++ src/paimon/core/core_options.h | 5 + src/paimon/core/core_options_test.cpp | 58 ++- src/paimon/core/options/map_storage_layout.h | 30 ++ src/paimon/core/schema/schema_validation.cpp | 63 +++ src/paimon/core/schema/schema_validation.h | 2 + .../core/schema/schema_validation_test.cpp | 118 ++++++ 14 files changed, 1295 insertions(+), 1 deletion(-) create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_utils.cpp create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_utils.h create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp create mode 100644 src/paimon/common/data/shredding/map_shredding_defs.h create mode 100644 src/paimon/core/options/map_storage_layout.h diff --git a/include/paimon/defs.h b/include/paimon/defs.h index b0266e43..91910ee5 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -369,6 +369,21 @@ struct PAIMON_EXPORT Options { /// "partition.legacy-name" - The legacy partition name is using `ToString` for all types. If /// false, using casting to string for all types. Default value is "true". static const char PARTITION_GENERATE_LEGACY_NAME[]; + /// "map.storage-layout" - Suffix for per-column MAP storage layout configuration. + /// Used as `fields..map.storage-layout`. Values: "default" (standard KV arrays) + /// or "shared-shredding" (columnar shredding with column reuse). Default is "default". + /// If set "shared-shredding", the column must be of type MAP. Each column must be + /// configured individually. For example, to enable shared-shredding layout for two columns + /// "metrics" and "tags": + /// fields.metrics.map.storage-layout = shared-shredding + /// fields.tags.map.storage-layout = shared-shredding + static const char MAP_STORAGE_LAYOUT[]; + /// "map.shared-shredding.max-columns" - Suffix for per-column upper bound K_max configuration. + /// Used as `fields..map.shared-shredding.max-columns`. Only effective when + /// map.storage-layout = shared-shredding. Rows with more fields than K_max spill to + /// __overflow. Default value is 256. Each column can have its own max-columns setting. + static const char MAP_SHARED_SHREDDING_MAX_COLUMNS[]; + /// "blob-as-descriptor" - Read blob field using blob descriptor rather than blob /// bytes. Default value is "false". static const char BLOB_AS_DESCRIPTOR[]; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index b6a0e73f..b1edfdb6 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -139,6 +139,7 @@ set(PAIMON_COMMON_SRCS common/utils/bloom_filter64.cpp common/utils/crc32c.cpp common/utils/decimal_utils.cpp + common/data/shredding/map_shared_shredding_utils.cpp common/utils/delta_varint_compressor.cpp common/utils/fields_comparator.cpp common/utils/path_util.cpp @@ -533,6 +534,7 @@ if(PAIMON_BUILD_TESTS) common/utils/decimal_utils_test.cpp common/utils/threadsafe_queue_test.cpp common/utils/generic_lru_cache_test.cpp + common/data/shredding/map_shared_shredding_utils_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp new file mode 100644 index 00000000..6725cb8b --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp @@ -0,0 +1,394 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" + +#include +#include + +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "fmt/format.h" +#include "paimon/common/compression/block_compression_factory.h" +#include "paimon/common/compression/block_compressor.h" +#include "paimon/common/compression/block_decompressor.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/options/map_storage_layout.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace paimon { +// ---- Column detection ---- + +bool MapSharedShreddingUtils::IsShreddingKeyMap( + const std::shared_ptr& arrow_type) { + if (arrow_type->id() != arrow::Type::MAP) { + return false; + } + auto map_type = std::static_pointer_cast(arrow_type); + return map_type->key_type()->id() == arrow::Type::STRING; +} + +Result> MapSharedShreddingUtils::DetectShreddingColumns( + const std::shared_ptr& schema, const CoreOptions& options) { + std::vector indices; + for (int32_t i = 0; i < schema->num_fields(); ++i) { + const auto& field = schema->field(i); + if (!IsShreddingKeyMap(field->type())) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(MapStorageLayout layout, options.GetMapStorageLayout(field->name())); + if (layout == MapStorageLayout::SHARED_SHREDDING) { + indices.push_back(i); + } + } + return indices; +} + +// ---- Schema conversion ---- + +std::shared_ptr MapSharedShreddingUtils::BuildPhysicalStructType( + const std::shared_ptr& value_type, int32_t num_columns, bool value_nullable) { + arrow::FieldVector struct_fields; + struct_fields.reserve(num_columns + 2); + + struct_fields.push_back( + arrow::field(MapSharedShreddingDefine::kFieldMapping, arrow::list(arrow::int32()), false)); + + for (int32_t i = 0; i < num_columns; ++i) { + struct_fields.push_back(arrow::field(MapSharedShreddingDefine::PhysicalColumnName(i), + value_type, value_nullable)); + } + + struct_fields.push_back(arrow::field( + MapSharedShreddingDefine::kOverflow, + arrow::map(arrow::int32(), arrow::field("value", value_type, value_nullable)), true)); + + return arrow::struct_(std::move(struct_fields)); +} + +Result> MapSharedShreddingUtils::LogicalToPhysicalSchema( + const std::shared_ptr& logical_schema, + const std::map& column_to_num_columns) { + arrow::FieldVector physical_fields; + physical_fields.reserve(logical_schema->num_fields()); + + for (int32_t i = 0; i < logical_schema->num_fields(); ++i) { + const auto& field = logical_schema->field(i); + auto it = column_to_num_columns.find(i); + if (it != column_to_num_columns.end()) { + auto map_type = std::static_pointer_cast(field->type()); + auto value_type = map_type->item_type(); + bool value_nullable = map_type->item_field()->nullable(); + auto physical_type = BuildPhysicalStructType(value_type, it->second, value_nullable); + physical_fields.push_back( + arrow::field(field->name(), physical_type, field->nullable())); + } else { + physical_fields.push_back(field); + } + } + + return arrow::schema(std::move(physical_fields)); +} + +Result> MapSharedShreddingUtils::BuildColumnToNumColumns( + const std::vector& shredding_column_indices, + const std::shared_ptr& schema, const CoreOptions& options) { + std::map column_to_num_columns; + for (int32_t col_index : shredding_column_indices) { + const std::string& field_name = schema->field(col_index)->name(); + PAIMON_ASSIGN_OR_RAISE(int32_t max_columns, + options.GetMapSharedShreddingMaxColumns(field_name)); + column_to_num_columns[col_index] = max_columns; + } + return column_to_num_columns; +} + +// ---- Metadata serialization helpers ---- + +namespace { + +std::string JsonEncodeObject( + std::function builder) { + rapidjson::Document doc(rapidjson::kObjectType); + auto allocator = doc.GetAllocator(); + builder(&doc, &allocator); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc.Accept(writer); + return buffer.GetString(); +} + +std::string JsonEncodeArray( + std::function builder) { + rapidjson::Document doc(rapidjson::kArrayType); + auto allocator = doc.GetAllocator(); + builder(&doc, &allocator); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc.Accept(writer); + return buffer.GetString(); +} + +Result CompressString(const std::string& input, const std::string& compression) { + CompressOptions compress_opts{compression, /*zstd_level=*/1}; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr factory, + BlockCompressionFactory::Create(compress_opts)); + std::shared_ptr compressor = factory->GetCompressor(); + if (!compressor) { + return input; + } + + auto src_size = static_cast(input.size()); + int32_t max_compressed = compressor->GetMaxCompressedSize(src_size); + std::string output(max_compressed, '\0'); + + PAIMON_ASSIGN_OR_RAISE( + int32_t actual_size, + compressor->Compress(input.data(), src_size, output.data(), max_compressed)); + + output.resize(actual_size); + return output; +} + +Result DecompressString(const std::string& input, int32_t original_len, + const std::string& compression) { + CompressOptions compress_opts{compression, /*zstd_level=*/1}; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr factory, + BlockCompressionFactory::Create(compress_opts)); + std::shared_ptr decompressor = factory->GetDecompressor(); + if (!decompressor) { + return input; + } + std::string output(original_len, '\0'); + PAIMON_ASSIGN_OR_RAISE( + int32_t decompressed_len, + decompressor->Decompress(input.data(), static_cast(input.size()), output.data(), + original_len)); + output.resize(decompressed_len); + return output; +} + +Result GetRequiredValue(const std::shared_ptr& metadata, + const char* key) { + int32_t index = metadata->FindKey(key); + if (index < 0) { + return Status::Invalid(fmt::format("missing shredding metadata key: {}", key)); + } + return metadata->value(index); +} + +Result GetRequiredInt32(const std::shared_ptr& metadata, + const char* key) { + PAIMON_ASSIGN_OR_RAISE(std::string value, GetRequiredValue(metadata, key)); + std::optional parsed = StringUtils::StringToValue(value); + if (!parsed.has_value()) { + return Status::Invalid(fmt::format("malformed shredding metadata value for key: {}", key)); + } + return parsed.value(); +} + +std::string SerializeFieldDict(const MapSharedShreddingFieldMeta& field_meta) { + return JsonEncodeObject([&](rapidjson::Document* doc, + rapidjson::Document::AllocatorType* alloc) { + for (const auto& [name, id] : field_meta.name_to_id) { + doc->AddMember(rapidjson::Value(name.c_str(), *alloc), rapidjson::Value(id), *alloc); + } + }); +} + +std::string SerializeFieldColumns(const MapSharedShreddingFieldMeta& field_meta) { + return JsonEncodeObject( + [&](rapidjson::Document* doc, rapidjson::Document::AllocatorType* alloc) { + for (const auto& [field_id, col_vec] : field_meta.field_to_columns) { + rapidjson::Value array(rapidjson::kArrayType); + std::vector sorted_cols(col_vec.begin(), col_vec.end()); + std::sort(sorted_cols.begin(), sorted_cols.end()); + for (int32_t col : sorted_cols) { + array.PushBack(col, *alloc); + } + std::string key = std::to_string(field_id); + doc->AddMember(rapidjson::Value(key.c_str(), *alloc), array, *alloc); + } + }); +} + +std::string SerializeOverflowSet(const MapSharedShreddingFieldMeta& field_meta) { + return JsonEncodeArray( + [&](rapidjson::Document* doc, rapidjson::Document::AllocatorType* alloc) { + std::vector sorted(field_meta.overflow_field_set.begin(), + field_meta.overflow_field_set.end()); + std::sort(sorted.begin(), sorted.end()); + for (int32_t field_id : sorted) { + doc->PushBack(field_id, *alloc); + } + }); +} + +/// Safe JSON integer extraction with error propagation. +Result JsonGetInt(const rapidjson::Value& val, const char* context_msg) { + if (!val.IsInt()) { + return Status::Invalid(fmt::format("malformed shredding metadata: {}", context_msg)); + } + return val.GetInt(); +} + +Result> DeserializeFieldDict(const std::string& json_str) { + rapidjson::Document doc; + doc.Parse(json_str.c_str()); + if (doc.HasParseError() || !doc.IsObject()) { + return Status::Invalid("malformed shredding field_dict metadata"); + } + std::map name_to_id; + for (auto it = doc.MemberBegin(); it != doc.MemberEnd(); ++it) { + PAIMON_ASSIGN_OR_RAISE(int32_t id, JsonGetInt(it->value, "field_dict value is not int")); + name_to_id[it->name.GetString()] = id; + } + return name_to_id; +} + +Result>> DeserializeFieldColumns( + const std::string& json_str) { + rapidjson::Document doc; + doc.Parse(json_str.c_str()); + if (doc.HasParseError() || !doc.IsObject()) { + return Status::Invalid("malformed shredding field_columns metadata"); + } + std::map> field_to_columns; + for (auto it = doc.MemberBegin(); it != doc.MemberEnd(); ++it) { + std::optional field_id = StringUtils::StringToValue(it->name.GetString()); + if (!field_id.has_value()) { + return Status::Invalid("malformed shredding field_columns: invalid field_id key"); + } + const auto& array = it->value; + if (!array.IsArray()) { + return Status::Invalid("malformed shredding field_columns: value is not array"); + } + std::vector cols; + cols.reserve(array.Size()); + for (rapidjson::SizeType i = 0; i < array.Size(); ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t col, + JsonGetInt(array[i], "field_columns element is not int")); + cols.push_back(col); + } + field_to_columns[field_id.value()] = std::move(cols); + } + return field_to_columns; +} + +Result> DeserializeOverflowSet(const std::string& json_str) { + rapidjson::Document doc; + doc.Parse(json_str.c_str()); + if (doc.HasParseError() || !doc.IsArray()) { + return Status::Invalid("malformed shredding overflow_set metadata"); + } + std::set overflow_set; + for (rapidjson::SizeType i = 0; i < doc.Size(); ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t field_id, + JsonGetInt(doc[i], "overflow_set element is not int")); + overflow_set.insert(field_id); + } + return overflow_set; +} + +} // namespace + +Status MapSharedShreddingUtils::SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta, + const std::string& compression, + arrow::KeyValueMetadata* metadata) { + metadata->Append(MapShreddingDefine::kStorageLayout, + MapShreddingDefine::kStorageLayoutSharedShredding); + metadata->Append(MapSharedShreddingDefine::kVersion, + std::to_string(MapSharedShreddingDefine::kCurrentVersion)); + + std::string field_dict_json = SerializeFieldDict(field_meta); + metadata->Append(MapSharedShreddingDefine::kFieldDictOriginalSize, + std::to_string(field_dict_json.size())); + PAIMON_ASSIGN_OR_RAISE(std::string compressed_dict, + CompressString(field_dict_json, compression)); + metadata->Append(MapSharedShreddingDefine::kFieldDict, std::move(compressed_dict)); + + metadata->Append(MapSharedShreddingDefine::kFieldColumns, SerializeFieldColumns(field_meta)); + metadata->Append(MapSharedShreddingDefine::kOverflowSet, SerializeOverflowSet(field_meta)); + metadata->Append(MapSharedShreddingDefine::kNumColumns, std::to_string(field_meta.num_columns)); + metadata->Append(MapSharedShreddingDefine::kMaxRowWidth, + std::to_string(field_meta.max_row_width)); + + return Status::OK(); +} + +Result MapSharedShreddingUtils::DeserializeMetadata( + const std::shared_ptr& metadata, const std::string& compression) { + if (!HasShreddingMetadata(metadata)) { + return Status::Invalid("metadata is null or storage layout is not shared-shredding"); + } + PAIMON_ASSIGN_OR_RAISE(int32_t version, + GetRequiredInt32(metadata, MapSharedShreddingDefine::kVersion)); + if (version != MapSharedShreddingDefine::kCurrentVersion) { + return Status::Invalid( + fmt::format("unsupported shared-shredding metadata version: {}, expected: {}", version, + MapSharedShreddingDefine::kCurrentVersion)); + } + + MapSharedShreddingFieldMeta result; + + // field_dict (compressed) + PAIMON_ASSIGN_OR_RAISE( + int32_t original_len, + GetRequiredInt32(metadata, MapSharedShreddingDefine::kFieldDictOriginalSize)); + PAIMON_ASSIGN_OR_RAISE(std::string compressed_dict, + GetRequiredValue(metadata, MapSharedShreddingDefine::kFieldDict)); + PAIMON_ASSIGN_OR_RAISE(std::string field_dict_json, + DecompressString(compressed_dict, original_len, compression)); + PAIMON_ASSIGN_OR_RAISE(result.name_to_id, DeserializeFieldDict(field_dict_json)); + + // field_columns + PAIMON_ASSIGN_OR_RAISE(std::string field_columns_json, + GetRequiredValue(metadata, MapSharedShreddingDefine::kFieldColumns)); + PAIMON_ASSIGN_OR_RAISE(result.field_to_columns, DeserializeFieldColumns(field_columns_json)); + + // overflow_set + PAIMON_ASSIGN_OR_RAISE(std::string overflow_json, + GetRequiredValue(metadata, MapSharedShreddingDefine::kOverflowSet)); + PAIMON_ASSIGN_OR_RAISE(result.overflow_field_set, DeserializeOverflowSet(overflow_json)); + + // num_columns & max_row_width + PAIMON_ASSIGN_OR_RAISE(result.num_columns, + GetRequiredInt32(metadata, MapSharedShreddingDefine::kNumColumns)); + PAIMON_ASSIGN_OR_RAISE(result.max_row_width, + GetRequiredInt32(metadata, MapSharedShreddingDefine::kMaxRowWidth)); + + return result; +} + +bool MapSharedShreddingUtils::HasShreddingMetadata( + const std::shared_ptr& metadata) { + if (!metadata) { + return false; + } + auto index = metadata->FindKey(MapShreddingDefine::kStorageLayout); + if (index < 0) { + return false; + } + return metadata->value(index) == MapShreddingDefine::kStorageLayoutSharedShredding; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.h b/src/paimon/common/data/shredding/map_shared_shredding_utils.h new file mode 100644 index 00000000..30e2a857 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.h @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace arrow { +class KeyValueMetadata; +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; + +/// Utility functions for shared-shredding MAP storage layout. +class MapSharedShreddingUtils { + public: + MapSharedShreddingUtils() = delete; + ~MapSharedShreddingUtils() = delete; + + // ---- Column detection ---- + + /// Checks whether a given arrow field is MAP (the type prerequisite for shredding). + /// @param arrow_type The Arrow data type of the column. + /// @return true if the type is MAP. + static bool IsShreddingKeyMap(const std::shared_ptr& arrow_type); + + /// Finds all shredding MAP column indices in a schema by checking per-column config + /// via CoreOptions. + /// @param schema The logical Arrow schema. + /// @param options CoreOptions containing per-column configuration. + /// @return Vector of column indices whose map.storage-layout is "shared-shredding", or error + /// if validation fails. + static Result> DetectShreddingColumns( + const std::shared_ptr& schema, const CoreOptions& options); + + // ---- Schema conversion ---- + + /// Converts a logical schema to a physical schema by replacing shredding MAP columns + /// with their physical Struct representation. + /// @param logical_schema The original schema with MAP columns. + /// @param column_to_num_columns Map from column index to its physical column count K. + /// Each shredding column can have its own width. + /// @return The physical schema for file writing. + static Result> LogicalToPhysicalSchema( + const std::shared_ptr& logical_schema, + const std::map& column_to_num_columns); + + /// Builds column_to_num_columns map from DetectShreddingColumns result and CoreOptions. + /// @param shredding_column_indices Indices returned by DetectShreddingColumns. + /// @param schema The logical Arrow schema (used to get field names). + /// @param options CoreOptions containing per-column max-columns config. + /// @return Map from column index to K (max physical columns for that column). + static Result> BuildColumnToNumColumns( + const std::vector& shredding_column_indices, + const std::shared_ptr& schema, const CoreOptions& options); + + // ---- Metadata serialization ---- + + /// Serializes shredding metadata and appends entries to an existing KeyValueMetadata. + /// @param field_meta The field-level shredding metadata to serialize. + /// @param compression Compression codec name for field_dict compression. + /// @param[out] metadata The KeyValueMetadata to append entries to. + static Status SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta, + const std::string& compression, + arrow::KeyValueMetadata* metadata); + + /// Deserializes shredding metadata from file footer KeyValueMetadata (per field). + /// @param metadata The KeyValueMetadata from file footer. + /// @param compression Compression codec name. + /// @return Parsed MapSharedShreddingFieldMeta, or error if metadata is missing/malformed. + static Result DeserializeMetadata( + const std::shared_ptr& metadata, const std::string& compression); + + /// Checks whether a KeyValueMetadata contains shredding MAP metadata. + static bool HasShreddingMetadata(const std::shared_ptr& metadata); + + private: + /// Builds the physical Arrow type for one shredding MAP column. + /// @param value_type The value type of the original MAP. + /// @param num_columns Number of physical columns K. + /// @param value_nullable Whether the MAP's value field is nullable. + static std::shared_ptr BuildPhysicalStructType( + const std::shared_ptr& value_type, int32_t num_columns, + bool value_nullable); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp new file mode 100644 index 00000000..87a0f089 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" + +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "gtest/gtest.h" +#include "paimon/core/core_options.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +// ---- IsShreddingKeyMap ---- + +TEST(MapSharedShreddingUtilsTest, IsShreddingKeyMap) { + ASSERT_TRUE( + MapSharedShreddingUtils::IsShreddingKeyMap(arrow::map(arrow::utf8(), arrow::int32()))); + ASSERT_TRUE( + MapSharedShreddingUtils::IsShreddingKeyMap(arrow::map(arrow::utf8(), arrow::float64()))); + // Nested value type (struct) + auto nested_value = + arrow::struct_({arrow::field("x", arrow::int32()), arrow::field("y", arrow::utf8())}); + ASSERT_TRUE( + MapSharedShreddingUtils::IsShreddingKeyMap(arrow::map(arrow::utf8(), nested_value))); + ASSERT_FALSE( + MapSharedShreddingUtils::IsShreddingKeyMap(arrow::map(arrow::int32(), arrow::utf8()))); + ASSERT_FALSE(MapSharedShreddingUtils::IsShreddingKeyMap(arrow::int32())); + ASSERT_FALSE(MapSharedShreddingUtils::IsShreddingKeyMap(arrow::list(arrow::utf8()))); +} + +// ---- DetectShreddingColumns ---- + +TEST(MapSharedShreddingUtilsTest, DetectShreddingColumnsBasic) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + arrow::field("metrics", arrow::map(arrow::utf8(), arrow::float64())), + arrow::field("name", arrow::utf8()), + }); + + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}})); + + ASSERT_OK_AND_ASSIGN(auto indices, + MapSharedShreddingUtils::DetectShreddingColumns(schema, options)); + ASSERT_EQ(indices.size(), 2); + ASSERT_EQ(indices[0], 1); + ASSERT_EQ(indices[1], 2); +} + +TEST(MapSharedShreddingUtilsTest, DetectShreddingColumnsNoShredding) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + }); + + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + ASSERT_OK_AND_ASSIGN(auto indices, + MapSharedShreddingUtils::DetectShreddingColumns(schema, options)); + ASSERT_TRUE(indices.empty()); +} + +// ---- LogicalToPhysicalSchema ---- + +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaBasic) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + arrow::field("name", arrow::utf8()), + }); + + std::map column_to_num_columns = {{1, 4}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + schema, column_to_num_columns)); + + // Build expected schema for comparison + auto expected_struct = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), false), + arrow::field("__col_0", arrow::utf8(), true), + arrow::field("__col_1", arrow::utf8(), true), + arrow::field("__col_2", arrow::utf8(), true), + arrow::field("__col_3", arrow::utf8(), true), + arrow::field("__overflow", arrow::map(arrow::int32(), arrow::utf8()), true), + }); + auto expected_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", expected_struct, true), + arrow::field("name", arrow::utf8()), + }); + ASSERT_TRUE(physical_schema->Equals(expected_schema)); +} + +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNestedValue) { + // MAP> + auto nested_value = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + auto map_type = arrow::map(arrow::utf8(), nested_value); + auto schema = arrow::schema({arrow::field("data", map_type)}); + + std::map column_to_num_columns = {{0, 2}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + schema, column_to_num_columns)); + + auto expected_struct = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), false), + arrow::field("__col_0", nested_value, true), + arrow::field("__col_1", nested_value, true), + arrow::field("__overflow", arrow::map(arrow::int32(), nested_value), true), + }); + auto expected_schema = arrow::schema({arrow::field("data", expected_struct, true)}); + ASSERT_TRUE(physical_schema->Equals(expected_schema)); +} + +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNullable) { + // MAP value is nullable + auto nullable_map = arrow::map(arrow::utf8(), arrow::field("item", arrow::int64(), true)); + auto schema_nullable = arrow::schema({arrow::field("m", nullable_map)}); + std::map col_map = {{0, 2}}; + + ASSERT_OK_AND_ASSIGN( + auto physical, MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_nullable, col_map)); + auto struct_type = physical->field(0)->type(); + ASSERT_TRUE(struct_type->field(1)->nullable()); + ASSERT_TRUE(struct_type->field(2)->nullable()); + + // MAP value is non-nullable + auto non_nullable_map = arrow::map(arrow::utf8(), arrow::field("item", arrow::int64(), false)); + auto schema_non_nullable = arrow::schema({arrow::field("m", non_nullable_map)}); + + ASSERT_OK_AND_ASSIGN(auto physical2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + schema_non_nullable, col_map)); + auto struct_type2 = physical2->field(0)->type(); + ASSERT_FALSE(struct_type2->field(1)->nullable()); + ASSERT_FALSE(struct_type2->field(2)->nullable()); +} + +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNoShreddingColumns) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("name", arrow::utf8()), + }); + + std::map empty_map; + ASSERT_OK_AND_ASSIGN(auto physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, empty_map)); + ASSERT_TRUE(physical_schema->Equals(schema)); +} + +// ---- BuildColumnToNumColumns ---- + +TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumns) { + auto schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + arrow::field("metrics", arrow::map(arrow::utf8(), arrow::float64())), + }); + + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.tags.map.shared-shredding.max-columns", "128"}, + {"fields.metrics.map.shared-shredding.max-columns", "64"}})); + + std::vector shredding_indices = {1, 2}; + ASSERT_OK_AND_ASSIGN(auto result, MapSharedShreddingUtils::BuildColumnToNumColumns( + shredding_indices, schema, options)); + + ASSERT_EQ(result.size(), 2); + ASSERT_EQ(result[1], 128); + ASSERT_EQ(result[2], 64); +} + +TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumnsDefault) { + auto schema = arrow::schema({ + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + }); + + // No explicit max-columns config -> default 256 + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + std::vector shredding_indices = {0}; + ASSERT_OK_AND_ASSIGN(auto result, MapSharedShreddingUtils::BuildColumnToNumColumns( + shredding_indices, schema, options)); + ASSERT_EQ(result[0], 256); +} + +// ---- SerializeMetadata / DeserializeMetadata roundtrip ---- + +TEST(MapSharedShreddingUtilsTest, MetadataRoundtripNoneCompression) { + MapSharedShreddingFieldMeta original; + original.name_to_id = {{"age", 0}, {"name", 1}}; + original.field_to_columns = {{0, {0}}, {1, {1, 2}}}; + original.overflow_field_set = {1, 5}; + original.num_columns = 3; + original.max_row_width = 2; + + auto metadata = std::make_shared(); + ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata(original, "none", metadata.get())); + + // Verify raw KV strings to get intuition of what's stored + auto find_value = [&](const char* key) -> std::string { + int32_t idx = metadata->FindKey(key); + EXPECT_GE(idx, 0); + return metadata->value(idx); + }; + ASSERT_EQ(find_value(MapShreddingDefine::kStorageLayout), "shared-shredding"); + ASSERT_EQ(find_value(MapSharedShreddingDefine::kVersion), "1"); + ASSERT_EQ(find_value(MapSharedShreddingDefine::kNumColumns), "3"); + ASSERT_EQ(find_value(MapSharedShreddingDefine::kMaxRowWidth), "2"); + + std::string expected_dict = R"({"age":0,"name":1})"; + ASSERT_EQ(find_value(MapSharedShreddingDefine::kFieldDict), expected_dict); + // field_dict_original_size should be the length of the JSON string + std::string field_dict_original_size = + find_value(MapSharedShreddingDefine::kFieldDictOriginalSize); + ASSERT_EQ(field_dict_original_size, std::to_string(expected_dict.size())); + + std::string expected_field_to_columns = R"({"0":[0],"1":[1,2]})"; + ASSERT_EQ(find_value(MapSharedShreddingDefine::kFieldColumns), expected_field_to_columns); + + // overflow_set is a JSON array of sorted field_ids + ASSERT_EQ(find_value(MapSharedShreddingDefine::kOverflowSet), "[1,5]"); + + // Roundtrip verify + ASSERT_OK_AND_ASSIGN(auto deserialized, + MapSharedShreddingUtils::DeserializeMetadata(metadata, "none")); + ASSERT_EQ(deserialized, original); +} + +TEST(MapSharedShreddingUtilsTest, MetadataRoundtripCompression) { + MapSharedShreddingFieldMeta original; + original.name_to_id = {{"alpha", 0}, {"beta", 1}, {"gamma", 2}}; + original.field_to_columns = {{0, {0, 1, 2}}, {1, {3}}, {2, {4, 5}}}; + original.overflow_field_set = {2}; + original.num_columns = 6; + original.max_row_width = 3; + + auto verify_roundtrip = [&](const std::string& compression) { + auto metadata = std::make_shared(); + ASSERT_OK( + MapSharedShreddingUtils::SerializeMetadata(original, compression, metadata.get())); + ASSERT_OK_AND_ASSIGN(auto deserialized, + MapSharedShreddingUtils::DeserializeMetadata(metadata, compression)); + ASSERT_EQ(deserialized, original); + }; + + verify_roundtrip("none"); + verify_roundtrip("lz4"); + verify_roundtrip("zstd"); +} + +TEST(MapSharedShreddingUtilsTest, MetadataRoundtripEmptyData) { + MapSharedShreddingFieldMeta original; + + auto verify_roundtrip = [&](const std::string& compression) { + auto metadata = std::make_shared(); + ASSERT_OK( + MapSharedShreddingUtils::SerializeMetadata(original, compression, metadata.get())); + ASSERT_OK_AND_ASSIGN(auto deserialized, + MapSharedShreddingUtils::DeserializeMetadata(metadata, compression)); + ASSERT_EQ(deserialized, original); + }; + + verify_roundtrip("none"); + verify_roundtrip("lz4"); + verify_roundtrip("zstd"); +} + +// ---- DeserializeMetadata error cases ---- + +TEST(MapSharedShreddingUtilsTest, DeserializeMetadataErrors) { + const std::string layout_error = "metadata is null or storage layout is not shared-shredding"; + // nullptr + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(nullptr, "none"), + layout_error); + // missing storage layout + { + auto metadata = std::make_shared(); + metadata->Append("some_key", "some_value"); + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"), + layout_error); + } + // wrong storage layout + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, "default"); + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"), + layout_error); + } + // missing version + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, + MapShreddingDefine::kStorageLayoutSharedShredding); + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"), + "missing shredding metadata key: paimon.map.shared-shredding.version"); + } + // wrong version + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, + MapShreddingDefine::kStorageLayoutSharedShredding); + metadata->Append(MapSharedShreddingDefine::kVersion, "999"); + metadata->Append(MapSharedShreddingDefine::kFieldDictOriginalSize, "2"); + metadata->Append(MapSharedShreddingDefine::kFieldDict, "{}"); + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"), + "unsupported shared-shredding metadata version: 999"); + } + // missing field_dict + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, + MapShreddingDefine::kStorageLayoutSharedShredding); + metadata->Append(MapSharedShreddingDefine::kVersion, "1"); + metadata->Append(MapSharedShreddingDefine::kFieldDictOriginalSize, "2"); + ASSERT_NOK_WITH_MSG( + MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"), + "missing shredding metadata key: paimon.map.shared-shredding.field-dict"); + } +} + +// ---- HasShreddingMetadata ---- + +TEST(MapSharedShreddingUtilsTest, HasShreddingMetadata) { + ASSERT_FALSE(MapSharedShreddingUtils::HasShreddingMetadata(nullptr)); + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, + MapShreddingDefine::kStorageLayoutSharedShredding); + ASSERT_TRUE(MapSharedShreddingUtils::HasShreddingMetadata(metadata)); + } + { + auto metadata = std::make_shared(); + metadata->Append(MapShreddingDefine::kStorageLayout, "default"); + ASSERT_FALSE(MapSharedShreddingUtils::HasShreddingMetadata(metadata)); + } + { + auto metadata = std::make_shared(); + ASSERT_FALSE(MapSharedShreddingUtils::HasShreddingMetadata(metadata)); + } +} + +// ---- PhysicalColumnName ---- + +TEST(MapSharedShreddingUtilsTest, PhysicalColumnName) { + ASSERT_EQ(MapSharedShreddingDefine::PhysicalColumnName(0), "__col_0"); + ASSERT_EQ(MapSharedShreddingDefine::PhysicalColumnName(1), "__col_1"); + ASSERT_EQ(MapSharedShreddingDefine::PhysicalColumnName(99), "__col_99"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/map_shredding_defs.h b/src/paimon/common/data/shredding/map_shredding_defs.h new file mode 100644 index 00000000..3cb028e0 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shredding_defs.h @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace paimon { + +/// Constants for MAP storage layout marker. +struct MapShreddingDefine { + /// Marker key indicating this file uses a specific MAP storage layout. + static constexpr const char* kStorageLayout = "paimon.map.storage-layout"; + /// Value for kStorageLayout when using shared-shredding layout. + static constexpr const char* kStorageLayoutSharedShredding = "shared-shredding"; +}; + +/// Constants for the shared-shredding MAP storage layout. +/// Includes file footer meta keys and physical sub-column names. +struct MapSharedShreddingDefine { + // ---- File footer meta keys ---- + + /// Version of the shared-shredding meta format. + static constexpr const char* kVersion = "paimon.map.shared-shredding.version"; + /// Current meta format version. + static constexpr int32_t kCurrentVersion = 1; + /// JSON-encoded field name <-> field id dictionary (may be compressed). + static constexpr const char* kFieldDict = "paimon.map.shared-shredding.field-dict"; + /// Original (uncompressed) size of field_dict value. + static constexpr const char* kFieldDictOriginalSize = + "paimon.map.shared-shredding.field-dict-original-size"; + /// JSON-encoded field_id -> set of physical column indices. + static constexpr const char* kFieldColumns = "paimon.map.shared-shredding.field-columns"; + /// JSON-encoded set of field_ids that ever spilled into __overflow. + static constexpr const char* kOverflowSet = "paimon.map.shared-shredding.overflow-set"; + /// The number of physical columns K used in this file. + static constexpr const char* kNumColumns = "paimon.map.shared-shredding.num-columns"; + /// The maximum row width observed in this file. + static constexpr const char* kMaxRowWidth = "paimon.map.shared-shredding.max-row-width"; + + // ---- Physical sub-column names ---- + + /// Per-row field mapping column name. + static constexpr const char* kFieldMapping = "__field_mapping"; + /// Overflow column name. + static constexpr const char* kOverflow = "__overflow"; + + /// Returns the name of the i-th physical column: "__col_0", "__col_1", etc. + static std::string PhysicalColumnName(int32_t index) { + return "__col_" + std::to_string(index); + } +}; + +/// Parsed file-level meta for one shared-shredding MAP column. +struct MapSharedShreddingFieldMeta { + /// field_name -> field_id + std::map name_to_id; + /// field_id -> set of physical column indices S + std::map> field_to_columns; + /// Set of field_ids that ever spilled into __overflow + std::set overflow_field_set; + /// Number of physical columns K in this file + int32_t num_columns = 0; + /// Maximum row width observed in this file + int32_t max_row_width = 0; + + bool operator==(const MapSharedShreddingFieldMeta& other) const { + if (this == &other) { + return true; + } + return name_to_id == other.name_to_id && field_to_columns == other.field_to_columns && + overflow_field_set == other.overflow_field_set && num_columns == other.num_columns && + max_row_width == other.max_row_width; + } +}; + +} // namespace paimon diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 0b36c37b..70339ecc 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -94,6 +94,8 @@ const char Options::ROW_TRACKING_PARTITION_GROUP_ON_COMMIT[] = "row-tracking.partition-group-on-commit"; const char Options::DATA_EVOLUTION_ENABLED[] = "data-evolution.enabled"; const char Options::PARTITION_GENERATE_LEGACY_NAME[] = "partition.legacy-name"; +const char Options::MAP_STORAGE_LAYOUT[] = "map.storage-layout"; +const char Options::MAP_SHARED_SHREDDING_MAX_COLUMNS[] = "map.shared-shredding.max-columns"; const char Options::BLOB_AS_DESCRIPTOR[] = "blob-as-descriptor"; const char Options::BLOB_FIELD[] = "blob-field"; const char Options::BLOB_DESCRIPTOR_FIELD[] = "blob-descriptor-field"; diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 96977e77..4fefaffe 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -28,6 +28,7 @@ #include "paimon/common/fs/resolving_file_system.h" #include "paimon/common/options/memory_size.h" #include "paimon/common/options/time_duration.h" +#include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/options/expire_config.h" @@ -1179,6 +1180,32 @@ Result CoreOptions::FieldCollectAggDistinct(const std::string& field_name) return distinct; } +Result CoreOptions::GetMapStorageLayout(const std::string& field_name) const { + std::string key = std::string(Options::FIELDS_PREFIX) + "." + field_name + "." + + std::string(Options::MAP_STORAGE_LAYOUT); + PAIMON_ASSIGN_OR_RAISE(std::string layout_str, OptionsUtils::GetValueFromMap( + impl_->raw_options, key, "default")); + std::string lower = StringUtils::ToLowerCase(layout_str); + if (lower == "shared-shredding") { + return MapStorageLayout::SHARED_SHREDDING; + } else if (lower == "default") { + return MapStorageLayout::DEFAULT; + } + return Status::Invalid(fmt::format("invalid map.storage-layout: {}", layout_str)); +} + +Result CoreOptions::GetMapSharedShreddingMaxColumns(const std::string& field_name) const { + std::string key = std::string(Options::FIELDS_PREFIX) + "." + field_name + "." + + std::string(Options::MAP_SHARED_SHREDDING_MAX_COLUMNS); + PAIMON_ASSIGN_OR_RAISE(int32_t max_columns, + OptionsUtils::GetValueFromMap(impl_->raw_options, key, 256)); + if (max_columns <= 0) { + return Status::Invalid(fmt::format("options {} must > 0", + std::string(Options::MAP_SHARED_SHREDDING_MAX_COLUMNS))); + } + return max_columns; +} + bool CoreOptions::DeletionVectorsEnabled() const { return impl_->deletion_vectors_enabled; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index b064dec2..6a7f3446 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -31,6 +31,7 @@ #include "paimon/core/options/external_path_strategy.h" #include "paimon/core/options/lookup_compact_mode.h" #include "paimon/core/options/lookup_strategy.h" +#include "paimon/core/options/map_storage_layout.h" #include "paimon/core/options/merge_engine.h" #include "paimon/core/options/sort_engine.h" #include "paimon/format/file_format.h" @@ -117,6 +118,10 @@ class PAIMON_EXPORT CoreOptions { Result FieldAggIgnoreRetract(const std::string& field_name) const; Result FieldListAggDelimiter(const std::string& field_name) const; Result FieldCollectAggDistinct(const std::string& field_name) const; + + Result GetMapStorageLayout(const std::string& field_name) const; + Result GetMapSharedShreddingMaxColumns(const std::string& field_name) const; + bool DeletionVectorsEnabled() const; bool DeletionVectorsBitmap64() const; int64_t DeletionVectorTargetFileSize() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 5d140d21..d8f9650a 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -90,6 +90,8 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_FALSE(core_options.FieldAggIgnoreRetract("f1").value()); ASSERT_EQ(",", core_options.FieldListAggDelimiter("f1").value()); ASSERT_FALSE(core_options.FieldCollectAggDistinct("f1").value()); + ASSERT_EQ(MapStorageLayout::DEFAULT, core_options.GetMapStorageLayout("any_col").value()); + ASSERT_EQ(256, core_options.GetMapSharedShreddingMaxColumns("any_col").value()); ASSERT_FALSE(core_options.DeletionVectorsEnabled()); ASSERT_FALSE(core_options.DeletionVectorsBitmap64()); ASSERT_EQ(2 * 1024 * 1024, core_options.DeletionVectorTargetFileSize()); @@ -264,7 +266,9 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::LOOKUP_REMOTE_LEVEL_THRESHOLD, "2"}, {Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}, {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}, - {Options::BUCKET_FUNCTION_TYPE, "mod"}}; + {Options::BUCKET_FUNCTION_TYPE, "mod"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "128"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); auto fs = core_options.GetFileSystem(); @@ -406,6 +410,9 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_TRUE(core_options.LookupRemoteFileEnabled()); ASSERT_EQ(core_options.GetLookupRemoteLevelThreshold(), 2); ASSERT_EQ(BucketFunctionType::MOD, core_options.GetBucketFunctionType()); + ASSERT_EQ(MapStorageLayout::SHARED_SHREDDING, + core_options.GetMapStorageLayout("metrics").value()); + ASSERT_EQ(128, core_options.GetMapSharedShreddingMaxColumns("metrics").value()); } TEST(CoreOptionsTest, TestInvalidCase) { @@ -909,4 +916,53 @@ TEST(CoreOptionsTest, TestFallback) { std::vector({"new_b1", "new_b2"})); } } + +TEST(CoreOptionsTest, TestMapStorageLayout) { + // Test shared-shredding layout configured for a specific column + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.ext_map.map.storage-layout", "shared-shredding"}, + {"fields.ext_map.map.shared-shredding.max-columns", "64"}, + {"fields.normal_map.map.storage-layout", "default"}})); + ASSERT_EQ(MapStorageLayout::SHARED_SHREDDING, + options.GetMapStorageLayout("ext_map").value()); + ASSERT_EQ(64, options.GetMapSharedShreddingMaxColumns("ext_map").value()); + ASSERT_EQ(MapStorageLayout::DEFAULT, options.GetMapStorageLayout("normal_map").value()); + // Unconfigured column falls back to default + ASSERT_EQ(MapStorageLayout::DEFAULT, options.GetMapStorageLayout("other").value()); + ASSERT_EQ(256, options.GetMapSharedShreddingMaxColumns("other").value()); + } + // Test case-insensitive layout value + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.metrics.map.storage-layout", "Shared-Shredding"}})); + ASSERT_EQ(MapStorageLayout::SHARED_SHREDDING, + options.GetMapStorageLayout("metrics").value()); + } + // Test invalid layout value + { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{"fields.col.map.storage-layout", "invalid"}})); + ASSERT_NOK_WITH_MSG(options.GetMapStorageLayout("col"), + "invalid map.storage-layout: invalid"); + } + // Test invalid max-columns value + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.col.map.shared-shredding.max-columns", "0"}})); + ASSERT_NOK_WITH_MSG(options.GetMapSharedShreddingMaxColumns("col"), + "options map.shared-shredding.max-columns must > 0"); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"fields.col.map.shared-shredding.max-columns", "-1"}})); + ASSERT_NOK_WITH_MSG(options.GetMapSharedShreddingMaxColumns("col"), + "options map.shared-shredding.max-columns must > 0"); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/options/map_storage_layout.h b/src/paimon/core/options/map_storage_layout.h new file mode 100644 index 00000000..5f27cdb5 --- /dev/null +++ b/src/paimon/core/options/map_storage_layout.h @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +namespace paimon { +/// Specifies the physical storage layout for MAP columns. +enum class MapStorageLayout { + /// Default KV-array storage (keys array + values array). + DEFAULT = 0, + /// Shared-shredding layout: K reusable typed columns with per-row field mapping. + SHARED_SHREDDING = 1 +}; +} // namespace paimon diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 14907d69..eb43c9c1 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -33,6 +33,7 @@ #include "fmt/format.h" #include "fmt/ranges.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/object_utils.h" @@ -41,6 +42,7 @@ #include "paimon/core/core_options.h" #include "paimon/core/options/changelog_producer.h" #include "paimon/core/options/expire_config.h" +#include "paimon/core/options/map_storage_layout.h" #include "paimon/core/options/merge_engine.h" #include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/core/schema/table_schema.h" @@ -123,6 +125,7 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { PAIMON_RETURN_NOT_OK(ValidateRowTracking(schema, options)); PAIMON_RETURN_NOT_OK(ValidateBlobFields(schema, options)); + PAIMON_RETURN_NOT_OK(ValidateMapStorageLayout(schema, options)); return Status::OK(); } @@ -508,4 +511,64 @@ Status SchemaValidation::ValidateBlobFields(const TableSchema& schema, const Cor return Status::OK(); } +Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema, + const CoreOptions& options) { + // Extract all field names that have map.storage-layout configured from options + const std::string layout_suffix = std::string(".") + std::string(Options::MAP_STORAGE_LAYOUT); + const auto& options_map = options.ToMap(); + + std::unordered_map> schema_fields; + for (const auto& field : schema.Fields()) { + schema_fields[field.Name()] = field.Type(); + } + + std::string fields_prefix_str = std::string(Options::FIELDS_PREFIX); + for (const auto& [key, value] : options_map) { + if (!StringUtils::StartsWith(key, fields_prefix_str)) { + continue; + } + if (!StringUtils::EndsWith(key, layout_suffix)) { + continue; + } + // key = "fields..map.storage-layout" + // Extract field_name: skip "fields." prefix and ".map.storage-layout" suffix + std::string field_name = + key.substr(fields_prefix_str.size() + 1, + key.size() - fields_prefix_str.size() - 1 - layout_suffix.size()); + + // Check field exists in schema + auto it = schema_fields.find(field_name); + if (it == schema_fields.end()) { + return Status::Invalid( + fmt::format("Column '{}' is configured with map.storage-layout " + "but does not exist in table schema.", + field_name)); + } + + // Any column configured with map.storage-layout must be a MAP type + const auto& field_type = it->second; + if (field_type->id() != arrow::Type::MAP) { + return Status::Invalid( + fmt::format("Column '{}' is configured with map.storage-layout " + "but its type is not MAP.", + field_name)); + } + + PAIMON_ASSIGN_OR_RAISE(MapStorageLayout layout, options.GetMapStorageLayout(field_name)); + if (layout != MapStorageLayout::SHARED_SHREDDING) { + continue; + } + // Column configured with shared-shredding must be MAP + if (!MapSharedShreddingUtils::IsShreddingKeyMap(field_type)) { + return Status::Invalid( + fmt::format("Column '{}' is configured with map.storage-layout=shared-shredding " + "but its type is not MAP.", + field_name)); + } + // Validate max-columns config + PAIMON_RETURN_NOT_OK(options.GetMapSharedShreddingMaxColumns(field_name)); + } + return Status::OK(); +} + } // namespace paimon diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 778f8a35..613372ff 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -73,6 +73,8 @@ class SchemaValidation { static Status ValidateBlobFields(const TableSchema& schema, const CoreOptions& options); + static Status ValidateMapStorageLayout(const TableSchema& schema, const CoreOptions& options); + static bool IsComplexType(const std::shared_ptr& field); }; diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 9dc7472a..afabc2f9 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -823,4 +823,122 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) { "Data evolution config must disabled with deletion-vectors.enabled"); } } +TEST(SchemaValidationTest, TestMapStorageLayout) { + auto f0 = arrow::field("f0", arrow::utf8()); + auto f1 = arrow::field("f1", arrow::int32()); + auto f2 = arrow::field("f2", arrow::map(arrow::utf8(), arrow::int64())); + auto f3 = arrow::field("f3", arrow::map(arrow::int32(), arrow::utf8())); + + // Valid: extend on MAP column + { + arrow::FieldVector fields = {f0, f1, f2}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.f2.map.storage-layout", "shared-shredding"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); + } + // Invalid: field not in schema failed in ValidateFieldsPrefix + { + arrow::FieldVector fields = {f0, f1}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.nonexist.map.storage-layout", "shared-shredding"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "Field nonexist can not be found in table schema"); + } + // Invalid: field not in schema failed in ValidateMapStorageLayout + { + arrow::FieldVector fields = {f0, f1}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.nonexist.map.storage-layout", "shared-shredding"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateMapStorageLayout(*table_schema, core_options), + "Column 'nonexist' is configured with map.storage-layout but does not " + "exist in table schema."); + } + + // Invalid: non-MAP column configured with map.storage-layout (any value) + { + arrow::FieldVector fields = {f0, f1}; + auto schema = arrow::schema(fields); + std::map options = {{Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.f1.map.storage-layout", "default"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "not MAP"); + } + // Invalid: shared-shredding on non-MAP column + { + arrow::FieldVector fields = {f0, f1}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.f1.map.storage-layout", "shared-shredding"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "not MAP"); + } + // Invalid: shared-shredding on MAP with non-STRING key + { + arrow::FieldVector fields = {f0, f1, f3}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.f3.map.storage-layout", "shared-shredding"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "not MAP"); + } + // Valid: default layout on a MAP column + { + arrow::FieldVector fields = {f0, f1, f2}; + auto schema = arrow::schema(fields); + std::map options = {{Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.f2.map.storage-layout", "default"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema)); + } + // Invalid: shared-shredding with invalid max-columns + { + arrow::FieldVector fields = {f0, f1, f2}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.f2.map.storage-layout", "shared-shredding"}, + {"fields.f2.map.shared-shredding.max-columns", "0"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "options map.shared-shredding.max-columns must > 0"); + } +} + } // namespace paimon::test From 9cd1a50f123f1b03b7df4b8bbe3559f942e72266 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Fri, 12 Jun 2026 15:04:23 +0800 Subject: [PATCH 052/138] fix: make byte hashing independent of char signedness --- src/paimon/common/utils/murmurhash_utils.h | 12 +++++++----- src/paimon/core/bucket/hive_bucket_function.cpp | 7 +++++-- src/paimon/core/bucket/hive_hasher.h | 4 +++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/paimon/common/utils/murmurhash_utils.h b/src/paimon/common/utils/murmurhash_utils.h index 52831c9a..e184d9a0 100644 --- a/src/paimon/common/utils/murmurhash_utils.h +++ b/src/paimon/common/utils/murmurhash_utils.h @@ -184,7 +184,9 @@ class MurmurHashUtils { int32_t length_aligned = length_in_bytes - length_in_bytes % 4; int32_t h1 = HashBytesByInt(segment, offset, length_aligned, seed); for (int32_t i = length_aligned; i < length_in_bytes; i++) { - int32_t k1 = MixK1(segment.Get(offset + i)); + auto byte = static_cast(segment.Get(offset + i)); + int32_t signed_byte = byte < 128 ? byte : static_cast(byte) - 256; + int32_t k1 = MixK1(signed_byte); h1 = MixH1(h1, k1); } return Fmix(h1, length_in_bytes); @@ -240,10 +242,10 @@ class MurmurHashUtils { return value; } - static char GetByte(const void* base, int64_t offset) { - char value; - std::memcpy(&value, static_cast(base) + offset, sizeof(char)); - return value; + static int32_t GetByte(const void* base, int64_t offset) { + uint8_t value; + std::memcpy(&value, static_cast(base) + offset, sizeof(uint8_t)); + return value < 128 ? value : static_cast(value) - 256; } public: diff --git a/src/paimon/core/bucket/hive_bucket_function.cpp b/src/paimon/core/bucket/hive_bucket_function.cpp index c78c7947..7bcf004a 100644 --- a/src/paimon/core/bucket/hive_bucket_function.cpp +++ b/src/paimon/core/bucket/hive_bucket_function.cpp @@ -88,8 +88,11 @@ uint32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_ind switch (info.type) { case FieldType::BOOLEAN: return HiveHasher::HashInt(row.GetBoolean(field_index) ? 1 : 0); - case FieldType::TINYINT: - return HiveHasher::HashInt(static_cast(row.GetByte(field_index))); + case FieldType::TINYINT: { + auto byte = static_cast(row.GetByte(field_index)); + int32_t signed_byte = byte < 128 ? byte : static_cast(byte) - 256; + return HiveHasher::HashInt(static_cast(signed_byte)); + } case FieldType::SMALLINT: return HiveHasher::HashInt(static_cast(row.GetShort(field_index))); case FieldType::INT: diff --git a/src/paimon/core/bucket/hive_hasher.h b/src/paimon/core/bucket/hive_hasher.h index fad87a1a..e9dad0e4 100644 --- a/src/paimon/core/bucket/hive_hasher.h +++ b/src/paimon/core/bucket/hive_hasher.h @@ -44,7 +44,9 @@ class HiveHasher { static uint32_t HashBytes(const char* bytes, int32_t length) { uint32_t result = 0; for (int32_t i = 0; i < length; i++) { - result = result * 31U + static_cast(static_cast(bytes[i])); + auto byte = static_cast(bytes[i]); + int32_t signed_byte = byte < 128 ? byte : static_cast(byte) - 256; + result = result * 31U + static_cast(signed_byte); } return result; } From 4d170d6ee68fc1cc5c3ebda0c4b21d9e449f8a3f Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Fri, 12 Jun 2026 01:27:01 -0700 Subject: [PATCH 053/138] feat(shared-shredding): support nested field projection in ORC format reader --- .../format/orc/orc_file_batch_reader.cpp | 84 ++-- src/paimon/format/orc/orc_file_batch_reader.h | 9 +- .../format/orc/orc_file_batch_reader_test.cpp | 392 ++++++++++++++++-- 3 files changed, 415 insertions(+), 70 deletions(-) diff --git a/src/paimon/format/orc/orc_file_batch_reader.cpp b/src/paimon/format/orc/orc_file_batch_reader.cpp index ed25327b..7836c06c 100644 --- a/src/paimon/format/orc/orc_file_batch_reader.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader.cpp @@ -174,56 +174,62 @@ std::shared_ptr OrcFileBatchReader::GetReaderMetrics() const { return metrics_; } -Result> OrcFileBatchReader::GetAndCheckIncludedFields( - const ::orc::Type* src_type, const ::orc::Type* target_type, - std::vector* target_column_ids) { - std::list include_fields; - std::unordered_map src_type_map; - for (uint64_t i = 0; i < src_type->getSubtypeCount(); i++) { - src_type_map[src_type->getFieldName(i)] = src_type->getSubtype(i); +Status OrcFileBatchReader::CollectTargetColumnIds(const ::orc::Type* src_type, + const ::orc::Type* target_type, + std::vector* target_column_ids) { + auto src_kind = src_type->getKind(); + auto target_kind = target_type->getKind(); + if (src_kind != target_kind) { + return Status::Invalid(fmt::format("type kind mismatch: src {} vs target {}", + src_type->toString(), target_type->toString())); } - int64_t prev_target_field_col_id = -1; - for (uint64_t i = 0; i < target_type->getSubtypeCount(); i++) { - auto& field_name = target_type->getFieldName(i); - auto iter = src_type_map.find(field_name); - if (iter == src_type_map.end()) { - return Status::Invalid( - fmt::format("field {} not in file schema {}", field_name, src_type->toString())); - } - // Noted that: do not support recall partial fields in nested type - if (iter->second->toString() != target_type->getSubtype(i)->toString()) { - return Status::Invalid( - fmt::format("target_type {} not match src_type {}, mismatch field name {}", - target_type->toString(), src_type->toString(), field_name)); + + switch (src_kind) { + case ::orc::TypeKind::STRUCT: { + std::unordered_map src_field_map; + for (uint64_t i = 0; i < src_type->getSubtypeCount(); i++) { + src_field_map[src_type->getFieldName(i)] = src_type->getSubtype(i); + } + for (uint64_t i = 0; i < target_type->getSubtypeCount(); i++) { + auto& field_name = target_type->getFieldName(i); + auto iter = src_field_map.find(field_name); + if (iter == src_field_map.end()) { + return Status::Invalid(fmt::format("field {} not in file schema {}", field_name, + src_type->toString())); + } + PAIMON_RETURN_NOT_OK(CollectTargetColumnIds( + iter->second, target_type->getSubtype(i), target_column_ids)); + } + break; } - int64_t target_field_col_id = iter->second->getColumnId(); - GetSubColumnIds(iter->second, target_column_ids); - if (prev_target_field_col_id >= target_field_col_id) { - return Status::Invalid( - "The column id of the target field should be monotonically increasing in " - "format reader"); + // Do not support partial field recall inside list/map types. + default: { + if (src_type->toString() != target_type->toString()) { + return Status::Invalid(fmt::format("type mismatch: src {} vs target {}", + src_type->toString(), target_type->toString())); + } + target_column_ids->push_back(src_type->getColumnId()); + break; } - prev_target_field_col_id = target_field_col_id; - include_fields.push_back(field_name); - } - return include_fields; -} - -void OrcFileBatchReader::GetSubColumnIds(const ::orc::Type* type, std::vector* col_ids) { - col_ids->push_back(type->getColumnId()); - for (uint64_t i = 0; i < type->getSubtypeCount(); i++) { - GetSubColumnIds(type->getSubtype(i), col_ids); } + return Status::OK(); } Result<::orc::RowReaderOptions> OrcFileBatchReader::CreateRowReaderOptions( const ::orc::Type* src_type, const ::orc::Type* target_type, std::unique_ptr<::orc::SearchArgument>&& search_arg, const std::map& options, std::vector* target_column_ids) { - PAIMON_ASSIGN_OR_RAISE(std::list include_fields, - GetAndCheckIncludedFields(src_type, target_type, target_column_ids)); + PAIMON_RETURN_NOT_OK(CollectTargetColumnIds(src_type, target_type, target_column_ids)); + for (size_t i = 1; i < target_column_ids->size(); i++) { + if ((*target_column_ids)[i - 1] >= (*target_column_ids)[i]) { + return Status::Invalid( + "The column id of the target field should be monotonically increasing in " + "format reader"); + } + } ::orc::RowReaderOptions row_reader_options; - row_reader_options.include(include_fields); + std::list include_type_ids(target_column_ids->begin(), target_column_ids->end()); + row_reader_options.includeTypes(include_type_ids); // In order to avoid issue like https://github.com/alibaba/paimon-cpp/issues/42, we explicitly // set GMT timezone. row_reader_options.setTimezoneName("GMT"); diff --git a/src/paimon/format/orc/orc_file_batch_reader.h b/src/paimon/format/orc/orc_file_batch_reader.h index 56e7a578..5ba4726d 100644 --- a/src/paimon/format/orc/orc_file_batch_reader.h +++ b/src/paimon/format/orc/orc_file_batch_reader.h @@ -103,18 +103,15 @@ class OrcFileBatchReader : public PrefetchFileBatchReader { const std::shared_ptr& arrow_pool, const std::shared_ptr<::orc::MemoryPool>& orc_pool); - static void GetSubColumnIds(const ::orc::Type* type, std::vector* col_ids); - static Result<::orc::RowReaderOptions> CreateRowReaderOptions( const ::orc::Type* src_type, const ::orc::Type* target_type, std::unique_ptr<::orc::SearchArgument>&& search_arg, const std::map& options, std::vector* target_column_ids); - static Result> GetAndCheckIncludedFields( - const ::orc::Type* src_type, const ::orc::Type* target_type, - std::vector* target_column_ids); - + static Status CollectTargetColumnIds(const ::orc::Type* src_type, + const ::orc::Type* target_type, + std::vector* target_column_ids); std::map options_; std::shared_ptr arrow_pool_; diff --git a/src/paimon/format/orc/orc_file_batch_reader_test.cpp b/src/paimon/format/orc/orc_file_batch_reader_test.cpp index 8fbb5161..ccb79699 100644 --- a/src/paimon/format/orc/orc_file_batch_reader_test.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader_test.cpp @@ -303,12 +303,13 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { std::string orc_schema = "struct"; std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(orc_schema); std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(orc_schema); + target_column_ids.clear(); ASSERT_OK_AND_ASSIGN(auto row_reader_option, OrcFileBatchReader::CreateRowReaderOptions( src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids)); - ASSERT_EQ(std::list({"col1", "col2", "col3"}), - row_reader_option.getIncludeNames()); + // col1(1), col2(2), col3(3) — struct container(0) is not included + ASSERT_EQ(target_column_ids, (std::vector{1, 2, 3})); ASSERT_EQ(row_reader_option.getEnableLazyDecoding(), false); } { @@ -319,28 +320,15 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { std::string target_orc_schema = "struct"; std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); ASSERT_OK_AND_ASSIGN(auto row_reader_option, OrcFileBatchReader::CreateRowReaderOptions( src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids)); - ASSERT_EQ(std::list({"col1", "col3"}), row_reader_option.getIncludeNames()); + // col1(1), col3(3) — struct container(0) not included, col2(2) skipped + ASSERT_EQ(target_column_ids, (std::vector{1, 3})); ASSERT_EQ(row_reader_option.getEnableLazyDecoding(), true); } - { - // read partial fields, sequence mismatch - std::map options; - std::string src_orc_schema = "struct"; - std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); - std::string target_orc_schema = "struct"; - std::unique_ptr<::orc::Type> target_type = - ::orc::Type::buildTypeFromString(target_orc_schema); - ASSERT_NOK_WITH_MSG( - OrcFileBatchReader::CreateRowReaderOptions(src_type.get(), target_type.get(), - /*search_arg=*/nullptr, options, - &target_column_ids), - "The column id of the target field should be monotonically increasing in format " - "reader"); - } { // read non exist column std::map options; @@ -349,12 +337,14 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { std::string target_orc_schema = "struct"; std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids), "field non_exist_col not in file schema"); } { + // read partial top-level fields with nested type (all sub-fields) std::map options; std::string src_orc_schema = "struct>,sub3:int>,col2:double,col3:" @@ -364,14 +354,16 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { "struct>,sub3:int>,col3:map>"; std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); ASSERT_OK_AND_ASSIGN(auto row_reader_option, OrcFileBatchReader::CreateRowReaderOptions( src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids)); - ASSERT_EQ(std::list({"col1", "col3"}), row_reader_option.getIncludeNames()); + // Struct IDs (0, 1) not included. Selected: sub1(2), sub2-list(3), sub3(6), col3-map(8). + ASSERT_EQ(target_column_ids, (std::vector{2, 3, 6, 8})); } { - // read with type mismatch + // read with type mismatch in nested field std::map options; std::string src_orc_schema = "struct,col2:double,col3:string>"; @@ -380,15 +372,107 @@ TEST_F(OrcFileBatchReaderTest, TestCreateRowReaderOptions) { "struct,col2:double,col3:string>"; std::unique_ptr<::orc::Type> target_type = ::orc::Type::buildTypeFromString(target_orc_schema); - + target_column_ids.clear(); + ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids), + "type kind mismatch"); + } + { + // read partial sub-fields of nested struct (nested field projection) + std::map options; + std::string src_orc_schema = + "struct,col2:int>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + // only read sub1 and sub3 from col1 + std::string target_orc_schema = "struct>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + ASSERT_OK_AND_ASSIGN(auto row_reader_option, + OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids)); + // src type IDs: struct(0), col1(1){sub1(2), sub2(3), sub3(4)}, col2(5) + // Struct IDs (0, 1) not included. Selected: sub1(2), sub3(4) + ASSERT_EQ(target_column_ids, (std::vector{2, 4})); + } + { + // nested struct sub-fields out-of-order should fail + std::map options; + std::string src_orc_schema = + "struct,col2:int>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + std::string target_orc_schema = "struct>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); ASSERT_NOK_WITH_MSG( OrcFileBatchReader::CreateRowReaderOptions(src_type.get(), target_type.get(), /*search_arg=*/nullptr, options, &target_column_ids), - "target_type " - "struct,col2:double,col3:string> not match " - "src_type struct,col2:double,col3:string>, " - "mismatch field name col1"); + "The column id of the target field should be monotonically increasing in format " + "reader"); + } + { + // top-level order correct but nested sub-fields out-of-order should fail + std::map options; + std::string src_orc_schema = + "struct,col2:int>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + // col1 before col2 (correct top-level order), but sub2 before sub1 (wrong nested order) + std::string target_orc_schema = "struct,col2:int>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + ASSERT_NOK_WITH_MSG( + OrcFileBatchReader::CreateRowReaderOptions(src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, + &target_column_ids), + "The column id of the target field should be monotonically increasing in format " + "reader"); + } + { + // decimal precision mismatch + std::map options; + std::string src_orc_schema = "struct"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + std::string target_orc_schema = "struct"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids), + "type mismatch"); + } + { + std::map options; + std::string src_orc_schema = "struct>>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + std::string target_orc_schema = "struct>>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + // list/map sub-field partial projection is not supported; toString() mismatch is reported + ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids), + "type mismatch"); + } + { + std::map options; + std::string src_orc_schema = "struct>>"; + std::unique_ptr<::orc::Type> src_type = ::orc::Type::buildTypeFromString(src_orc_schema); + std::string target_orc_schema = "struct>>"; + std::unique_ptr<::orc::Type> target_type = + ::orc::Type::buildTypeFromString(target_orc_schema); + target_column_ids.clear(); + // list/map sub-field partial projection is not supported; toString() mismatch is reported + ASSERT_NOK_WITH_MSG(OrcFileBatchReader::CreateRowReaderOptions( + src_type.get(), target_type.get(), + /*search_arg=*/nullptr, options, &target_column_ids), + "type mismatch"); } } @@ -791,4 +875,262 @@ TEST_P(OrcFileBatchReaderTest, TestTimestampType) { // TODO(liancheng.lsz): TestBitmapPushDownWithMultiRowGroups, TestPredicateAndBitmapPushDown // TODO(liancheng.lsz): TestGenReadRanges + +TEST_F(OrcFileBatchReaderTest, TestNestedFieldProjection) { + // Write data with nested struct: struct,col2:int> + auto sub1 = arrow::field("sub1", arrow::int32()); + auto sub2 = arrow::field("sub2", arrow::float64()); + auto sub3 = arrow::field("sub3", arrow::utf8()); + auto col1 = arrow::field("col1", arrow::struct_({sub1, sub2, sub3})); + auto col2 = arrow::field("col2", arrow::int32()); + + arrow::FieldVector write_fields = {col1, col2}; + auto write_schema = arrow::schema(write_fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_fields), R"([ + [[10, 1.1, "aaa"], 100], + [[20, 2.2, "bbb"], 200], + [[30, 3.3, "ccc"], 300], + [null, 400], + [[50, null, "eee"], null] + ])") + .ValueOrDie()); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string data_path = dir->Str() + "/nested_test.orc"; + WriteArray(dir->GetFileSystem(), data_path, src_array, write_schema, /*options=*/{}); + + { + // Read partial sub-fields: col1.sub1 and col1.sub3 only + auto read_col1 = arrow::field("col1", arrow::struct_({sub1, sub3})); + arrow::Schema read_schema({read_col1}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1}), R"([ + [[10, "aaa"]], + [[20, "bbb"]], + [[30, "ccc"]], + [null], + [[50, "eee"]] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } + { + // Read partial sub-fields + top-level field: col1.sub2 and col2 + auto read_col1 = arrow::field("col1", arrow::struct_({sub2})); + arrow::Schema read_schema({read_col1, col2}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1, col2}), R"([ + [[1.1], 100], + [[2.2], 200], + [[3.3], 300], + [null, 400], + [[null], null] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } + { + // Read single nested sub-field: col1.sub1 only + auto read_col1 = arrow::field("col1", arrow::struct_({sub1})); + arrow::Schema read_schema({read_col1}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1}), R"([ + [[10]], + [[20]], + [[30]], + [null], + [[50]] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } +} + +TEST_F(OrcFileBatchReaderTest, TestDeepNestedFieldProjection) { + // struct,e:double>,f:int> + auto field_c = arrow::field("c", arrow::int32()); + auto field_d = arrow::field("d", arrow::utf8()); + auto field_b = arrow::field("b", arrow::struct_({field_c, field_d})); + auto field_e = arrow::field("e", arrow::float64()); + auto field_a = arrow::field("a", arrow::struct_({field_b, field_e})); + auto field_f = arrow::field("f", arrow::int32()); + + arrow::FieldVector write_fields = {field_a, field_f}; + auto write_schema = arrow::schema(write_fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_fields), R"([ + [[[1, "x"], 10.0], 100], + [[[2, "y"], 20.0], 200], + [null, 300] + ])") + .ValueOrDie()); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string data_path = dir->Str() + "/deep_nested_test.orc"; + WriteArray(dir->GetFileSystem(), data_path, src_array, write_schema, /*options=*/{}); + + { + // Read a.b.c only (skip a.b.d and a.e) + auto read_b = arrow::field("b", arrow::struct_({field_c})); + auto read_a = arrow::field("a", arrow::struct_({read_b})); + arrow::Schema read_schema({read_a}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_a}), R"([ + [[[1]]], + [[[2]]], + [null] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } +} + +TEST_F(OrcFileBatchReaderTest, TestNestedFieldProjectionWithListAndMap) { + // struct,sub3:map>,col2:string> + auto sub1 = arrow::field("sub1", arrow::int32()); + auto sub2 = arrow::field("sub2", arrow::list(arrow::int32())); + auto sub3 = arrow::field("sub3", arrow::map(arrow::utf8(), arrow::int32())); + auto col1 = arrow::field("col1", arrow::struct_({sub1, sub2, sub3})); + auto col2 = arrow::field("col2", arrow::utf8()); + + arrow::FieldVector write_fields = {col1, col2}; + auto write_schema = arrow::schema(write_fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_fields), R"([ + [[10, [1, 2, 3], [["a", 1], ["b", 2]]], "hello"], + [[20, [4, 5], [["c", 3]]], "world"], + [[30, null, null], null] + ])") + .ValueOrDie()); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string data_path = dir->Str() + "/nested_list_map_test.orc"; + WriteArray(dir->GetFileSystem(), data_path, src_array, write_schema, /*options=*/{}); + + { + // Read col1.sub2 (list type) only — nested field projection skipping sub1 and sub3 + auto read_col1 = arrow::field("col1", arrow::struct_({sub2})); + arrow::Schema read_schema({read_col1}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1}), R"([ + [[[1, 2, 3]]], + [[[4, 5]]], + [[null]] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } + { + // Read col1.sub1 + col1.sub3(map) — skip sub2 + auto read_col1 = arrow::field("col1", arrow::struct_({sub1, sub3})); + arrow::Schema read_schema({read_col1, col2}); + auto orc_batch_reader = + PrepareOrcFileBatchReader(data_path, &read_schema, + /*batch_size=*/10, DEFAULT_NATURAL_READ_SIZE); + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + orc_batch_reader.get())); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({read_col1, col2}), R"([ + [[10, [["a", 1], ["b", 2]]], "hello"], + [[20, [["c", 3]]], "world"], + [[30, null], null] + ])") + .ValueOrDie()); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(result_array->Equals(expected_chunked)) + << "actual: " << result_array->ToString() + << "\nexpected: " << expected_chunked->ToString(); + } +} + +TEST_F(OrcFileBatchReaderTest, TestListStructPartialProjection) { + // Verify that array> read as array> is rejected. + // Partial projection inside list/map elements is not supported; the list type toString() + // comparison catches the mismatch and returns an error before any ORC batch is read. + auto field_a = arrow::field("a", arrow::int32()); + auto field_b = arrow::field("b", arrow::float64()); + auto col1 = arrow::field("col1", arrow::list(arrow::struct_({field_a, field_b}))); + + arrow::FieldVector write_fields = {col1}; + auto write_schema = arrow::schema(write_fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_fields), R"([ + [[[1, 1.1], [2, 2.2]]], + [[[3, 3.3]]], + [[null]], + [null] + ])") + .ValueOrDie()); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string data_path = dir->Str() + "/list_struct_projection_test.orc"; + WriteArray(dir->GetFileSystem(), data_path, src_array, write_schema, /*options=*/{}); + + auto read_col1 = arrow::field("col1", arrow::list(arrow::struct_({field_a}))); + arrow::Schema read_schema({read_col1}); + std::shared_ptr file_system = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(data_path)); + ASSERT_OK_AND_ASSIGN(auto in_stream, + OrcInputStreamImpl::Create(input_stream, DEFAULT_NATURAL_READ_SIZE)); + ASSERT_OK_AND_ASSIGN( + auto orc_batch_reader, + OrcFileBatchReader::Create(std::move(in_stream), pool_, /*options=*/{}, /*batch_size=*/10)); + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(read_schema, c_schema.get()).ok()); + ASSERT_NOK_WITH_MSG(orc_batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + "type mismatch"); +} + } // namespace paimon::orc::test From 4e9cb8b3283741444e18ad54fe8da152126d26cb Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Fri, 12 Jun 2026 22:34:03 +0800 Subject: [PATCH 054/138] fix: prevent zero-thread DefaultExecutor and validate FileSystem/SchemeMap conflict --- include/paimon/executor.h | 3 +- .../common/executor/default_executor_test.cpp | 12 ++++++-- src/paimon/common/executor/executor.cpp | 11 +++++-- .../apply_bitmap_index_batch_reader_test.cpp | 2 +- src/paimon/common/fs/file_system_test.cpp | 6 ++-- src/paimon/common/logging/logging_test.cpp | 2 +- .../prefetch_file_batch_reader_impl_test.cpp | 8 +++-- ...pply_deletion_vector_batch_reader_test.cpp | 2 +- .../global_index/global_index_scan_impl.cpp | 3 +- .../operation/abstract_file_store_write.cpp | 2 +- .../operation/data_evolution_split_read.cpp | 2 +- .../operation/merge_file_split_read_test.cpp | 2 +- .../operation/raw_file_split_read_test.cpp | 21 +++++++------ src/paimon/core/operation/read_context.cpp | 9 +++++- .../core/operation/read_context_test.cpp | 30 +++++++++++++++++-- test/inte/global_index_test.cpp | 3 +- 16 files changed, 84 insertions(+), 34 deletions(-) diff --git a/include/paimon/executor.h b/include/paimon/executor.h index c835ef6d..7092af7b 100644 --- a/include/paimon/executor.h +++ b/include/paimon/executor.h @@ -23,6 +23,7 @@ #include #include +#include "paimon/result.h" #include "paimon/visibility.h" namespace paimon { @@ -37,7 +38,7 @@ PAIMON_EXPORT std::shared_ptr GetGlobalDefaultExecutor(); PAIMON_EXPORT std::unique_ptr CreateDefaultExecutor(); /// Create a default implementation of executor with specified thread_count. -PAIMON_EXPORT std::unique_ptr CreateDefaultExecutor(uint32_t thread_count); +PAIMON_EXPORT Result> CreateDefaultExecutor(uint32_t thread_count); /// Interface class for defining basic operations of a task executor. /// diff --git a/src/paimon/common/executor/default_executor_test.cpp b/src/paimon/common/executor/default_executor_test.cpp index 2745a69b..91f2c94c 100644 --- a/src/paimon/common/executor/default_executor_test.cpp +++ b/src/paimon/common/executor/default_executor_test.cpp @@ -31,6 +31,7 @@ #include "paimon/executor.h" #include "paimon/result.h" #include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -86,7 +87,7 @@ TEST(DefaultExecutorTest, TestViaWithException) { } TEST(DefaultExecutorTest, TestShutdownNowDropsPendingTasks) { - auto executor = CreateDefaultExecutor(/*thread_count=*/1); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/1)); std::atomic first_started = false; std::atomic executed_count = 0; std::promise release_first_task; @@ -114,7 +115,7 @@ TEST(DefaultExecutorTest, TestShutdownNowDropsPendingTasks) { } TEST(DefaultExecutorTest, TestAddTaskAfterShutdownNowIgnored) { - auto executor = CreateDefaultExecutor(/*thread_count=*/1); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/1)); std::atomic executed_count = 0; executor->ShutdownNow(); @@ -125,7 +126,7 @@ TEST(DefaultExecutorTest, TestAddTaskAfterShutdownNowIgnored) { } TEST(DefaultExecutorTest, TestAddTaskFromMultipleThreads) { - auto executor = CreateDefaultExecutor(/*thread_count=*/4); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/4)); constexpr int32_t kSubmitterCount = 8; constexpr int32_t kTaskCountPerSubmitter = 64; @@ -181,4 +182,9 @@ TEST(DefaultExecutorTest, TestAddTaskFromMultipleThreads) { } } +TEST(DefaultExecutorTest, TestCreateWithZeroThreadCount) { + ASSERT_NOK_WITH_MSG(CreateDefaultExecutor(/*thread_count=*/0), + "default executor thread count should be greater than 0"); +} + } // namespace paimon::test diff --git a/src/paimon/common/executor/executor.cpp b/src/paimon/common/executor/executor.cpp index 0f944b4b..cd3f699d 100644 --- a/src/paimon/common/executor/executor.cpp +++ b/src/paimon/common/executor/executor.cpp @@ -19,6 +19,7 @@ #include "paimon/executor.h" +#include #include #include #include @@ -53,6 +54,7 @@ class DefaultExecutor : public Executor { }; DefaultExecutor::DefaultExecutor(uint32_t thread_count) : thread_count_(thread_count) { + assert(thread_count > 0); for (uint32_t i = 0; i < thread_count_; ++i) { workers_.emplace_back(&DefaultExecutor::WorkerThread, this); } @@ -137,15 +139,18 @@ void DefaultExecutor::WorkerThread() { PAIMON_EXPORT std::shared_ptr GetGlobalDefaultExecutor() { static uint32_t all_cores = std::thread::hardware_concurrency(); static std::shared_ptr internal = - std::make_shared(/*thread_count=*/all_cores); + std::make_shared(/*thread_count=*/all_cores > 0 ? all_cores : 1); return internal; } PAIMON_EXPORT std::unique_ptr CreateDefaultExecutor() { - return CreateDefaultExecutor(DEFAULT_EXECUTOR_THREAD_COUNT); + return std::make_unique(DEFAULT_EXECUTOR_THREAD_COUNT); } -PAIMON_EXPORT std::unique_ptr CreateDefaultExecutor(uint32_t thread_count) { +PAIMON_EXPORT Result> CreateDefaultExecutor(uint32_t thread_count) { + if (thread_count == 0) { + return Status::Invalid("default executor thread count should be greater than 0"); + } return std::make_unique(thread_count); } diff --git a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp index 1168a358..7b1ebd3f 100644 --- a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp +++ b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader_test.cpp @@ -56,7 +56,7 @@ class ApplyBitmapIndexBatchReaderTest : public ::testing::Test, pool_ = GetDefaultPool(); fs_ = std::make_shared(); - executor_ = CreateDefaultExecutor(/*thread_count=*/2); + ASSERT_OK_AND_ASSIGN(executor_, CreateDefaultExecutor(/*thread_count=*/2)); } void TearDown() override {} diff --git a/src/paimon/common/fs/file_system_test.cpp b/src/paimon/common/fs/file_system_test.cpp index d6e14d6d..7a80904f 100644 --- a/src/paimon/common/fs/file_system_test.cpp +++ b/src/paimon/common/fs/file_system_test.cpp @@ -1089,7 +1089,7 @@ TEST_P(FileSystemTest, TestMkdir2) { TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameNonExistParentDir) { uint32_t runs_count = 10; uint32_t thread_count = 10; - auto executor = CreateDefaultExecutor(thread_count); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(thread_count)); for (uint32_t i = 0; i < runs_count; i++) { std::string uuid; @@ -1114,7 +1114,7 @@ TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameNonExistParentDir) { TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameName) { uint32_t runs_count = 10; uint32_t thread_count = 10; - auto executor = CreateDefaultExecutor(thread_count); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(thread_count)); for (uint32_t i = 0; i < runs_count; i++) { std::string uuid; @@ -1139,7 +1139,7 @@ TEST_P(FileSystemTest, TestMkdirMultiThreadWithSameNameWithRelativePath) { } uint32_t runs_count = 10; uint32_t thread_count = 10; - auto executor = CreateDefaultExecutor(thread_count); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(thread_count)); for (uint32_t i = 0; i < runs_count; i++) { std::string uuid; diff --git a/src/paimon/common/logging/logging_test.cpp b/src/paimon/common/logging/logging_test.cpp index 8b6e3f69..88ffc45b 100644 --- a/src/paimon/common/logging/logging_test.cpp +++ b/src/paimon/common/logging/logging_test.cpp @@ -27,7 +27,7 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { TEST(LoggerTest, TestMultiThreadGetLogger) { - auto executor = CreateDefaultExecutor(/*thread_count=*/4); + ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/4)); auto get_logger = []() { auto logger = Logger::GetLogger("my_log"); ASSERT_TRUE(logger); diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp index 036b1d3b..4d9fcf09 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp @@ -125,7 +125,7 @@ class PrefetchFileBatchReaderImplTest : public ::testing::Test, data_type_ = arrow::struct_(fields_); mock_fs_ = std::make_shared(); local_fs_ = std::make_shared(); - executor_ = CreateDefaultExecutor(/*thread_count=*/2); + ASSERT_OK_AND_ASSIGN(executor_, CreateDefaultExecutor(/*thread_count=*/2)); dir_ = ::paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(dir_); } @@ -196,14 +196,16 @@ class PrefetchFileBatchReaderImplTest : public ::testing::Test, EXPECT_OK_AND_ASSIGN(std::unique_ptr file_format, FileFormatFactory::Get(file_format_str, {})); EXPECT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(batch_size)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr executor, + CreateDefaultExecutor(prefetch_max_parallel_num - 1)); EXPECT_OK_AND_ASSIGN( std::unique_ptr reader, PrefetchFileBatchReaderImpl::Create( PathUtil::JoinPath(dir_->Str(), "file." + file_format->Identifier()), reader_builder.get(), local_fs_, prefetch_max_parallel_num, batch_size, prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, - CreateDefaultExecutor(prefetch_max_parallel_num - 1), - /*initialize_read_ranges=*/false, cache_mode, CacheConfig(), GetDefaultPool())); + executor, /*initialize_read_ranges=*/false, cache_mode, CacheConfig(), + GetDefaultPool())); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); EXPECT_TRUE(arrow_status.ok()); diff --git a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp index 28920224..daa42d37 100644 --- a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp +++ b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader_test.cpp @@ -51,7 +51,7 @@ class ApplyDeletionVectorBatchReaderTest : public ::testing::Test, pool_ = GetDefaultPool(); fs_ = std::make_shared(); - executor_ = CreateDefaultExecutor(/*thread_count=*/2); + ASSERT_OK_AND_ASSIGN(executor_, CreateDefaultExecutor(/*thread_count=*/2)); } void TearDown() override {} diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index 45b2cc73..b40478e1 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -64,7 +64,8 @@ Result> GlobalIndexScanImpl::Create( uint32_t cpu_count = std::thread::hardware_concurrency(); thread_num = cpu_count > 0 ? static_cast(cpu_count) : 1; } - final_executor = CreateDefaultExecutor(static_cast(thread_num.value())); + PAIMON_ASSIGN_OR_RAISE(final_executor, + CreateDefaultExecutor(static_cast(thread_num.value()))); } auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); diff --git a/src/paimon/core/operation/abstract_file_store_write.cpp b/src/paimon/core/operation/abstract_file_store_write.cpp index b81375e7..2b2cda88 100644 --- a/src/paimon/core/operation/abstract_file_store_write.cpp +++ b/src/paimon/core/operation/abstract_file_store_write.cpp @@ -76,7 +76,7 @@ AbstractFileStoreWrite::AbstractFileStoreWrite( dv_maintainer_factory_(dv_maintainer_factory), io_manager_(io_manager), options_(options), - compact_executor_(CreateDefaultExecutor(4)), + compact_executor_(CreateDefaultExecutor()), compaction_metrics_(std::make_shared()), ignore_previous_files_(ignore_previous_files), is_streaming_mode_(is_streaming_mode), diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 8fb4adb7..4da17383 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -187,7 +187,7 @@ Result> DataEvolutionSplitRead::WrapWithBlobViewRes // use global thread number uint32_t cpu_count = std::thread::hardware_concurrency(); uint32_t thread_num = cpu_count > 0 ? cpu_count : 1; - std::shared_ptr executor = CreateDefaultExecutor(thread_num); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr executor, CreateDefaultExecutor(thread_num)); PAIMON_ASSIGN_OR_RAISE( BlobViewResolver resolver, BlobViewLookup::CreateResolver(blob_view_structs, catalog_context, pool_, executor)); diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp index 70383b22..054de2f9 100644 --- a/src/paimon/core/operation/merge_file_split_read_test.cpp +++ b/src/paimon/core/operation/merge_file_split_read_test.cpp @@ -363,7 +363,7 @@ class MergeFileSplitReadTest : public ::testing::Test, private: std::shared_ptr pool_ = GetDefaultPool(); std::shared_ptr fs_ = std::make_shared(); - std::shared_ptr executor_ = CreateDefaultExecutor(/*thread_count=*/4); + std::shared_ptr executor_ = CreateDefaultExecutor(); }; // test GenerateKeyValueReadSchema with user define fields diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index e108a10f..3f557b5f 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -159,9 +159,10 @@ class RawFileSplitReadTest : public ::testing::Test { core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), pool_)); - auto split_read = - std::make_unique(path_factory, std::move(internal_context), pool_, - CreateDefaultExecutor(/*thread_count=*/2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, + CreateDefaultExecutor(/*thread_count=*/2)); + auto split_read = std::make_unique( + path_factory, std::move(internal_context), pool_, executor); std::vector> batch_readers; batch_readers.reserve(data_splits.size()); @@ -404,9 +405,10 @@ TEST_F(RawFileSplitReadTest, TestEmptyPlan) { external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), pool_)); - auto split_read = - std::make_unique(path_factory, std::move(internal_context), pool_, - CreateDefaultExecutor(/*thread_count=*/2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, + CreateDefaultExecutor(/*thread_count=*/2)); + auto split_read = std::make_unique(path_factory, std::move(internal_context), + pool_, executor); DataSplitImpl::Builder builder(BinaryRowGenerator::GenerateRow({10, 0}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ paimon::test::GetDataDir() + @@ -435,9 +437,10 @@ TEST_F(RawFileSplitReadTest, TestMatch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr internal_context, InternalReadContext::Create(std::move(read_context), table_schema, table_schema->Options())); - auto split_read = - std::make_unique(/*path_factory=*/nullptr, std::move(internal_context), - pool_, CreateDefaultExecutor(/*thread_count=*/2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, + CreateDefaultExecutor(/*thread_count=*/2)); + auto split_read = std::make_unique( + /*path_factory=*/nullptr, std::move(internal_context), pool_, executor); auto create_data_split = [this](bool is_streaming, bool raw_convertible) -> std::shared_ptr { auto meta = std::make_shared( diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index 81fe2365..e329a35e 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -224,6 +224,9 @@ Result> ReadContextBuilder::Finish() { if (impl_->path_.empty()) { return Status::Invalid("cannot read with empty table path"); } + if (impl_->enable_prefetch_ && impl_->prefetch_max_parallel_num_ == 0) { + return Status::Invalid("prefetch max parallel num should be greater than 0"); + } if (impl_->enable_prefetch_ && impl_->prefetch_batch_count_ <= 0) { return Status::Invalid("prefetch batch count should be greater than 0"); } @@ -232,10 +235,14 @@ Result> ReadContextBuilder::Finish() { return Status::Invalid( "prefetch batch count should be greater than or equal to prefetch max parallel num"); } + if (impl_->specific_file_system_ && !impl_->fs_scheme_to_identifier_map_.empty()) { + return Status::Invalid( + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); + } if (!impl_->executor_) { // If the user do not set executor, create default executor by prefetch batch count uint32_t thread_count = impl_->enable_prefetch_ ? impl_->prefetch_max_parallel_num_ : 1; - impl_->executor_ = CreateDefaultExecutor(thread_count); + PAIMON_ASSIGN_OR_RAISE(impl_->executor_, CreateDefaultExecutor(thread_count)); } if (impl_->enable_multi_thread_row_to_batch_ && impl_->row_to_batch_thread_number_ <= 0) { diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index dc79b69e..fbe1a595 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -77,7 +77,6 @@ TEST(ReadContextTest, TestSetContent) { builder.SetTableSchema("table-schema-json"); builder.WithBranch("rt"); builder.WithCacheConfig(cache_config); - builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); auto fs = std::make_shared(); builder.WithFileSystem(fs); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -105,8 +104,7 @@ TEST(ReadContextTest, TestSetContent) { ASSERT_EQ(512U, ctx->GetCacheConfig().GetRangeSizeLimit()); ASSERT_EQ(128U, ctx->GetCacheConfig().GetHoleSizeLimit()); ASSERT_EQ(2048U, ctx->GetCacheConfig().GetPreBufferLimit()); - std::map expected_fs_map = {{"file", "local"}}; - ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_TRUE(ctx->GetFileSystemSchemeToIdentifierMap().empty()); std::map expected_options = {{"key", "value"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); @@ -123,4 +121,30 @@ TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) { ASSERT_EQ(expected_options, ctx->GetOptions()); } +TEST(ReadContextTest, TestFileSystemAndSchemeMapConflict) { + ReadContextBuilder builder("table_root_path"); + auto fs = std::make_shared(); + builder.WithFileSystem(fs); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_NOK_WITH_MSG( + builder.Finish(), + "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be used together"); +} + +TEST(ReadContextTest, TestSchemeMapWithoutFileSystem) { + ReadContextBuilder builder("table_root_path"); + builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}}); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + std::map expected_fs_map = {{"file", "local"}}; + ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); +} + +TEST(ReadContextTest, TestPrefetchMaxParallelNumZero) { + ReadContextBuilder builder("table_root_path"); + builder.EnablePrefetch(true); + builder.SetPrefetchMaxParallelNum(0); + ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num should be greater than 0"); +} + } // namespace paimon::test diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 3091dc48..def51bf3 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -2597,7 +2597,8 @@ TEST_P(GlobalIndexTest, TestBTreeWithPartitionAndCustomExecutor) { /*options=*/{}, Range(5, 7))); // Create a GlobalIndexScan with an explicit 8-thread executor - std::shared_ptr executor = CreateDefaultExecutor(/*thread_count=*/8); + ASSERT_OK_AND_ASSIGN(std::shared_ptr executor, + CreateDefaultExecutor(/*thread_count=*/8)); ASSERT_OK_AND_ASSIGN( std::shared_ptr global_index_scan, GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, From 390dd8d58a2ade651897c00ae2943dbb95b900d3 Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Sat, 13 Jun 2026 10:47:04 +0800 Subject: [PATCH 055/138] fix: small fixes for AI reviewing feedback --- .../format/parquet/file_reader_wrapper.cpp | 24 ++++++++++++------- .../page_filtered_row_group_reader.cpp | 5 ++-- .../parquet/page_filtered_row_group_reader.h | 5 ++-- .../parquet/parquet_file_batch_reader.cpp | 4 ++-- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index c3b395d2..11222963 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -107,7 +107,7 @@ Result> FileReaderWrapper::Create( std::move(file_reader), all_row_group_ranges, num_rows, batch_size, pool)); std::vector all_target_row_groups; for (int32_t i = 0; i < file_reader_wrapper->GetNumberOfRowGroups(); i++) { - all_target_row_groups.emplace_back(/*rg_index=*/i, /*page_filtered=*/false, + all_target_row_groups.emplace_back(/*rg_index=*/i, /*is_partially_matched=*/false, /*ranges=*/RowRanges()); } PAIMON_RETURN_NOT_OK( @@ -146,7 +146,7 @@ FileReaderWrapper::FileReaderWrapper( int64_t batch_size, std::shared_ptr<::arrow::MemoryPool> pool) : file_reader_(std::move(file_reader)), all_row_group_ranges_(all_row_group_ranges), - pool_(pool), + pool_(std::move(pool)), batch_size_(batch_size), num_rows_(num_rows) {} @@ -186,7 +186,7 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { if (target_row_groups_[i].excluded_by_read_range) { continue; } - uint32_t rg_id = target_row_groups_[i].row_group_index; + int32_t rg_id = target_row_groups_[i].row_group_index; uint64_t rg_start = all_row_group_ranges_[rg_id].first; uint64_t rg_end = all_row_group_ranges_[rg_id].second; if (row_number > rg_start && row_number < rg_end) { @@ -299,12 +299,13 @@ Result> FileReaderWrapper::Next() { } while (current_row_group_idx_ < target_row_groups_.size()) { - bool is_page_filtered = target_row_groups_[current_row_group_idx_].is_partially_matched; + bool is_partially_matched = + target_row_groups_[current_row_group_idx_].is_partially_matched; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, - is_page_filtered ? NextPageFiltered() : NextFullyMatched()); + is_partially_matched ? NextPageFiltered() : NextFullyMatched()); if (batch) { return batch; - } else if (!is_page_filtered) { + } else if (!is_partially_matched) { // Null from fully-matched path means batch_reader_ is globally exhausted. break; } @@ -426,13 +427,18 @@ Status FileReaderWrapper::PrepareForReading(const std::vector& t } } - bool has_page_filtered = fully_matched_row_groups.size() != active_count; - if (has_page_filtered) { + bool has_partially_matched = fully_matched_row_groups.size() != active_count; + if (has_partially_matched) { PAIMON_RETURN_NOT_OK(BuildPageFilteredSchema(column_indices)); } WaitForPendingPreBuffer(); + // TODO(Yonghao Fang): Neither Paimon nor Arrow manage the size and lifecycle of prebuffered + // caches. So when a lot of row is needed, there is possibility of OOM due to too much + // prebuffering. Also, DispatchPreBuffer will drop previous prebuffered ranges by + // GetRecordBatchReader, which cause IO wastes. + // Create standard reader for fully-matched row groups. if (!fully_matched_row_groups.empty()) { PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetRecordBatchReader( @@ -443,7 +449,7 @@ Status FileReaderWrapper::PrepareForReading(const std::vector& t // When page-filtered RGs exist, issue a single PreBuffer covering both kinds. // Otherwise GetRecordBatchReader already issued PreBuffer internally. - if (has_page_filtered) { + if (has_partially_matched) { auto all_ranges = CollectPreBufferRanges(column_indices); DispatchPreBuffer(std::move(all_ranges)); } diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index af073b13..080ba300 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -126,8 +126,9 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa } Status PageFilteredRowGroupReader::ExecuteSkipReadPattern( - std::shared_ptr<::parquet::internal::RecordReader> record_reader, const RowRanges& ranges, - int64_t total_row_count, int32_t row_group_index, int32_t column_index) { + const std::shared_ptr<::parquet::internal::RecordReader>& record_reader, + const RowRanges& ranges, int64_t total_row_count, int32_t row_group_index, + int32_t column_index) { int64_t current_row = 0; for (const auto& range : ranges.GetRanges()) { if (range.from > current_row) { diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index 30c9746a..7ff46c5a 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -91,8 +91,9 @@ class PageFilteredRowGroupReader { /// Execute the skip/read pattern on a RecordReader based on RowRanges. static Status ExecuteSkipReadPattern( - std::shared_ptr<::parquet::internal::RecordReader> record_reader, const RowRanges& ranges, - int64_t total_row_count, int32_t row_group_index, int32_t column_index); + const std::shared_ptr<::parquet::internal::RecordReader>& record_reader, + const RowRanges& ranges, int64_t total_row_count, int32_t row_group_index, + int32_t column_index); /// Create a data_page_filter callback for a column based on RowRanges + OffsetIndex. static std::function MakePageFilter( diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index cd14b837..241e65c6 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -201,11 +201,11 @@ Status ParquetFileBatchReader::SetReadSchema( for (int32_t rg_id : row_groups) { auto it = row_group_row_ranges.find(rg_id); if (it != row_group_row_ranges.end()) { - target_row_groups.emplace_back(/*rg_index=*/rg_id, /*page_filtered=*/true, + target_row_groups.emplace_back(/*rg_index=*/rg_id, /*is_partially_matched=*/true, /*ranges=*/it->second); } else { target_row_groups.emplace_back(/*rg_index=*/rg_id, - /*page_filtered=*/false, + /*is_partially_matched=*/false, /*ranges=*/RowRanges()); } } From f179c45b7c4dcb5bb48c70c43a150135f0822c22 Mon Sep 17 00:00:00 2001 From: dalingmeng <49717204+dalingmeng@users.noreply.github.com> Date: Sat, 13 Jun 2026 12:16:27 +0800 Subject: [PATCH 056/138] feat: Optimize PK MOR read performance by reduce per-row object allocation and improve merge read performance. --- src/paimon/common/data/generic_row.h | 9 ++++++ src/paimon/common/data/generic_row_test.cpp | 31 +++++++++++++++++++ .../io/key_value_data_file_record_reader.cpp | 4 +-- .../io/key_value_data_file_record_reader.h | 5 ++- .../aggregate/aggregate_merge_function.cpp | 6 +++- .../aggregate/aggregate_merge_function.h | 6 +++- .../compact/partial_update_merge_function.cpp | 12 +++++-- 7 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/paimon/common/data/generic_row.h b/src/paimon/common/data/generic_row.h index 7e5da05d..36c3f471 100644 --- a/src/paimon/common/data/generic_row.h +++ b/src/paimon/common/data/generic_row.h @@ -223,6 +223,15 @@ class GenericRow : public InternalRow { return row; } + void ResetFields() { + for (auto& field : fields_) { + field = VariantType{}; + } + kind_ = RowKind::Insert(); + row_holder_.clear(); + bytes_holder_.reset(); + } + private: /// The array to store the actual internal format values. std::vector fields_; diff --git a/src/paimon/common/data/generic_row_test.cpp b/src/paimon/common/data/generic_row_test.cpp index 3d6abdee..1aca20a7 100644 --- a/src/paimon/common/data/generic_row_test.cpp +++ b/src/paimon/common/data/generic_row_test.cpp @@ -107,4 +107,35 @@ TEST(GenericRowTest, TestSimple) { "00:00:00.100000020,123.45678998765432145678,array,row,map,null)", row.ToString()); } + +TEST(GenericRowTest, TestResetFields) { + auto pool = GetDefaultPool(); + GenericRow row(3); + row.SetField(0, static_cast(1)); + row.SetField(1, BinaryString::FromString("old", pool.get())); + row.SetField(2, Bytes::AllocateBytes("bytes", pool.get())); + row.SetRowKind(RowKind::Delete()); + + ASSERT_FALSE(row.IsNullAt(0)); + ASSERT_FALSE(row.IsNullAt(1)); + ASSERT_FALSE(row.IsNullAt(2)); + ASSERT_EQ(row.GetRowKind().value(), RowKind::Delete()); + + row.ResetFields(); + + ASSERT_EQ(row.GetFieldCount(), 3); + ASSERT_TRUE(row.IsNullAt(0)); + ASSERT_TRUE(row.IsNullAt(1)); + ASSERT_TRUE(row.IsNullAt(2)); + ASSERT_EQ(row.GetRowKind().value(), RowKind::Insert()); + + row.SetField(0, static_cast(2)); + row.SetField(1, BinaryString::FromString("new", pool.get())); + row.SetRowKind(RowKind::UpdateAfter()); + + ASSERT_EQ(row.GetInt(0), static_cast(2)); + ASSERT_EQ(row.GetString(1), BinaryString::FromString("new", pool.get())); + ASSERT_TRUE(row.IsNullAt(2)); + ASSERT_EQ(row.GetRowKind().value(), RowKind::UpdateAfter()); +} } // namespace paimon::test diff --git a/src/paimon/core/io/key_value_data_file_record_reader.cpp b/src/paimon/core/io/key_value_data_file_record_reader.cpp index 0ad68585..85bc553b 100644 --- a/src/paimon/core/io/key_value_data_file_record_reader.cpp +++ b/src/paimon/core/io/key_value_data_file_record_reader.cpp @@ -52,11 +52,11 @@ KeyValueDataFileRecordReader::KeyValueDataFileRecordReader( Result KeyValueDataFileRecordReader::Iterator::HasNext() const { int64_t array_length = reader_->row_kind_array_->length(); - const auto& selection_bitmap = reader_->selection_bitmap_; - if (selection_bitmap.Cardinality() == array_length) { + if (selection_cardinality_ == array_length) { // all rows are selected in bitmap return cursor_ < array_length; } + const auto& selection_bitmap = reader_->selection_bitmap_; auto iter = selection_bitmap.EqualOrLarger(cursor_); if (iter == selection_bitmap.End()) { // no row are selected diff --git a/src/paimon/core/io/key_value_data_file_record_reader.h b/src/paimon/core/io/key_value_data_file_record_reader.h index 5cd1f589..398b290a 100644 --- a/src/paimon/core/io/key_value_data_file_record_reader.h +++ b/src/paimon/core/io/key_value_data_file_record_reader.h @@ -57,7 +57,9 @@ class KeyValueDataFileRecordReader : public KeyValueRecordReader { class Iterator : public KeyValueRecordReader::Iterator { public: Iterator(KeyValueDataFileRecordReader* reader, int64_t previous_batch_first_row_number) - : previous_batch_first_row_number_(previous_batch_first_row_number), reader_(reader) {} + : previous_batch_first_row_number_(previous_batch_first_row_number), + reader_(reader), + selection_cardinality_(reader->selection_bitmap_.Cardinality()) {} Result HasNext() const override; Result Next() override; Result> NextWithFilePos(); @@ -66,6 +68,7 @@ class KeyValueDataFileRecordReader : public KeyValueRecordReader { int64_t previous_batch_first_row_number_; mutable int64_t cursor_ = 0; KeyValueDataFileRecordReader* reader_ = nullptr; + int64_t selection_cardinality_ = 0; }; Result> NextBatch() override; diff --git a/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.cpp b/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.cpp index 45b5f16e..aae80509 100644 --- a/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.cpp +++ b/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.cpp @@ -64,7 +64,11 @@ Status AggregateMergeFunction::Add(KeyValue&& kv) { // mark the current row for deletion and initialize the row with input values. if (remove_record_on_delete_ && kv.value_kind == RowKind::Delete()) { current_delete_row_ = true; - row_ = std::make_unique(getters_.size()); + if (row_) { + row_->ResetFields(); + } else { + row_ = std::make_unique(getters_.size()); + } for (size_t i = 0; i < getters_.size(); i++) { row_->SetField(i, getters_[i](*(kv.value))); } diff --git a/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h b/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h index 243ce6f7..7a1a07c4 100644 --- a/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h +++ b/src/paimon/core/mergetree/compact/aggregate/aggregate_merge_function.h @@ -54,7 +54,11 @@ class AggregateMergeFunction : public MergeFunction { void Reset() override { latest_kv_ = std::nullopt; current_delete_row_ = false; - row_ = std::make_unique(getters_.size()); + if (row_) { + row_->ResetFields(); + } else { + row_ = std::make_unique(getters_.size()); + } for (const auto& agg : aggregators_) { agg->Reset(); } diff --git a/src/paimon/core/mergetree/compact/partial_update_merge_function.cpp b/src/paimon/core/mergetree/compact/partial_update_merge_function.cpp index 22c1fb1a..85ed9e27 100644 --- a/src/paimon/core/mergetree/compact/partial_update_merge_function.cpp +++ b/src/paimon/core/mergetree/compact/partial_update_merge_function.cpp @@ -271,7 +271,11 @@ void PartialUpdateMergeFunction::Reset() { current_key_.reset(); meet_insert_ = false; not_null_column_filled_ = false; - row_ = std::make_unique(getters_.size()); + if (row_) { + row_->ResetFields(); + } else { + row_ = std::make_unique(getters_.size()); + } last_seq_num_ = 0; for (auto& [_, agg] : field_aggregators_) { assert(agg); @@ -306,7 +310,11 @@ Status PartialUpdateMergeFunction::Add(KeyValue&& moved_kv) { if (remove_record_on_delete_) { if (kv.value_kind == RowKind::Delete()) { current_delete_row_ = true; - row_ = std::make_unique(getters_.size()); + if (row_) { + row_->ResetFields(); + } else { + row_ = std::make_unique(getters_.size()); + } InitRowAndHoldData(std::move(kv.value)); } else if (!not_null_column_filled_) { InitRowAndHoldData(std::move(kv.value)); From dea2ed3f3a8937ca73dd04e1aa4059648e580914 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 15 Jun 2026 10:56:07 +0800 Subject: [PATCH 057/138] feat: Add manifests and files system tables --- .../utils/binary_row_partition_computer.cpp | 4 +- .../utils/binary_row_partition_computer.h | 3 +- .../core/catalog/file_system_catalog_test.cpp | 53 +- .../table/system/in_memory_system_table.cpp | 3 + .../table/system/metadata_system_tables.cpp | 503 +++++++++++++++++- .../table/system/metadata_system_tables.h | 37 ++ src/paimon/core/table/system/system_table.cpp | 18 + test/inte/read_inte_test.cpp | 271 ++++++++++ 8 files changed, 884 insertions(+), 8 deletions(-) diff --git a/src/paimon/common/utils/binary_row_partition_computer.cpp b/src/paimon/common/utils/binary_row_partition_computer.cpp index b9b0685e..43ec7d40 100644 --- a/src/paimon/common/utils/binary_row_partition_computer.cpp +++ b/src/paimon/common/utils/binary_row_partition_computer.cpp @@ -139,13 +139,13 @@ Result BinaryRowPartitionComputer::GetTypeFromArrowSchema( Result BinaryRowPartitionComputer::PartToSimpleString( const std::shared_ptr& partition_type, const BinaryRow& partition, - const std::string& delimiter, int32_t max_length) { + const std::string& delimiter, int32_t max_length, bool legacy_partition_name_enabled) { std::vector partition_converters; partition_converters.reserve(partition_type->num_fields()); for (const auto& field : partition_type->fields()) { PAIMON_ASSIGN_OR_RAISE(DataConverterUtils::BinaryRowFieldToStrConverter converter, DataConverterUtils::CreateBinaryRowFieldToStringConverter( - field->type()->id(), /*legacy_partition_name_enabled=*/true)); + field->type()->id(), legacy_partition_name_enabled)); partition_converters.emplace_back(converter); } std::vector partition_vec; diff --git a/src/paimon/common/utils/binary_row_partition_computer.h b/src/paimon/common/utils/binary_row_partition_computer.h index ae371bf5..63aa7549 100644 --- a/src/paimon/common/utils/binary_row_partition_computer.h +++ b/src/paimon/common/utils/binary_row_partition_computer.h @@ -63,7 +63,8 @@ class BinaryRowPartitionComputer { static Result PartToSimpleString( const std::shared_ptr& partition_type, const BinaryRow& partition, - const std::string& delimiter, int32_t max_length); + const std::string& delimiter, int32_t max_length, + bool legacy_partition_name_enabled = true); private: BinaryRowPartitionComputer(const std::vector& partition_keys, diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 87dbd5a7..b2f1da16 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -302,8 +302,8 @@ TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { /*ignore_if_exists=*/false)); ArrowSchemaRelease(&schema); - std::vector metadata_tables = {"snapshots", "schemas", "tags", "branches", - "consumers"}; + std::vector metadata_tables = {"snapshots", "schemas", "tags", "branches", + "consumers", "manifests", "files"}; for (const auto& table_name : metadata_tables) { Identifier system_identifier("db1", "tbl1$" + table_name); ASSERT_OK_AND_ASSIGN(bool exists, catalog.TableExists(system_identifier)); @@ -365,6 +365,55 @@ TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { (std::vector{"consumer_id", "next_snapshot_id"})); ASSERT_FALSE(consumers_arrow_schema->field(1)->nullable()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr manifests_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$manifests"))); + ASSERT_OK_AND_ASSIGN(auto manifests_c_schema, manifests_schema->GetArrowSchema()); + auto manifests_arrow_schema = arrow::ImportSchema(manifests_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(manifests_arrow_schema->field_names(), + (std::vector{"file_name", "file_size", "num_added_files", + "num_deleted_files", "schema_id", "min_partition_stats", + "max_partition_stats", "min_row_id", "max_row_id"})); + ASSERT_FALSE(manifests_arrow_schema->field(0)->nullable()); + ASSERT_EQ(manifests_arrow_schema->field(1)->type()->id(), arrow::Type::INT64); + ASSERT_FALSE(manifests_arrow_schema->field(4)->nullable()); + ASSERT_TRUE(manifests_arrow_schema->field(5)->nullable()); + ASSERT_TRUE(manifests_arrow_schema->field(8)->nullable()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr files_schema, + catalog.LoadTableSchema(Identifier("db1", "tbl1$files"))); + ASSERT_OK_AND_ASSIGN(auto files_c_schema, files_schema->GetArrowSchema()); + auto files_arrow_schema = arrow::ImportSchema(files_c_schema.get()).ValueUnsafe(); + ASSERT_EQ(files_arrow_schema->field_names(), (std::vector{"partition", + "bucket", + "file_path", + "file_format", + "schema_id", + "level", + "record_count", + "file_size_in_bytes", + "min_key", + "max_key", + "null_value_counts", + "min_value_stats", + "max_value_stats", + "min_sequence_number", + "max_sequence_number", + "creation_time", + "deleteRowCount", + "file_source", + "first_row_id", + "write_cols"})); + ASSERT_TRUE(files_arrow_schema->field(0)->nullable()); + ASSERT_FALSE(files_arrow_schema->field(1)->nullable()); + ASSERT_FALSE(files_arrow_schema->field(2)->nullable()); + ASSERT_FALSE(files_arrow_schema->field(10)->nullable()); + ASSERT_EQ(files_arrow_schema->field(15)->type()->id(), arrow::Type::TIMESTAMP); + ASSERT_EQ(files_arrow_schema->field(19)->type()->id(), arrow::Type::LIST); + auto write_cols_type = + std::dynamic_pointer_cast(files_arrow_schema->field(19)->type()); + ASSERT_TRUE(write_cols_type); + ASSERT_EQ(write_cols_type->value_type()->id(), arrow::Type::STRING); + Identifier snapshots_identifier("db1", "tbl1$snapshots"); ::ArrowSchema system_create_schema; ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); diff --git a/src/paimon/core/table/system/in_memory_system_table.cpp b/src/paimon/core/table/system/in_memory_system_table.cpp index 411eb092..91cb9eec 100644 --- a/src/paimon/core/table/system/in_memory_system_table.cpp +++ b/src/paimon/core/table/system/in_memory_system_table.cpp @@ -49,6 +49,9 @@ class InMemorySystemTableBatchReader : public BatchReader { emitted_ = true; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, table_->ArrowSchema()); PAIMON_ASSIGN_OR_RAISE(std::vector rows, table_->BuildRows()); + if (rows.empty()) { + return BatchReader::MakeEofBatch(); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr converter, GenericRowToArrowArrayConverter::Create(schema, arrow_pool_.get())); return converter->NextBatch(rows); diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 4545b357..36a4c419 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -27,22 +27,49 @@ #include #include #include +#include #include #include +#include "fmt/format.h" +#include "fmt/ranges.h" #include "paimon/common/data/binary_string.h" +#include "paimon/common/data/data_define.h" #include "paimon/common/data/generic_row.h" +#include "paimon/common/data/internal_array.h" +#include "paimon/common/data/internal_row.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/date_time_utils.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/common/utils/internal_row_utils.h" +#include "paimon/common/utils/object_utils.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/rapidjson_util.h" +#include "paimon/core/casting/cast_executor_factory.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_entry.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_file_meta.h" +#include "paimon/core/manifest/manifest_list.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" +#include "paimon/core/stats/simple_stats_evolution.h" #include "paimon/core/tag/tag.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/consumer_manager.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/core/utils/tag_manager.h" +#include "paimon/data/timestamp.h" #include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" #include "paimon/status.h" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" @@ -51,6 +78,8 @@ namespace paimon { namespace { +constexpr int32_t kMaxPartitionStatsLength = 255; + template Result JsonString(const T& value) { rapidjson::Document document; @@ -153,12 +182,290 @@ VariantType OptionalTimestampMillisValue(const std::optional& value) { MetadataSystemTableContext CreateMetadataContext(std::shared_ptr fs, std::string table_path, std::string branch) { return { - std::move(fs), - std::move(table_path), - BranchManager::NormalizeBranch(branch), + std::move(fs), std::move(table_path), BranchManager::NormalizeBranch(branch), nullptr, {}, + }; +} + +MetadataSystemTableContext CreateMetadataContext(std::shared_ptr fs, + std::string table_path, std::string branch, + std::shared_ptr table_schema, + std::map options) { + return { + std::move(fs), std::move(table_path), BranchManager::NormalizeBranch(branch), + std::move(table_schema), std::move(options), }; } +Result CreateCoreOptions(const MetadataSystemTableContext& context) { + return CoreOptions::FromMap(context.options, context.fs); +} + +Result> CreatePathFactory( + const MetadataSystemTableContext& context, const CoreOptions& core_options, + const std::shared_ptr& pool) { + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(context.table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, + core_options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + context.table_path, arrow_schema, context.table_schema->PartitionKeys(), + core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), + core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), + pool)); + return path_factory; +} + +Result> LatestSnapshot(const MetadataSystemTableContext& context) { + SnapshotManager snapshot_manager(context.fs, context.table_path, context.branch); + return snapshot_manager.LatestSnapshot(); +} + +Result> ReadDataManifests( + const MetadataSystemTableContext& context, const Snapshot& snapshot, + const std::shared_ptr& path_factory, const CoreOptions& core_options, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr manifest_list, + ManifestList::Create(context.fs, core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, pool)); + std::vector manifests; + // TODO(suxiaogang223): Align Java ReadAllManifests semantics by including changelog + // manifests. ReadAllManifests currently delegates to ReadChangelogManifests, which returns + // NotImplemented when a snapshot has a changelog manifest list. + PAIMON_RETURN_NOT_OK(manifest_list->ReadDataManifests(snapshot, &manifests)); + return manifests; +} + +Result> CreateManifestFile( + const MetadataSystemTableContext& context, + const std::shared_ptr& path_factory, const CoreOptions& core_options, + const std::shared_ptr& pool) { + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(context.table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, context.table_schema->PartitionKeys())); + return ManifestFile::Create(context.fs, core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, + core_options.GetManifestTargetFileSize(), pool, core_options, + partition_schema); +} + +Result> ReadLatestManifestEntries( + const MetadataSystemTableContext& context, + const std::shared_ptr& path_factory, const CoreOptions& core_options, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, LatestSnapshot(context)); + if (!snapshot) { + return std::vector(); + } + PAIMON_ASSIGN_OR_RAISE( + std::vector manifests, + ReadDataManifests(context, snapshot.value(), path_factory, core_options, pool)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_file, + CreateManifestFile(context, path_factory, core_options, pool)); + std::vector entries; + for (const auto& manifest : manifests) { + PAIMON_RETURN_NOT_OK( + manifest_file->Read(manifest.FileName(), /*filter=*/nullptr, &entries)); + } + return entries; +} + +Result> ReadLatestDataFiles( + const MetadataSystemTableContext& context, + const std::shared_ptr& path_factory, const CoreOptions& core_options, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::vector entries, + ReadLatestManifestEntries(context, path_factory, core_options, pool)); + std::vector merged_entries; + PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(entries, &merged_entries)); + return merged_entries; +} + +Result> OptionalPartitionString( + const BinaryRow& row, const std::shared_ptr& partition_schema) { + if (row.GetFieldCount() <= 0) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(std::string value, + BinaryRowPartitionComputer::PartToSimpleString( + partition_schema, row, ",", kMaxPartitionStatsLength, + /*legacy_partition_name_enabled=*/false)); + return std::optional(fmt::format("{{{}}}", value)); +} + +Result OptionalPartitionStringValue( + const BinaryRow& row, const std::shared_ptr& partition_schema) { + PAIMON_ASSIGN_OR_RAISE(std::optional value, + OptionalPartitionString(row, partition_schema)); + return OptionalStringValue(value); +} + +Result FilePath(const std::shared_ptr& path_factory, + const ManifestEntry& entry, const DataFileMeta& file) { + if (file.external_path) { + return file.external_path.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::string bucket_path, + path_factory->BucketPath(entry.Partition(), entry.Bucket())); + return PathUtil::JoinPath(bucket_path, file.file_name); +} + +Result FieldValueString(const DataField& field, const VariantType& value) { + PAIMON_ASSIGN_OR_RAISE(FieldType field_type, + FieldTypeUtils::ConvertToFieldType(field.Type()->id())); + std::shared_ptr cast_executor = + CastExecutorFactory::GetCastExecutorFactory()->GetCastExecutor(field_type, + FieldType::STRING); + if (!cast_executor) { + return DataDefine::VariantValueToString(value); + } + PAIMON_ASSIGN_OR_RAISE(Literal literal, + DataDefine::VariantValueToLiteral(value, field.Type()->id())); + PAIMON_ASSIGN_OR_RAISE(Literal string_literal, cast_executor->Cast(literal, arrow::utf8())); + return string_literal.GetValue(); +} + +Result> RowValueStrings(const std::vector& fields, + const InternalRow& row) { + std::shared_ptr schema = DataField::ConvertDataFieldsToArrowSchema(fields); + PAIMON_ASSIGN_OR_RAISE(std::vector getters, + InternalRowUtils::CreateFieldGetters(schema, /*use_view=*/false)); + std::vector values; + int32_t length = std::min(static_cast(fields.size()), row.GetFieldCount()); + values.reserve(length); + for (int32_t i = 0; i < length; ++i) { + std::string value = "null"; + if (!row.IsNullAt(i)) { + VariantType field_value = getters[i](row); + PAIMON_ASSIGN_OR_RAISE(value, FieldValueString(fields[i], field_value)); + } + values.push_back(std::move(value)); + } + return values; +} + +Result RowValuesString(const std::vector& fields, const InternalRow& row, + std::string_view left, std::string_view right) { + PAIMON_ASSIGN_OR_RAISE(std::vector values, RowValueStrings(fields, row)); + return fmt::format("{}{}{}", left, fmt::join(values, ", "), right); +} + +Result> OptionalRowValuesString(const std::vector& fields, + const InternalRow& row, + std::string_view left, + std::string_view right) { + if (row.GetFieldCount() <= 0) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(std::string value, RowValuesString(fields, row, left, right)); + return std::optional(value); +} + +Result FieldsValueMapString(const std::vector& fields, + const InternalRow& row) { + PAIMON_ASSIGN_OR_RAISE(std::vector values, RowValueStrings(fields, row)); + std::vector> field_values; + size_t length = std::min(fields.size(), values.size()); + field_values.reserve(length); + for (size_t i = 0; i < length; ++i) { + field_values.emplace_back(fields[i].Name(), std::move(values[i])); + } + std::sort(field_values.begin(), field_values.end(), + [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; }); + + std::vector entries; + entries.reserve(field_values.size()); + for (const auto& [name, value] : field_values) { + entries.emplace_back(fmt::format("{}={}", name, value)); + } + return fmt::format("{{{}}}", fmt::join(entries, ", ")); +} + +Result NullValueCountsString(const std::vector& fields, + const InternalArray& null_counts) { + std::vector> field_values; + int32_t length = std::min(static_cast(fields.size()), null_counts.Size()); + field_values.reserve(length); + for (int32_t i = 0; i < length; ++i) { + std::string value = + null_counts.IsNullAt(i) ? "null" : std::to_string(null_counts.GetLong(i)); + field_values.emplace_back(fields[i].Name(), std::move(value)); + } + std::sort(field_values.begin(), field_values.end(), + [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; }); + + std::vector entries; + entries.reserve(field_values.size()); + for (const auto& [name, value] : field_values) { + entries.emplace_back(fmt::format("{}={}", name, value)); + } + return fmt::format("{{{}}}", fmt::join(entries, ", ")); +} + +Result> LoadDataSchema(const MetadataSystemTableContext& context, + int64_t schema_id) { + if (schema_id == context.table_schema->Id()) { + return context.table_schema; + } + SchemaManager schema_manager(context.fs, context.table_path, context.branch); + return schema_manager.ReadSchema(schema_id); +} + +Result> ProjectWriteFields(const std::shared_ptr& data_schema, + const DataFileMeta& file) { + if (!file.write_cols) { + return data_schema->Fields(); + } + + std::vector fields; + fields.reserve(file.write_cols->size() + data_schema->PartitionKeys().size()); + for (const auto& write_col : file.write_cols.value()) { + if (SpecialFields::IsSpecialFieldName(write_col)) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(DataField field, data_schema->GetField(write_col)); + fields.push_back(std::move(field)); + } + + // Partial writes may omit partition columns from write_cols. Keep them in the stats source + // fields so SimpleStatsEvolution can map partition stats consistently. + for (const auto& partition_key : data_schema->PartitionKeys()) { + if (!ObjectUtils::Contains(file.write_cols.value(), partition_key)) { + PAIMON_ASSIGN_OR_RAISE(DataField field, data_schema->GetField(partition_key)); + fields.push_back(std::move(field)); + } + } + return fields; +} + +Result> KeyFieldsForFilesTable( + const std::shared_ptr& data_schema) { + PAIMON_ASSIGN_OR_RAISE(std::vector key_fields, + data_schema->TrimmedPrimaryKeyFields()); + // Java FilesTable falls back to logicalRowType when logicalTrimmedPrimaryKeysType is empty. + if (key_fields.empty()) { + return data_schema->Fields(); + } + return key_fields; +} + +Result> WriteColsValue( + const std::optional>& write_cols, + const std::shared_ptr& pool) { + if (!write_cols) { + return std::shared_ptr(); + } + return std::make_shared( + InternalRowUtils::ToNotNullStringArrayData(write_cols.value(), pool)); +} + } // namespace OptionsSystemTable::OptionsSystemTable(std::string table_path, @@ -427,4 +734,194 @@ Result> ConsumersSystemTable::BuildRows() const { return rows; } +ManifestsSystemTable::ManifestsSystemTable(std::shared_ptr fs, std::string table_path, + std::string branch, + std::shared_ptr table_schema, + std::map options) + : InMemorySystemTable(table_path), + context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), + std::move(table_schema), std::move(options))) {} + +std::string ManifestsSystemTable::Name() const { + return kName; +} + +Result> ManifestsSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("file_name", arrow::utf8(), /*nullable=*/false), + arrow::field("file_size", arrow::int64(), /*nullable=*/false), + arrow::field("num_added_files", arrow::int64(), /*nullable=*/false), + arrow::field("num_deleted_files", arrow::int64(), /*nullable=*/false), + arrow::field("schema_id", arrow::int64(), /*nullable=*/false), + arrow::field("min_partition_stats", arrow::utf8(), /*nullable=*/true), + arrow::field("max_partition_stats", arrow::utf8(), /*nullable=*/true), + arrow::field("min_row_id", arrow::int64(), /*nullable=*/true), + arrow::field("max_row_id", arrow::int64(), /*nullable=*/true), + }); +} + +Result> ManifestsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, LatestSnapshot(context_)); + if (!snapshot) { + return std::vector(); + } + + std::shared_ptr pool = GetDefaultPool(); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CreateCoreOptions(context_)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + CreatePathFactory(context_, core_options, pool)); + PAIMON_ASSIGN_OR_RAISE( + std::vector manifests, + ReadDataManifests(context_, snapshot.value(), path_factory, core_options, pool)); + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(context_.table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, context_.table_schema->PartitionKeys())); + + std::vector rows; + rows.reserve(manifests.size()); + for (const auto& manifest : manifests) { + GenericRow row(schema->num_fields()); + row.SetField(0, StringValue(manifest.FileName())); + row.SetField(1, manifest.FileSize()); + row.SetField(2, manifest.NumAddedFiles()); + row.SetField(3, manifest.NumDeletedFiles()); + row.SetField(4, manifest.SchemaId()); + PAIMON_ASSIGN_OR_RAISE( + VariantType min_partition, + OptionalPartitionStringValue(manifest.PartitionStats().MinValues(), partition_schema)); + PAIMON_ASSIGN_OR_RAISE( + VariantType max_partition, + OptionalPartitionStringValue(manifest.PartitionStats().MaxValues(), partition_schema)); + row.SetField(5, min_partition); + row.SetField(6, max_partition); + row.SetField(7, OptionalInt64Value(manifest.MinRowId())); + row.SetField(8, OptionalInt64Value(manifest.MaxRowId())); + rows.push_back(std::move(row)); + } + return rows; +} + +FilesSystemTable::FilesSystemTable(std::shared_ptr fs, std::string table_path, + std::string branch, std::shared_ptr table_schema, + std::map options) + : InMemorySystemTable(table_path), + context_(CreateMetadataContext(std::move(fs), std::move(table_path), std::move(branch), + std::move(table_schema), std::move(options))) {} + +std::string FilesSystemTable::Name() const { + return kName; +} + +Result> FilesSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("partition", arrow::utf8(), /*nullable=*/true), + arrow::field("bucket", arrow::int32(), /*nullable=*/false), + arrow::field("file_path", arrow::utf8(), /*nullable=*/false), + arrow::field("file_format", arrow::utf8(), /*nullable=*/false), + arrow::field("schema_id", arrow::int64(), /*nullable=*/false), + arrow::field("level", arrow::int32(), /*nullable=*/false), + arrow::field("record_count", arrow::int64(), /*nullable=*/false), + arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/false), + arrow::field("min_key", arrow::utf8(), /*nullable=*/true), + arrow::field("max_key", arrow::utf8(), /*nullable=*/true), + arrow::field("null_value_counts", arrow::utf8(), /*nullable=*/false), + arrow::field("min_value_stats", arrow::utf8(), /*nullable=*/false), + arrow::field("max_value_stats", arrow::utf8(), /*nullable=*/false), + arrow::field("min_sequence_number", arrow::int64(), /*nullable=*/true), + arrow::field("max_sequence_number", arrow::int64(), /*nullable=*/true), + arrow::field("creation_time", arrow::timestamp(arrow::TimeUnit::MILLI), + /*nullable=*/true), + arrow::field("deleteRowCount", arrow::int64(), /*nullable=*/true), + arrow::field("file_source", arrow::utf8(), /*nullable=*/true), + arrow::field("first_row_id", arrow::int64(), /*nullable=*/true), + arrow::field("write_cols", arrow::list(arrow::utf8()), /*nullable=*/true), + }); +} + +Result> FilesSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::shared_ptr pool = GetDefaultPool(); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CreateCoreOptions(context_)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr path_factory, + CreatePathFactory(context_, core_options, pool)); + PAIMON_ASSIGN_OR_RAISE(std::vector entries, + ReadLatestDataFiles(context_, path_factory, core_options, pool)); + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(context_.table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, context_.table_schema->PartitionKeys())); + const std::vector& value_stats_fields = context_.table_schema->Fields(); + + std::vector rows; + rows.reserve(entries.size()); + for (const auto& entry : entries) { + if (!(entry.Kind() == FileKind::Add())) { + continue; + } + + const std::shared_ptr& file = entry.File(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_schema, + LoadDataSchema(context_, file->schema_id)); + PAIMON_ASSIGN_OR_RAISE(std::vector data_stats_fields, + ProjectWriteFields(data_schema, *file)); + PAIMON_ASSIGN_OR_RAISE(std::vector key_fields, + KeyFieldsForFilesTable(data_schema)); + auto stats_evolution = std::make_shared( + data_stats_fields, value_stats_fields, + data_schema->Id() != context_.table_schema->Id() || file->write_cols.has_value(), pool); + PAIMON_ASSIGN_OR_RAISE( + SimpleStatsEvolution::EvolutionStats stats, + stats_evolution->Evolution(file->value_stats, file->row_count, file->value_stats_cols)); + + GenericRow row(schema->num_fields()); + if (context_.table_schema->PartitionKeys().empty()) { + row.SetField(0, NullType()); + } else { + PAIMON_ASSIGN_OR_RAISE(VariantType partition, OptionalPartitionStringValue( + entry.Partition(), partition_schema)); + row.SetField(0, partition); + } + row.SetField(1, entry.Bucket()); + PAIMON_ASSIGN_OR_RAISE(std::string file_path, FilePath(path_factory, entry, *file)); + row.SetField(2, StringValue(file_path)); + PAIMON_ASSIGN_OR_RAISE(std::string file_format, file->FileFormat()); + row.SetField(3, StringValue(file_format)); + row.SetField(4, file->schema_id); + row.SetField(5, file->level); + row.SetField(6, file->row_count); + row.SetField(7, file->file_size); + PAIMON_ASSIGN_OR_RAISE(std::optional min_key, + OptionalRowValuesString(key_fields, file->min_key, "[", "]")); + PAIMON_ASSIGN_OR_RAISE(std::optional max_key, + OptionalRowValuesString(key_fields, file->max_key, "[", "]")); + row.SetField(8, OptionalStringValue(min_key)); + row.SetField(9, OptionalStringValue(max_key)); + PAIMON_ASSIGN_OR_RAISE(std::string null_value_counts, + NullValueCountsString(value_stats_fields, *stats.null_counts)); + row.SetField(10, StringValue(null_value_counts)); + PAIMON_ASSIGN_OR_RAISE(std::string min_value_stats, + FieldsValueMapString(value_stats_fields, *stats.min_values)); + row.SetField(11, StringValue(min_value_stats)); + PAIMON_ASSIGN_OR_RAISE(std::string max_value_stats, + FieldsValueMapString(value_stats_fields, *stats.max_values)); + row.SetField(12, StringValue(max_value_stats)); + row.SetField(13, file->min_sequence_number); + row.SetField(14, file->max_sequence_number); + row.SetField(15, TimestampMillisValue(file->creation_time.GetMillisecond())); + row.SetField(16, OptionalInt64Value(file->delete_row_count)); + row.SetField(17, file->file_source ? StringValue(file->file_source.value().ToString()) + : VariantType(NullType())); + row.SetField(18, OptionalInt64Value(file->first_row_id)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr write_cols, + WriteColsValue(file->write_cols, pool)); + row.SetField(19, write_cols ? VariantType(write_cols) : VariantType(NullType())); + rows.push_back(std::move(row)); + } + return rows; +} + } // namespace paimon diff --git a/src/paimon/core/table/system/metadata_system_tables.h b/src/paimon/core/table/system/metadata_system_tables.h index c2803538..e3961e47 100644 --- a/src/paimon/core/table/system/metadata_system_tables.h +++ b/src/paimon/core/table/system/metadata_system_tables.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include #include @@ -49,6 +50,8 @@ struct MetadataSystemTableContext { std::shared_ptr fs; std::string table_path; std::string branch; + std::shared_ptr table_schema; + std::map options; }; /// System table for `T$snapshots`, exposing snapshot commit history. @@ -128,4 +131,38 @@ class ConsumersSystemTable : public InMemorySystemTable { MetadataSystemTableContext context_; }; +/// System table for `T$manifests`, exposing data manifest metadata in the latest snapshot. +class ManifestsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "manifests"; + + ManifestsSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, + std::shared_ptr table_schema, + std::map options); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + MetadataSystemTableContext context_; +}; + +/// System table for `T$files`, exposing data file metadata in the latest snapshot. +class FilesSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "files"; + + FilesSystemTable(std::shared_ptr fs, std::string table_path, std::string branch, + std::shared_ptr table_schema, + std::map options); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + MetadataSystemTableContext context_; +}; + } // namespace paimon diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 52d60f61..bbbb5b98 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -129,6 +129,24 @@ const std::vector& SystemTableRegistry() { auto options = MergeOptions(table_schema, dynamic_options); return std::make_shared(fs, table_path, LoadBranch(options)); }}, + {ManifestsSystemTable::kName, + [](const std::shared_ptr& fs, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + auto options = MergeOptions(table_schema, dynamic_options); + return std::make_shared(fs, table_path, LoadBranch(options), + table_schema, std::move(options)); + }}, + {FilesSystemTable::kName, + [](const std::shared_ptr& fs, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + auto options = MergeOptions(table_schema, dynamic_options); + return std::make_shared(fs, table_path, LoadBranch(options), + table_schema, std::move(options)); + }}, }; return registry; } diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 685159d0..0958014b 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -701,6 +701,277 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTables) { auto branch_create_time_array = std::dynamic_pointer_cast(branches_array->field(1)); ASSERT_TRUE(branch_create_time_array); + + ASSERT_OK_AND_ASSIGN(auto manifests_result, + ReadSystemTable(table_path + "$manifests", options)); + auto manifests_array = SingleStructChunk(manifests_result); + ASSERT_EQ(StructFieldNames(manifests_array), + (std::vector{"file_name", "file_size", "num_added_files", + "num_deleted_files", "schema_id", "min_partition_stats", + "max_partition_stats", "min_row_id", "max_row_id"})); + ASSERT_GT(manifests_array->length(), 0); + auto manifest_file_name_array = + std::dynamic_pointer_cast(manifests_array->field(0)); + auto manifest_file_size_array = + std::dynamic_pointer_cast(manifests_array->field(1)); + auto manifest_num_added_files_array = + std::dynamic_pointer_cast(manifests_array->field(2)); + auto manifest_schema_id_array = + std::dynamic_pointer_cast(manifests_array->field(4)); + ASSERT_TRUE(manifest_file_name_array); + ASSERT_TRUE(manifest_file_size_array); + ASSERT_TRUE(manifest_num_added_files_array); + ASSERT_TRUE(manifest_schema_id_array); + ASSERT_EQ(manifest_file_name_array->GetString(0).find("manifest-"), 0); + ASSERT_GT(manifest_file_size_array->Value(0), 0); + ASSERT_GE(manifest_num_added_files_array->Value(0), 1); + ASSERT_EQ(manifest_schema_id_array->Value(0), 0); + + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(StructFieldNames(files_array), (std::vector{"partition", + "bucket", + "file_path", + "file_format", + "schema_id", + "level", + "record_count", + "file_size_in_bytes", + "min_key", + "max_key", + "null_value_counts", + "min_value_stats", + "max_value_stats", + "min_sequence_number", + "max_sequence_number", + "creation_time", + "deleteRowCount", + "file_source", + "first_row_id", + "write_cols"})); + ASSERT_GT(files_array->length(), 0); + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + auto bucket_array = std::dynamic_pointer_cast(files_array->field(1)); + auto file_path_array = std::dynamic_pointer_cast(files_array->field(2)); + auto file_format_array = std::dynamic_pointer_cast(files_array->field(3)); + auto file_schema_id_array = std::dynamic_pointer_cast(files_array->field(4)); + auto record_count_array = std::dynamic_pointer_cast(files_array->field(6)); + auto file_size_array = std::dynamic_pointer_cast(files_array->field(7)); + auto min_sequence_number_array = + std::dynamic_pointer_cast(files_array->field(13)); + auto max_sequence_number_array = + std::dynamic_pointer_cast(files_array->field(14)); + auto creation_time_array = + std::dynamic_pointer_cast(files_array->field(15)); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(bucket_array); + ASSERT_TRUE(file_path_array); + ASSERT_TRUE(file_format_array); + ASSERT_TRUE(file_schema_id_array); + ASSERT_TRUE(record_count_array); + ASSERT_TRUE(file_size_array); + ASSERT_TRUE(min_sequence_number_array); + ASSERT_TRUE(max_sequence_number_array); + ASSERT_TRUE(creation_time_array); + ASSERT_TRUE(partition_array->IsNull(0)); + ASSERT_EQ(bucket_array->Value(0), 0); + ASSERT_NE(file_path_array->GetString(0).find("/bucket-0/"), std::string::npos); + ASSERT_EQ(file_format_array->GetString(0), "parquet"); + ASSERT_EQ(file_schema_id_array->Value(0), 0); + ASSERT_EQ(record_count_array->Value(0), 1); + ASSERT_GT(file_size_array->Value(0), 0); + ASSERT_GE(min_sequence_number_array->Value(0), 0); + ASSERT_GE(max_sequence_number_array->Value(0), min_sequence_number_array->Value(0)); + ASSERT_FALSE(creation_time_array->IsNull(0)); +} + +TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedTable) { + arrow::FieldVector fields = { + arrow::field("dt", arrow::utf8()), + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{"dt", "pk"}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["20260527", "a", 1]])", + /*partition_map=*/{{"dt", "20260527"}}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(files_array->length(), 1); + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + auto file_path_array = std::dynamic_pointer_cast(files_array->field(2)); + auto min_key_array = std::dynamic_pointer_cast(files_array->field(8)); + auto max_key_array = std::dynamic_pointer_cast(files_array->field(9)); + auto null_value_counts_array = + std::dynamic_pointer_cast(files_array->field(10)); + auto min_value_stats_array = + std::dynamic_pointer_cast(files_array->field(11)); + auto max_value_stats_array = + std::dynamic_pointer_cast(files_array->field(12)); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(file_path_array); + ASSERT_TRUE(min_key_array); + ASSERT_TRUE(max_key_array); + ASSERT_TRUE(null_value_counts_array); + ASSERT_TRUE(min_value_stats_array); + ASSERT_TRUE(max_value_stats_array); + ASSERT_EQ(partition_array->GetString(0), "{20260527}"); + ASSERT_NE(file_path_array->GetString(0).find("/dt=20260527/bucket-0/"), std::string::npos); + ASSERT_EQ(min_key_array->GetString(0), "[a]"); + ASSERT_EQ(max_key_array->GetString(0), "[a]"); + ASSERT_EQ(null_value_counts_array->GetString(0), "{dt=0, pk=0, v=0}"); + ASSERT_EQ(min_value_stats_array->GetString(0), "{dt=20260527, pk=a, v=1}"); + ASSERT_EQ(max_value_stats_array->GetString(0), "{dt=20260527, pk=a, v=1}"); +} + +TEST(SystemTableReadInteTest, TestReadFilesSystemTableForDatePartition) { + arrow::FieldVector fields = { + arrow::field("dt", arrow::date32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[10440, 1]])", + /*partition_map=*/{{"dt", "1998-08-02"}}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(files_array->length(), 1); + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + ASSERT_TRUE(partition_array); + ASSERT_EQ(partition_array->GetString(0), "{1998-08-02}"); +} + +TEST(SystemTableReadInteTest, TestReadFilesSystemTableWithSchemaEvolutionStats) { + std::map options = {{Options::FILE_SYSTEM, "local"}}; + std::string table_path = paimon::test::GetDataDir() + + "/orc/append_table_with_alter_table_with_dense_field.db/" + "append_table_with_alter_table_with_dense_field"; + + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + auto files_array = SingleStructChunk(files_result); + ASSERT_EQ(StructFieldNames(files_array), (std::vector{"partition", + "bucket", + "file_path", + "file_format", + "schema_id", + "level", + "record_count", + "file_size_in_bytes", + "min_key", + "max_key", + "null_value_counts", + "min_value_stats", + "max_value_stats", + "min_sequence_number", + "max_sequence_number", + "creation_time", + "deleteRowCount", + "file_source", + "first_row_id", + "write_cols"})); + ASSERT_GT(files_array->length(), 0); + + auto partition_array = std::dynamic_pointer_cast(files_array->field(0)); + auto schema_id_array = std::dynamic_pointer_cast(files_array->field(4)); + auto null_value_counts_array = + std::dynamic_pointer_cast(files_array->field(10)); + auto min_value_stats_array = + std::dynamic_pointer_cast(files_array->field(11)); + auto max_value_stats_array = + std::dynamic_pointer_cast(files_array->field(12)); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(schema_id_array); + ASSERT_TRUE(null_value_counts_array); + ASSERT_TRUE(min_value_stats_array); + ASSERT_TRUE(max_value_stats_array); + + bool found_old_schema_file = false; + bool found_latest_schema_file = false; + for (int64_t i = 0; i < files_array->length(); ++i) { + std::string partition = partition_array->GetString(i); + ASSERT_TRUE(partition == "{0}" || partition == "{1}"); + + std::string null_value_counts = null_value_counts_array->GetString(i); + std::string min_value_stats = min_value_stats_array->GetString(i); + std::string max_value_stats = max_value_stats_array->GetString(i); + ASSERT_NE(null_value_counts.find("f4="), std::string::npos); + ASSERT_NE(min_value_stats.find("f4="), std::string::npos); + ASSERT_NE(max_value_stats.find("f4="), std::string::npos); + ASSERT_EQ(null_value_counts.find("f0="), std::string::npos); + ASSERT_EQ(min_value_stats.find("f0="), std::string::npos); + ASSERT_EQ(max_value_stats.find("f0="), std::string::npos); + + if (schema_id_array->Value(i) == 0) { + found_old_schema_file = true; + ASSERT_NE(null_value_counts.find("f4="), std::string::npos); + ASSERT_NE(min_value_stats.find("f4=null"), std::string::npos); + ASSERT_NE(max_value_stats.find("f4=null"), std::string::npos); + } else if (schema_id_array->Value(i) == 1) { + found_latest_schema_file = true; + } + } + ASSERT_TRUE(found_old_schema_file); + ASSERT_TRUE(found_latest_schema_file); +} + +TEST(SystemTableReadInteTest, TestReadManifestAndFilesSystemTablesForEmptyTable) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + ASSERT_OK(catalog->CreateDatabase("db1", options, /*ignore_if_exists=*/false)); + + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("db1", "tbl1"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(std::string table_path, + catalog->GetTableLocation(Identifier("db1", "tbl1"))); + ASSERT_OK_AND_ASSIGN(auto manifests_result, + ReadSystemTable(table_path + "$manifests", options)); + ASSERT_EQ(manifests_result.array, nullptr); + ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options)); + ASSERT_EQ(files_result.array, nullptr); } TEST(SystemTableReadInteTest, TestReadTagBranchAndConsumerSystemTables) { From 868d9db30adb3bdb9151aa741420b4a548856f0f Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Mon, 15 Jun 2026 20:55:17 +0800 Subject: [PATCH 058/138] fix: canonicalize NaN for Hive bucket hash --- .../bucket/default_bucket_function_test.cpp | 55 +++++++++++++++++++ .../core/bucket/hive_bucket_function.cpp | 4 ++ .../core/bucket/hive_bucket_function_test.cpp | 51 +++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/src/paimon/core/bucket/default_bucket_function_test.cpp b/src/paimon/core/bucket/default_bucket_function_test.cpp index 16d18ca3..c02eadab 100644 --- a/src/paimon/core/bucket/default_bucket_function_test.cpp +++ b/src/paimon/core/bucket/default_bucket_function_test.cpp @@ -18,14 +18,28 @@ #include "paimon/core/bucket/default_bucket_function.h" +#include + #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { + +void CheckDefaultBucket(const DefaultBucketFunction& func, const BinaryRow& row, + int32_t expected_hash, int32_t expected_bucket) { + constexpr int32_t kNumBuckets = 1000; + ASSERT_EQ(expected_hash, row.HashCode()); + ASSERT_EQ(expected_bucket, func.Bucket(row, kNumBuckets)); +} + +} // namespace + TEST(DefaultBucketFunctionTest, TestBasicHashMod) { auto pool = GetDefaultPool(); DefaultBucketFunction func; @@ -81,4 +95,45 @@ TEST(DefaultBucketFunctionTest, TestMultiFieldRow) { ASSERT_EQ(std::abs(row.HashCode() % num_buckets), bucket); } +TEST(DefaultBucketFunctionTest, TestFloatSpecialValuesCompatibleWithJava) { + auto pool = GetDefaultPool(); + DefaultBucketFunction func; + + // Verified with Java DefaultBucketFunction and NUM_BUCKETS = 1000. + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({std::numeric_limits::quiet_NaN()}, pool.get()), + -2039172089, 89); + CheckDefaultBucket( + func, BinaryRowGenerator::GenerateRow({std::numeric_limits::infinity()}, pool.get()), + 2139216202, 202); + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({-std::numeric_limits::infinity()}, pool.get()), + -106221671, 671); + CheckDefaultBucket(func, BinaryRowGenerator::GenerateRow({0.0f}, pool.get()), -300363099, 99); + CheckDefaultBucket(func, BinaryRowGenerator::GenerateRow({-0.0f}, pool.get()), 916225219, 219); +} + +TEST(DefaultBucketFunctionTest, TestDoubleSpecialValuesCompatibleWithJava) { + auto pool = GetDefaultPool(); + DefaultBucketFunction func; + + // Verified with Java DefaultBucketFunction and NUM_BUCKETS = 1000. + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({std::numeric_limits::quiet_NaN()}, pool.get()), + -1323214697, 697); + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({std::numeric_limits::infinity()}, pool.get()), + -1556713404, 404); + CheckDefaultBucket( + func, + BinaryRowGenerator::GenerateRow({-std::numeric_limits::infinity()}, pool.get()), + -2079171840, 840); + CheckDefaultBucket(func, BinaryRowGenerator::GenerateRow({0.0}, pool.get()), -300363099, 99); + CheckDefaultBucket(func, BinaryRowGenerator::GenerateRow({-0.0}, pool.get()), 302122119, 119); +} + } // namespace paimon::test diff --git a/src/paimon/core/bucket/hive_bucket_function.cpp b/src/paimon/core/bucket/hive_bucket_function.cpp index 7bcf004a..913053c1 100644 --- a/src/paimon/core/bucket/hive_bucket_function.cpp +++ b/src/paimon/core/bucket/hive_bucket_function.cpp @@ -105,6 +105,8 @@ uint32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_ind uint32_t bits; if (float_value == -0.0f) { bits = 0; + } else if (std::isnan(float_value)) { + bits = 0x7FC00000U; } else { std::memcpy(&bits, &float_value, sizeof(bits)); } @@ -115,6 +117,8 @@ uint32_t HiveBucketFunction::ComputeHash(const BinaryRow& row, int32_t field_ind uint64_t bits; if (double_value == -0.0) { bits = 0; + } else if (std::isnan(double_value)) { + bits = 0x7FF8000000000000ULL; } else { std::memcpy(&bits, &double_value, sizeof(bits)); } diff --git a/src/paimon/core/bucket/hive_bucket_function_test.cpp b/src/paimon/core/bucket/hive_bucket_function_test.cpp index b6c948c5..c2b7971b 100644 --- a/src/paimon/core/bucket/hive_bucket_function_test.cpp +++ b/src/paimon/core/bucket/hive_bucket_function_test.cpp @@ -18,6 +18,7 @@ #include "paimon/core/bucket/hive_bucket_function.h" +#include #include #include "gtest/gtest.h" @@ -100,6 +101,23 @@ class HiveBucketFunctionTest : public ::testing::Test { auto pool = GetDefaultPool(); return BinaryRowGenerator::GenerateRow({value}, pool.get()); } + + BinaryRow CreateByteRow(int8_t value) { + auto pool = GetDefaultPool(); + return BinaryRowGenerator::GenerateRow({value}, pool.get()); + } + + float FloatFromBits(uint32_t bits) { + float value; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } + + double DoubleFromBits(uint64_t bits) { + double value; + std::memcpy(&value, &bits, sizeof(value)); + return value; + } }; /// Test matching Java: testHiveBucketFunction @@ -207,6 +225,39 @@ TEST_F(HiveBucketFunctionTest, TestDoubleNegativeZero) { ASSERT_EQ(func->Bucket(CreateDoubleRow(0.0), 5), func->Bucket(CreateDoubleRow(-0.0), 5)); } +TEST_F(HiveBucketFunctionTest, TestFloatNaNCanonicalizationCompatibleWithJava) { + std::vector field_types = {FieldType::FLOAT}; + ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_types)); + + // Verified with Java HiveBucketFunction: + // Float.NaN, Float.intBitsToFloat(0x7fa12345), and Float.intBitsToFloat(0x7fc00000) + // all hash through Float.floatToIntBits(...) = 0x7fc00000. + ASSERT_EQ(344, func->Bucket(CreateFloatRow(std::numeric_limits::quiet_NaN()), 1000)); + ASSERT_EQ(344, func->Bucket(CreateFloatRow(FloatFromBits(0x7FA12345U)), 1000)); + ASSERT_EQ(344, func->Bucket(CreateFloatRow(FloatFromBits(0x7FC00000U)), 1000)); +} + +TEST_F(HiveBucketFunctionTest, TestDoubleNaNCanonicalizationCompatibleWithJava) { + std::vector field_types = {FieldType::DOUBLE}; + ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_types)); + + // Verified with Java HiveBucketFunction: + // Double.NaN, Double.longBitsToDouble(0x7ff123456789abcd), and canonical NaN + // all hash through Double.doubleToLongBits(...) = 0x7ff8000000000000. + ASSERT_EQ(360, func->Bucket(CreateDoubleRow(std::numeric_limits::quiet_NaN()), 1000)); + ASSERT_EQ(360, func->Bucket(CreateDoubleRow(DoubleFromBits(0x7FF123456789ABCDULL)), 1000)); + ASSERT_EQ(360, func->Bucket(CreateDoubleRow(DoubleFromBits(0x7FF8000000000000ULL)), 1000)); +} + +TEST_F(HiveBucketFunctionTest, TestTinyintNegativeValuesCompatibleWithJava) { + std::vector field_types = {FieldType::TINYINT}; + ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_types)); + + // Verified with Java HiveBucketFunction using DataTypes.TINYINT(). + ASSERT_EQ(647, func->Bucket(CreateByteRow(static_cast(-1)), 1000)); + ASSERT_EQ(520, func->Bucket(CreateByteRow(std::numeric_limits::min()), 1000)); +} + /// Test STRING field TEST_F(HiveBucketFunctionTest, TestStringField) { std::vector field_types = {FieldType::STRING}; From 72de354fe087bcfd29030c5958fb7e87e0b8f732 Mon Sep 17 00:00:00 2001 From: gripleaf <425797155@qq.com> Date: Tue, 16 Jun 2026 13:22:41 +0800 Subject: [PATCH 059/138] feat(cache): support manifest file cache --- docs/source/user_guide.rst | 1 + docs/source/user_guide/manifest_cache.rst | 99 ++++++++ .../io => include/paimon}/cache/cache.h | 65 +++-- .../paimon}/memory/memory_segment.h | 9 +- include/paimon/read_context.h | 13 +- include/paimon/scan_context.h | 13 +- src/paimon/CMakeLists.txt | 1 + .../common/data/abstract_binary_writer.h | 2 +- src/paimon/common/data/binary_array.cpp | 2 +- src/paimon/common/data/binary_array_writer.h | 2 +- src/paimon/common/data/binary_map.h | 2 +- src/paimon/common/data/binary_row.cpp | 2 +- src/paimon/common/data/binary_row_test.cpp | 2 +- src/paimon/common/data/binary_row_writer.cpp | 2 +- src/paimon/common/data/binary_row_writer.h | 2 +- src/paimon/common/data/binary_section.h | 2 +- src/paimon/common/data/binary_string.h | 2 +- .../data/serializer/binary_row_serializer.cpp | 2 +- .../serializer/binary_row_serializer_test.cpp | 2 +- .../serializer/row_compacted_serializer.h | 2 +- src/paimon/common/io/cache/cache.cpp | 48 ++++ src/paimon/common/io/cache/cache_key.cpp | 14 +- src/paimon/common/io/cache/cache_key.h | 30 +-- src/paimon/common/io/cache/cache_manager.h | 4 +- src/paimon/common/io/cache/lru_cache.h | 2 +- src/paimon/common/io/cache/lru_cache_test.cpp | 25 +- .../common/io/memory_segment_output_stream.h | 2 +- src/paimon/common/memory/memory_segment.cpp | 12 +- .../common/memory/memory_segment_test.cpp | 2 +- .../common/memory/memory_segment_utils.h | 2 +- src/paimon/common/memory/memory_slice.h | 2 +- src/paimon/common/sst/block_cache.h | 2 +- src/paimon/common/sst/block_handle.h | 2 +- src/paimon/common/sst/bloom_filter_handle.h | 3 + src/paimon/common/utils/murmurhash_utils.h | 2 +- src/paimon/common/utils/serialization_utils.h | 2 +- .../append/append_compact_coordinator.cpp | 3 +- src/paimon/core/core_options.cpp | 11 +- src/paimon/core/core_options.h | 4 + src/paimon/core/core_options_test.cpp | 4 + .../core/manifest/index_manifest_file.cpp | 7 +- .../core/manifest/index_manifest_file.h | 3 +- src/paimon/core/manifest/manifest_file.cpp | 2 +- .../core/manifest/manifest_file_test.cpp | 133 ++++++++++ src/paimon/core/manifest/manifest_list.cpp | 7 +- src/paimon/core/manifest/manifest_list.h | 5 +- .../core/manifest/manifest_list_test.cpp | 7 +- .../append_only_file_store_write.cpp | 3 +- .../core/operation/expire_snapshots_test.cpp | 7 +- .../core/operation/file_store_commit.cpp | 3 +- .../core/operation/internal_read_context.cpp | 1 + .../key_value_file_store_scan_test.cpp | 2 +- .../operation/key_value_file_store_write.cpp | 3 +- .../core/operation/orphan_files_cleaner.cpp | 3 +- src/paimon/core/operation/read_context.cpp | 14 +- .../core/operation/read_context_test.cpp | 4 + src/paimon/core/operation/scan_context.cpp | 15 +- .../core/operation/scan_context_test.cpp | 4 + src/paimon/core/table/source/table_scan.cpp | 9 +- .../table/system/audit_log_system_table.cpp | 6 +- .../table/system/metadata_system_tables.cpp | 8 +- .../core/utils/file_store_path_factory.cpp | 2 +- src/paimon/core/utils/objects_file.h | 63 ++++- .../testing/utils/manifest_cache_test_utils.h | 100 ++++++++ test/inte/scan_inte_test.cpp | 237 +++++++++++------- 65 files changed, 835 insertions(+), 216 deletions(-) create mode 100644 docs/source/user_guide/manifest_cache.rst rename {src/paimon/common/io => include/paimon}/cache/cache.h (61%) rename {src/paimon/common => include/paimon}/memory/memory_segment.h (95%) create mode 100644 src/paimon/common/io/cache/cache.cpp create mode 100644 src/paimon/testing/utils/manifest_cache_test_utils.h diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index 5b14faac..d7597f48 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -27,6 +27,7 @@ User Guide user_guide/schema user_guide/snapshot user_guide/manifest + user_guide/manifest_cache user_guide/data_types user_guide/primary_key_table user_guide/append_only_table diff --git a/docs/source/user_guide/manifest_cache.rst b/docs/source/user_guide/manifest_cache.rst new file mode 100644 index 00000000..0e027509 --- /dev/null +++ b/docs/source/user_guide/manifest_cache.rst @@ -0,0 +1,99 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Manifest Cache +============== + +Overview +-------- + +paimon-cpp caches raw manifest file bytes at the ``ObjectsFile::Read()`` +layer. The cache uses the public ``Cache`` abstraction and is injected through +``ScanContextBuilder`` or ``ReadContextBuilder``. The cache covers data +manifests, manifest lists, and index manifests because they all read through +``ObjectsFile``. + +For repeated ``get``, ``scan``, or batch ``get/scan -f`` requests in the same +process, the same snapshot often reads the same manifest files repeatedly. On a +cache hit, the read path skips remote filesystem ``open/read``, builds an +in-memory input stream from cached bytes, and still runs the format reader, +Arrow decoding, and object deserialization. This design primarily reduces +remote IO latency and bandwidth while keeping cache weight aligned with the +actual cached bytes. + +Configuration +------------- + +Manifest caching is disabled by default. Embedding applications that need it can +provide a custom ``Cache`` implementation and inject it through ``WithCache``. +Manifest reads create cache keys with ``CacheKind::MANIFEST`` internally, so +callers do not need to pass the cache kind through scan or read contexts. The +same cache instance can be reused across multiple scan or read contexts when +process-local sharing is desired. + +Example: + +.. code-block:: cpp + + class RoutingCache : public paimon::Cache { + public: + RoutingCache(std::shared_ptr default_cache, + std::shared_ptr manifest_cache) + : default_cache_(std::move(default_cache)), + manifest_cache_(std::move(manifest_cache)) {} + + paimon::Result> Get( + const std::shared_ptr& key, + std::function>( + const std::shared_ptr&)> supplier) override { + return Select(key)->Get(key, std::move(supplier)); + } + + // Put(), Invalidate(), InvalidateAll(), and Size() route in the same way. + + private: + std::shared_ptr Select( + const std::shared_ptr& key) const { + return key && key->GetKind() == paimon::CacheKind::MANIFEST + ? manifest_cache_ + : default_cache_; + } + + std::shared_ptr default_cache_; + std::shared_ptr manifest_cache_; + }; + + auto cache = std::make_shared( + std::make_shared(), + std::make_shared()); + + paimon::ScanContextBuilder scan_builder(table_path); + scan_builder.WithCache(cache); + + paimon::ReadContextBuilder read_builder(table_path); + read_builder.WithCache(cache); + +Passing ``nullptr`` or omitting ``WithCache()`` leaves manifest caching disabled. + +Future Optimizations +-------------------- + +- Add hit, miss, bypass, and eviction metrics to read trace or metrics. +- Add single-flight loading for high-concurrency misses on the same manifest + path. +- Evaluate a decoded-records second-level cache, configurable as a + CPU-vs-memory tradeoff. diff --git a/src/paimon/common/io/cache/cache.h b/include/paimon/cache/cache.h similarity index 61% rename from src/paimon/common/io/cache/cache.h rename to include/paimon/cache/cache.h index d2cfeab1..fba26a99 100644 --- a/src/paimon/common/io/cache/cache.h +++ b/include/paimon/cache/cache.h @@ -17,25 +17,58 @@ */ #pragma once + +#include #include #include #include #include -#include "paimon/common/io/cache/cache_key.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" +#include "paimon/visibility.h" namespace paimon { class CacheValue; +enum class CacheKind { + DEFAULT, + MANIFEST, +}; + +class PAIMON_EXPORT CacheKey { + public: + static std::shared_ptr ForPosition(const std::string& file_path, int64_t position, + int32_t length, bool is_index); + static std::shared_ptr ForKind(const std::string& file_path, int64_t position, + int32_t length, CacheKind kind); + + public: + virtual ~CacheKey() = default; + + virtual bool IsIndex() const = 0; + + CacheKind GetKind() const { + return kind_; + } + + virtual bool Equals(const CacheKey& other) const = 0; + + virtual size_t HashCode() const = 0; + + protected: + explicit CacheKey(CacheKind kind) : kind_(kind) {} + + private: + CacheKind kind_ = CacheKind::DEFAULT; +}; -/// Callback invoked when a cache entry is evicted by the LRU policy. using CacheCallback = std::function&)>; class PAIMON_EXPORT Cache { public: virtual ~Cache() = default; + virtual Result> Get( const std::shared_ptr& key, std::function>(const std::shared_ptr&)> @@ -51,31 +84,21 @@ class PAIMON_EXPORT Cache { virtual size_t Size() const = 0; }; -class CacheValue { +class PAIMON_EXPORT CacheValue { public: - CacheValue(const MemorySegment& segment, CacheCallback callback) - : segment_(segment), callback_(std::move(callback)) {} + CacheValue(const MemorySegment& segment, CacheCallback callback); - const MemorySegment& GetSegment() const { - return segment_; - } + ~CacheValue(); - /// Invoke the eviction callback, if one was registered. - void OnEvict(const std::shared_ptr& key) const { - if (callback_) { - callback_(key); - } - } + const MemorySegment& GetSegment() const; - bool operator==(const CacheValue& other) const { - if (this == &other) { - return true; - } - return segment_ == other.segment_; - } + void OnEvict(const std::shared_ptr& key) const; + + bool operator==(const CacheValue& other) const; private: MemorySegment segment_; CacheCallback callback_; }; + } // namespace paimon diff --git a/src/paimon/common/memory/memory_segment.h b/include/paimon/memory/memory_segment.h similarity index 95% rename from src/paimon/common/memory/memory_segment.h rename to include/paimon/memory/memory_segment.h index 7fc2a111..f40526ae 100644 --- a/src/paimon/common/memory/memory_segment.h +++ b/include/paimon/memory/memory_segment.h @@ -25,7 +25,6 @@ #include #include -#include "paimon/common/utils/math.h" #include "paimon/io/byte_order.h" #include "paimon/memory/bytes.h" #include "paimon/visibility.h" @@ -141,13 +140,7 @@ class PAIMON_EXPORT MemorySegment { std::memcpy(MutableData() + index, &value, sizeof(T)); } - inline uint64_t GetLongBigEndian(int32_t index) const { - auto value = GetValue(index); - if constexpr (SystemByteOrder() == ByteOrder::PAIMON_LITTLE_ENDIAN) { - return EndianSwapValue(value); - } - return value; - } + uint64_t GetLongBigEndian(int32_t index) const; void CopyTo(int32_t offset, MemorySegment* target, int32_t target_offset, int32_t num_bytes) const { diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index e4d79b2f..6eeb022d 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -25,6 +25,7 @@ #include #include +#include "paimon/cache/cache.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/type_fwd.h" @@ -56,7 +57,8 @@ class PAIMON_EXPORT ReadContext { const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::map& options, - PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config); + PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config, + const std::shared_ptr& cache); ~ReadContext(); const std::string& GetPath() const { @@ -126,6 +128,10 @@ class PAIMON_EXPORT ReadContext { return cache_config_; } + std::shared_ptr GetCache() const { + return cache_; + } + private: std::string path_; std::string branch_; @@ -146,6 +152,7 @@ class PAIMON_EXPORT ReadContext { std::map options_; PrefetchCacheMode prefetch_cache_mode_; CacheConfig cache_config_; + std::shared_ptr cache_; }; /// `ReadContextBuilder` used to build a `ReadContext`, has input validation. @@ -341,6 +348,10 @@ class PAIMON_EXPORT ReadContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ReadContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Inject a cache for read operations. Passing nullptr disables cache. + /// @return Reference to this builder for method chaining. + ReadContextBuilder& WithCache(const std::shared_ptr& cache); + /// Build and return a `ReadContext` instance with input validation. /// @return Result containing the constructed `ReadContext` or an error status. Result> Finish(); diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 30644402..93cdf413 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -25,6 +25,7 @@ #include #include +#include "paimon/cache/cache.h" #include "paimon/global_index/global_index_result.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" @@ -50,7 +51,8 @@ class PAIMON_EXPORT ScanContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options); + const std::map& options, + const std::shared_ptr& cache); ~ScanContext(); @@ -88,6 +90,10 @@ class PAIMON_EXPORT ScanContext { return specific_file_system_; } + std::shared_ptr GetCache() const { + return cache_; + } + private: std::string path_; bool is_streaming_mode_; @@ -98,6 +104,7 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr executor_; std::shared_ptr specific_file_system_; std::map options_; + std::shared_ptr cache_; }; /// Filter configuration for table scan operations @@ -180,6 +187,10 @@ class PAIMON_EXPORT ScanContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Inject a cache for scan operations. Passing nullptr disables cache. + /// @return Reference to this builder for method chaining. + ScanContextBuilder& WithCache(const std::shared_ptr& cache); + /// Build and return a `ScanContext` instance with input validation. /// @return Result containing the constructed `ScanContext` or an error status. Result> Finish(); diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index b1edfdb6..b9ef998a 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -68,6 +68,7 @@ set(PAIMON_COMMON_SRCS common/io/data_output_stream.cpp common/io/memory_segment_output_stream.cpp common/io/offset_input_stream.cpp + common/io/cache/cache.cpp common/io/cache/cache_key.cpp common/io/cache/cache_manager.cpp common/io/cache/lru_cache.cpp diff --git a/src/paimon/common/data/abstract_binary_writer.h b/src/paimon/common/data/abstract_binary_writer.h index 7565543a..fcbb396b 100644 --- a/src/paimon/common/data/abstract_binary_writer.h +++ b/src/paimon/common/data/abstract_binary_writer.h @@ -24,7 +24,7 @@ #include #include "paimon/common/data/binary_writer.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" namespace paimon { class BinaryArray; diff --git a/src/paimon/common/data/binary_array.cpp b/src/paimon/common/data/binary_array.cpp index de195b8c..f92e89b7 100644 --- a/src/paimon/common/data/binary_array.cpp +++ b/src/paimon/common/data/binary_array.cpp @@ -25,8 +25,8 @@ #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_data_read_utils.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" namespace paimon { diff --git a/src/paimon/common/data/binary_array_writer.h b/src/paimon/common/data/binary_array_writer.h index 6b214d92..626f996b 100644 --- a/src/paimon/common/data/binary_array_writer.h +++ b/src/paimon/common/data/binary_array_writer.h @@ -23,7 +23,7 @@ #include "arrow/api.h" #include "paimon/common/data/abstract_binary_writer.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" namespace paimon { class BinaryArray; class MemoryPool; diff --git a/src/paimon/common/data/binary_map.h b/src/paimon/common/data/binary_map.h index 71694617..4783833f 100644 --- a/src/paimon/common/data/binary_map.h +++ b/src/paimon/common/data/binary_map.h @@ -20,8 +20,8 @@ #pragma once #include "paimon/common/data/binary_array.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" +#include "paimon/memory/memory_segment.h" namespace paimon { /// A binary implementation of `InternalMap` which is backed by a single `MemorySegment`. /// Binary layout: [4 byte(keyArray size in bytes)] + [Key BinaryArray] + [Value BinaryArray]. diff --git a/src/paimon/common/data/binary_row.cpp b/src/paimon/common/data/binary_row.cpp index 1cdc66bf..284b095f 100644 --- a/src/paimon/common/data/binary_row.cpp +++ b/src/paimon/common/data/binary_row.cpp @@ -23,11 +23,11 @@ #include "fmt/format.h" #include "paimon/common/data/binary_data_read_utils.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/io/byte_order.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" namespace paimon { const int64_t BinaryRow::FIRST_BYTE_ZERO = diff --git a/src/paimon/common/data/binary_row_test.cpp b/src/paimon/common/data/binary_row_test.cpp index 17ee3331..681c4e54 100644 --- a/src/paimon/common/data/binary_row_test.cpp +++ b/src/paimon/common/data/binary_row_test.cpp @@ -30,7 +30,6 @@ #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/serializer/binary_row_serializer.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/decimal_utils.h" @@ -39,6 +38,7 @@ #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" diff --git a/src/paimon/common/data/binary_row_writer.cpp b/src/paimon/common/data/binary_row_writer.cpp index 0b346e04..462a8561 100644 --- a/src/paimon/common/data/binary_row_writer.cpp +++ b/src/paimon/common/data/binary_row_writer.cpp @@ -29,12 +29,12 @@ #include "arrow/util/checked_cast.h" #include "fmt/format.h" #include "paimon/common/data/binary_string.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/status.h" namespace paimon { diff --git a/src/paimon/common/data/binary_row_writer.h b/src/paimon/common/data/binary_row_writer.h index 954ba1d6..875d0341 100644 --- a/src/paimon/common/data/binary_row_writer.h +++ b/src/paimon/common/data/binary_row_writer.h @@ -26,9 +26,9 @@ #include "paimon/common/data/abstract_binary_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/data_define.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/types/row_kind.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" namespace arrow { diff --git a/src/paimon/common/data/binary_section.h b/src/paimon/common/data/binary_section.h index 73eeeb67..74ab46b0 100644 --- a/src/paimon/common/data/binary_section.h +++ b/src/paimon/common/data/binary_section.h @@ -23,9 +23,9 @@ #include #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/visibility.h" namespace paimon { diff --git a/src/paimon/common/data/binary_string.h b/src/paimon/common/data/binary_string.h index 2de9b588..a9d71aff 100644 --- a/src/paimon/common/data/binary_string.h +++ b/src/paimon/common/data/binary_string.h @@ -29,7 +29,7 @@ #include #include "paimon/common/data/binary_section.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include "paimon/visibility.h" namespace paimon { diff --git a/src/paimon/common/data/serializer/binary_row_serializer.cpp b/src/paimon/common/data/serializer/binary_row_serializer.cpp index 826f7558..56075ac0 100644 --- a/src/paimon/common/data/serializer/binary_row_serializer.cpp +++ b/src/paimon/common/data/serializer/binary_row_serializer.cpp @@ -26,9 +26,9 @@ #include "fmt/format.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" namespace paimon { diff --git a/src/paimon/common/data/serializer/binary_row_serializer_test.cpp b/src/paimon/common/data/serializer/binary_row_serializer_test.cpp index 22c2f6b8..51a6d3aa 100644 --- a/src/paimon/common/data/serializer/binary_row_serializer_test.cpp +++ b/src/paimon/common/data/serializer/binary_row_serializer_test.cpp @@ -25,12 +25,12 @@ #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/binary_string.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/io/data_input_stream.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { diff --git a/src/paimon/common/data/serializer/row_compacted_serializer.h b/src/paimon/common/data/serializer/row_compacted_serializer.h index 9acda1cb..6d73f606 100644 --- a/src/paimon/common/data/serializer/row_compacted_serializer.h +++ b/src/paimon/common/data/serializer/row_compacted_serializer.h @@ -23,10 +23,10 @@ #include "arrow/api.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_writer.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/memory/memory_slice.h" #include "paimon/common/utils/var_length_int_utils.h" +#include "paimon/memory/memory_segment.h" namespace paimon { class RowCompactedSerializer { diff --git a/src/paimon/common/io/cache/cache.cpp b/src/paimon/common/io/cache/cache.cpp new file mode 100644 index 00000000..37a26e91 --- /dev/null +++ b/src/paimon/common/io/cache/cache.cpp @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/cache/cache.h" + +#include + +namespace paimon { + +CacheValue::CacheValue(const MemorySegment& segment, CacheCallback callback) + : segment_(segment), callback_(std::move(callback)) {} + +CacheValue::~CacheValue() = default; + +const MemorySegment& CacheValue::GetSegment() const { + return segment_; +} + +void CacheValue::OnEvict(const std::shared_ptr& key) const { + if (callback_) { + callback_(key); + } +} + +bool CacheValue::operator==(const CacheValue& other) const { + if (this == &other) { + return true; + } + return segment_ == other.segment_; +} + +} // namespace paimon diff --git a/src/paimon/common/io/cache/cache_key.cpp b/src/paimon/common/io/cache/cache_key.cpp index 383bb205..4529e87d 100644 --- a/src/paimon/common/io/cache/cache_key.cpp +++ b/src/paimon/common/io/cache/cache_key.cpp @@ -22,7 +22,15 @@ namespace paimon { std::shared_ptr CacheKey::ForPosition(const std::string& file_path, int64_t position, int32_t length, bool is_index) { - return std::make_shared(file_path, position, length, is_index); + return std::make_shared(file_path, position, length, is_index, + CacheKind::DEFAULT); +} + +std::shared_ptr CacheKey::ForKind(const std::string& file_path, int64_t position, + int32_t length, CacheKind kind) { + auto key = std::make_shared(file_path, position, length, + /*is_index=*/false, kind); + return key; } bool PositionCacheKey::IsIndex() const { @@ -43,7 +51,7 @@ bool PositionCacheKey::Equals(const CacheKey& other) const { return false; } return file_path_ == rhs->file_path_ && position_ == rhs->position_ && - length_ == rhs->length_ && is_index_ == rhs->is_index_; + length_ == rhs->length_ && is_index_ == rhs->is_index_ && GetKind() == rhs->GetKind(); } size_t PositionCacheKey::HashCode() const { @@ -52,6 +60,8 @@ size_t PositionCacheKey::HashCode() const { seed ^= std::hash{}(position_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); seed ^= std::hash{}(length_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); seed ^= std::hash{}(is_index_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); + seed ^= std::hash{}(static_cast(GetKind())) + HASH_CONSTANT + (seed << 6) + + (seed >> 2); return seed; } diff --git a/src/paimon/common/io/cache/cache_key.h b/src/paimon/common/io/cache/cache_key.h index 75f8cb82..988735d1 100644 --- a/src/paimon/common/io/cache/cache_key.h +++ b/src/paimon/common/io/cache/cache_key.h @@ -17,36 +17,24 @@ */ #pragma once + #include #include #include -#include -#include "paimon/visibility.h" +#include "paimon/cache/cache.h" namespace paimon { -class CacheValue; - -class PAIMON_EXPORT CacheKey { - public: - static std::shared_ptr ForPosition(const std::string& file_path, int64_t position, - int32_t length, bool is_index); - - public: - virtual ~CacheKey() = default; - - virtual bool IsIndex() const = 0; - - virtual bool Equals(const CacheKey& other) const = 0; - - virtual size_t HashCode() const = 0; -}; - class PositionCacheKey : public CacheKey { public: - PositionCacheKey(const std::string& file_path, int64_t position, int32_t length, bool is_index) - : file_path_(file_path), position_(position), length_(length), is_index_(is_index) {} + PositionCacheKey(const std::string& file_path, int64_t position, int32_t length, bool is_index, + CacheKind kind) + : CacheKey(kind), + file_path_(file_path), + position_(position), + length_(length), + is_index_(is_index) {} bool IsIndex() const override; size_t HashCode() const override; diff --git a/src/paimon/common/io/cache/cache_manager.h b/src/paimon/common/io/cache/cache_manager.h index bad3f68d..f899d46c 100644 --- a/src/paimon/common/io/cache/cache_manager.h +++ b/src/paimon/common/io/cache/cache_manager.h @@ -22,10 +22,10 @@ #include #include -#include "paimon/common/io/cache/cache.h" +#include "paimon/cache/cache.h" #include "paimon/common/io/cache/cache_key.h" #include "paimon/common/io/cache/lru_cache.h" -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" namespace paimon { diff --git a/src/paimon/common/io/cache/lru_cache.h b/src/paimon/common/io/cache/lru_cache.h index 07d6038b..745ca7f5 100644 --- a/src/paimon/common/io/cache/lru_cache.h +++ b/src/paimon/common/io/cache/lru_cache.h @@ -22,7 +22,7 @@ #include #include -#include "paimon/common/io/cache/cache.h" +#include "paimon/cache/cache.h" #include "paimon/common/io/cache/cache_key.h" #include "paimon/common/utils/generic_lru_cache.h" #include "paimon/result.h" diff --git a/src/paimon/common/io/cache/lru_cache_test.cpp b/src/paimon/common/io/cache/lru_cache_test.cpp index 9e8ae449..ff3b72c0 100644 --- a/src/paimon/common/io/cache/lru_cache_test.cpp +++ b/src/paimon/common/io/cache/lru_cache_test.cpp @@ -27,10 +27,10 @@ #include #include "gtest/gtest.h" -#include "paimon/common/io/cache/cache.h" +#include "paimon/cache/cache.h" #include "paimon/common/io/cache/cache_key.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -362,6 +362,27 @@ TEST_F(LruCacheTest, TestPutMovesToFront) { ASSERT_EQ(evicted[0], 1); } +TEST_F(LruCacheTest, TestForKindSetsKeyKind) { + LruCache cache(1024); + + auto key = CacheKey::ForKind("test_file", 0, 64, CacheKind::MANIFEST); + bool supplier_seen_kind = false; + auto supplier = + [&](const std::shared_ptr& supplier_key) -> Result> { + supplier_seen_kind = supplier_key->GetKind() == CacheKind::MANIFEST; + return MakeValue(64, 'M'); + }; + + ASSERT_OK_AND_ASSIGN(auto value, cache.Get(key, supplier)); + ASSERT_TRUE(supplier_seen_kind); + ASSERT_EQ(CacheKind::MANIFEST, key->GetKind()); + ASSERT_EQ('M', value->GetSegment().Get(0)); + + auto put_key = CacheKey::ForKind("test_file", 1, 64, CacheKind::MANIFEST); + ASSERT_OK(cache.Put(put_key, MakeValue(64, 'P'))); + ASSERT_EQ(CacheKind::MANIFEST, put_key->GetKind()); +} + /// Verifies that multiple evictions happen when a single large entry is inserted. TEST_F(LruCacheTest, TestMultipleEvictions) { LruCache cache(300); diff --git a/src/paimon/common/io/memory_segment_output_stream.h b/src/paimon/common/io/memory_segment_output_stream.h index dbe3816f..0986344e 100644 --- a/src/paimon/common/io/memory_segment_output_stream.h +++ b/src/paimon/common/io/memory_segment_output_stream.h @@ -26,9 +26,9 @@ #include #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/utils/math.h" #include "paimon/io/byte_order.h" +#include "paimon/memory/memory_segment.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" diff --git a/src/paimon/common/memory/memory_segment.cpp b/src/paimon/common/memory/memory_segment.cpp index be4a8d5f..5e4cb8df 100644 --- a/src/paimon/common/memory/memory_segment.cpp +++ b/src/paimon/common/memory/memory_segment.cpp @@ -17,12 +17,22 @@ * under the License. */ -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include +#include "paimon/common/utils/math.h" + namespace paimon { +uint64_t MemorySegment::GetLongBigEndian(int32_t index) const { + auto value = GetValue(index); + if constexpr (SystemByteOrder() == ByteOrder::PAIMON_LITTLE_ENDIAN) { + return EndianSwapValue(value); + } + return value; +} + int32_t MemorySegment::Compare(const MemorySegment& seg2, int32_t offset1, int32_t offset2, int32_t len) const { while (len >= 8) { diff --git a/src/paimon/common/memory/memory_segment_test.cpp b/src/paimon/common/memory/memory_segment_test.cpp index 05012950..c7c25a7e 100644 --- a/src/paimon/common/memory/memory_segment_test.cpp +++ b/src/paimon/common/memory/memory_segment_test.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/memory/memory_segment.h" +#include "paimon/memory/memory_segment.h" #include #include diff --git a/src/paimon/common/memory/memory_segment_utils.h b/src/paimon/common/memory/memory_segment_utils.h index 4f1fdc65..19785ec6 100644 --- a/src/paimon/common/memory/memory_segment_utils.h +++ b/src/paimon/common/memory/memory_segment_utils.h @@ -29,10 +29,10 @@ #include "fmt/format.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/io/byte_order.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/status.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" diff --git a/src/paimon/common/memory/memory_slice.h b/src/paimon/common/memory/memory_slice.h index 2f45683f..4ed9f385 100644 --- a/src/paimon/common/memory/memory_slice.h +++ b/src/paimon/common/memory/memory_slice.h @@ -24,8 +24,8 @@ #include #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" #include "paimon/visibility.h" namespace paimon { diff --git a/src/paimon/common/sst/block_cache.h b/src/paimon/common/sst/block_cache.h index 4fd8bb2a..8e6a8631 100644 --- a/src/paimon/common/sst/block_cache.h +++ b/src/paimon/common/sst/block_cache.h @@ -23,8 +23,8 @@ #include #include "paimon/common/io/cache/cache_manager.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/fs/file_system.h" +#include "paimon/memory/memory_segment.h" #include "paimon/reader/batch_reader.h" #include "paimon/result.h" namespace paimon { diff --git a/src/paimon/common/sst/block_handle.h b/src/paimon/common/sst/block_handle.h index c00a5af6..7244b516 100644 --- a/src/paimon/common/sst/block_handle.h +++ b/src/paimon/common/sst/block_handle.h @@ -20,9 +20,9 @@ #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_slice_input.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" namespace paimon { diff --git a/src/paimon/common/sst/bloom_filter_handle.h b/src/paimon/common/sst/bloom_filter_handle.h index 1974604e..0814231b 100644 --- a/src/paimon/common/sst/bloom_filter_handle.h +++ b/src/paimon/common/sst/bloom_filter_handle.h @@ -21,6 +21,9 @@ #include #include +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" +#include "paimon/result.h" #include "paimon/visibility.h" namespace paimon { diff --git a/src/paimon/common/utils/murmurhash_utils.h b/src/paimon/common/utils/murmurhash_utils.h index e184d9a0..18430f44 100644 --- a/src/paimon/common/utils/murmurhash_utils.h +++ b/src/paimon/common/utils/murmurhash_utils.h @@ -60,8 +60,8 @@ #include #include -#include "paimon/common/memory/memory_segment.h" #include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" namespace paimon { diff --git a/src/paimon/common/utils/serialization_utils.h b/src/paimon/common/utils/serialization_utils.h index 4718ea92..c1e97d03 100644 --- a/src/paimon/common/utils/serialization_utils.h +++ b/src/paimon/common/utils/serialization_utils.h @@ -27,7 +27,6 @@ #include "fmt/format.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/io/memory_segment_output_stream.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/memory/memory_segment_utils.h" #include "paimon/common/utils/math.h" #include "paimon/io/byte_order.h" @@ -35,6 +34,7 @@ #include "paimon/macros.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" diff --git a/src/paimon/core/append/append_compact_coordinator.cpp b/src/paimon/core/append/append_compact_coordinator.cpp index 318a9c66..37693a58 100644 --- a/src/paimon/core/append/append_compact_coordinator.cpp +++ b/src/paimon/core/append/append_compact_coordinator.cpp @@ -137,7 +137,8 @@ Result> CreateFileStoreScan( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(core_options.GetFileSystem(), core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, pool)); + core_options.GetManifestCompression(), path_factory, + core_options.GetCache(), pool)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_file, ManifestFile::Create(core_options.GetFileSystem(), core_options.GetManifestFormat(), diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 4fefaffe..736506d8 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -374,6 +374,7 @@ struct CoreOptions::Impl { std::shared_ptr file_format; std::shared_ptr file_system; std::shared_ptr manifest_file_format; + std::shared_ptr cache; std::optional scan_snapshot_id; std::optional scan_timestamp_millis; @@ -871,7 +872,6 @@ Result CoreOptions::FromMap( PAIMON_RETURN_NOT_OK(impl->ParseIndexOptions(parser)); PAIMON_RETURN_NOT_OK(impl->ParseCompactionOptions(parser)); PAIMON_RETURN_NOT_OK(impl->ParseLookupOptions(parser)); - return options; } @@ -974,6 +974,15 @@ int64_t CoreOptions::GetManifestTargetFileSize() const { return impl_->manifest_target_file_size; } +std::shared_ptr CoreOptions::GetCache() const { + return impl_->cache; +} + +CoreOptions& CoreOptions::WithCache(const std::shared_ptr& cache) { + impl_->cache = cache; + return *this; +} + int32_t CoreOptions::GetManifestMergeMinCount() const { return impl_->manifest_merge_min_count; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index 6a7f3446..12969e49 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -26,6 +26,7 @@ #include #include "paimon/bucket/bucket_function_type.h" +#include "paimon/cache/cache.h" #include "paimon/core/options/changelog_producer.h" #include "paimon/core/options/compress_options.h" #include "paimon/core/options/external_path_strategy.h" @@ -44,6 +45,7 @@ namespace paimon { class ExpireConfig; +class Cache; class PAIMON_EXPORT CoreOptions { public: @@ -80,6 +82,8 @@ class PAIMON_EXPORT CoreOptions { std::optional GetScanTimestampMillis() const; int64_t GetManifestTargetFileSize() const; + std::shared_ptr GetCache() const; + CoreOptions& WithCache(const std::shared_ptr& cache); StartupMode GetStartupMode() const; int32_t GetReadBatchSize() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index d8f9650a..402b6b48 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -57,6 +57,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(8 * 1024 * 1024L, core_options.GetManifestTargetFileSize()); ASSERT_EQ(16 * 1024 * 1024L, core_options.GetManifestFullCompactionThresholdSize()); ASSERT_EQ(30, core_options.GetManifestMergeMinCount()); + ASSERT_EQ(nullptr, core_options.GetCache()); ASSERT_EQ(128 * 1024 * 1024L, core_options.GetSourceSplitTargetSize()); ASSERT_EQ(4 * 1024 * 1024L, core_options.GetSourceSplitOpenFileCost()); ASSERT_EQ(1024, core_options.GetReadBatchSize()); @@ -291,6 +292,7 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(16 * 1024 * 1024L, core_options.GetManifestTargetFileSize()); ASSERT_EQ(32 * 1024 * 1024L, core_options.GetManifestFullCompactionThresholdSize()); ASSERT_EQ(2, core_options.GetManifestMergeMinCount()); + ASSERT_EQ(nullptr, core_options.GetCache()); ASSERT_EQ(24 * 1024 * 1024L, core_options.GetSourceSplitTargetSize()); ASSERT_EQ(32 * 1024 * 1024L, core_options.GetSourceSplitOpenFileCost()); ASSERT_EQ(2048, core_options.GetReadBatchSize()); @@ -865,9 +867,11 @@ TEST(CoreOptionsTest, TestCopyAssignmentOperator) { // Verify the target's ToMap matches the source's ToMap ASSERT_EQ(source.ToMap(), target.ToMap()); + ASSERT_EQ(source.GetCache(), target.GetCache()); CoreOptions target2 = source; ASSERT_EQ(source.ToMap(), target2.ToMap()); + ASSERT_EQ(source.GetCache(), target2.GetCache()); } TEST(CoreOptionsTest, TestAssignmentIndependence) { diff --git a/src/paimon/core/manifest/index_manifest_file.cpp b/src/paimon/core/manifest/index_manifest_file.cpp index fda878c1..648c7302 100644 --- a/src/paimon/core/manifest/index_manifest_file.cpp +++ b/src/paimon/core/manifest/index_manifest_file.cpp @@ -63,7 +63,7 @@ Result> IndexManifestFile::Create( path_factory->CreateIndexManifestFileFactory(); return std::unique_ptr( new IndexManifestFile(file_system, reader_builder, writer_builder, compression, - index_manifest_file_factory, bucket_mode, pool)); + index_manifest_file_factory, bucket_mode, options.GetCache(), pool)); } IndexManifestFile::IndexManifestFile(const std::shared_ptr& file_system, @@ -71,10 +71,11 @@ IndexManifestFile::IndexManifestFile(const std::shared_ptr& file_sys const std::shared_ptr& writer_builder, const std::string& compression, const std::shared_ptr& path_factory, - int32_t bucket_mode, const std::shared_ptr& pool) + int32_t bucket_mode, const std::shared_ptr& cache, + const std::shared_ptr& pool) : ObjectsFile(file_system, reader_builder, writer_builder, std::make_unique(pool), - compression, path_factory, pool), + compression, path_factory, cache, pool), bucket_mode_(bucket_mode) {} Result> IndexManifestFile::WriteIndexFiles( diff --git a/src/paimon/core/manifest/index_manifest_file.h b/src/paimon/core/manifest/index_manifest_file.h index e98e0923..f6fd46f3 100644 --- a/src/paimon/core/manifest/index_manifest_file.h +++ b/src/paimon/core/manifest/index_manifest_file.h @@ -33,6 +33,7 @@ namespace paimon { class CoreOptions; +class Cache; class FileFormat; class FileStorePathFactory; class FileSystem; @@ -62,7 +63,7 @@ class IndexManifestFile : public ObjectsFile { const std::shared_ptr& writer_builder, const std::string& compression, const std::shared_ptr& path_factory, int32_t bucket_mode, - const std::shared_ptr& pool); + const std::shared_ptr& cache, const std::shared_ptr& pool); const int32_t bucket_mode_; }; diff --git a/src/paimon/core/manifest/manifest_file.cpp b/src/paimon/core/manifest/manifest_file.cpp index 52637bbb..acdd2de4 100644 --- a/src/paimon/core/manifest/manifest_file.cpp +++ b/src/paimon/core/manifest/manifest_file.cpp @@ -56,7 +56,7 @@ ManifestFile::ManifestFile(const std::shared_ptr& file_system, const std::shared_ptr& partition_type) : ObjectsFile(file_system, reader_builder, writer_builder, std::make_unique(pool), compression, - path_factory, pool), + path_factory, options.GetCache(), pool), target_file_size_(target_file_size), options_(options), partition_type_(partition_type) {} diff --git a/src/paimon/core/manifest/manifest_file_test.cpp b/src/paimon/core/manifest/manifest_file_test.cpp index 38180003..499cc756 100644 --- a/src/paimon/core/manifest/manifest_file_test.cpp +++ b/src/paimon/core/manifest/manifest_file_test.cpp @@ -43,9 +43,62 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/manifest_cache_test_utils.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { + +class CountingFileSystem : public FileSystem { + public: + Result> Open(const std::string& path) const override { + ++open_count; + return local_.Open(path); + } + + Result> Create(const std::string& path, + bool overwrite) const override { + return local_.Create(path, overwrite); + } + + Status Mkdirs(const std::string& path) const override { + return local_.Mkdirs(path); + } + + Status Rename(const std::string& src, const std::string& dst) const override { + return local_.Rename(src, dst); + } + + Status Delete(const std::string& path, bool recursive = true) const override { + return local_.Delete(path, recursive); + } + + Result> GetFileStatus(const std::string& path) const override { + ++get_file_status_count; + return local_.GetFileStatus(path); + } + + Status ListDir(const std::string& directory, + std::vector>* file_status_list) const override { + return local_.ListDir(directory, file_status_list); + } + + Status ListFileStatus( + const std::string& path, + std::vector>* file_status_list) const override { + return local_.ListFileStatus(path, file_status_list); + } + + Result Exists(const std::string& path) const override { + return local_.Exists(path); + } + + mutable int open_count = 0; + mutable int get_file_status_count = 0; + + private: + LocalFileSystem local_; +}; + class ManifestFileTest : public testing::Test { public: std::vector ReadManifestEntry(const std::string& file_format_str, @@ -183,6 +236,86 @@ TEST_F(ManifestFileTest, TestSimple) { ASSERT_EQ(expected_manifest_entries, manifest_entries); } +TEST_F(ManifestFileTest, TestManifestCacheIsDisabledWithoutInjectedCache) { + auto pool = GetDefaultPool(); + auto counting_file_system = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format, + FileFormatFactory::Get("orc", {})); + std::string root_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr path_factory, + FileStorePathFactory::Create(root_path, unused_schema, /*partition_keys=*/{}, + /*default_part_value=*/"", file_format->Identifier(), + /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, + /*global_index_external_path=*/std::nullopt, + /*index_file_in_data_file_dir=*/false, pool)); + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}})); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr manifest_file, + ManifestFile::Create(counting_file_system, file_format, "zstd", path_factory, + /*target_file_size=*/1024, pool, options, unused_schema)); + + std::vector first_read; + ASSERT_OK(manifest_file->Read("manifest-3ea5ee21-d399-4f1c-a749-2fc63dbf0852-1", + /*filter=*/nullptr, &first_read)); + ASSERT_EQ(5, first_read.size()); + ASSERT_EQ(1, counting_file_system->open_count); + ASSERT_EQ(0, counting_file_system->get_file_status_count); + + std::vector filtered_read; + ASSERT_OK(manifest_file->Read( + "manifest-3ea5ee21-d399-4f1c-a749-2fc63dbf0852-1", + [](const ManifestEntry& entry) -> Result { return entry.Kind() == FileKind::Add(); }, + &filtered_read)); + ASSERT_EQ(1, filtered_read.size()); + ASSERT_EQ(2, counting_file_system->open_count); + ASSERT_EQ(0, counting_file_system->get_file_status_count); +} + +TEST_F(ManifestFileTest, TestManifestCacheReusesCachedBytes) { + auto pool = GetDefaultPool(); + auto counting_file_system = std::make_shared(); + auto manifest_cache = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format, + FileFormatFactory::Get("orc", {})); + std::string root_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + auto unused_schema = arrow::schema(arrow::FieldVector({arrow::field("f0", arrow::utf8())})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr path_factory, + FileStorePathFactory::Create(root_path, unused_schema, /*partition_keys=*/{}, + /*default_part_value=*/"", file_format->Identifier(), + /*data_file_prefix=*/"data-", + /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, + /*global_index_external_path=*/std::nullopt, + /*index_file_in_data_file_dir=*/false, pool)); + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}})); + options.WithCache(manifest_cache); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr manifest_file, + ManifestFile::Create(counting_file_system, file_format, "zstd", path_factory, + /*target_file_size=*/1024, pool, options, unused_schema)); + + std::vector first_read; + ASSERT_OK(manifest_file->Read("manifest-3ea5ee21-d399-4f1c-a749-2fc63dbf0852-1", + /*filter=*/nullptr, &first_read)); + std::vector second_read; + ASSERT_OK(manifest_file->Read("manifest-3ea5ee21-d399-4f1c-a749-2fc63dbf0852-1", + /*filter=*/nullptr, &second_read)); + + ASSERT_EQ(first_read, second_read); + ASSERT_EQ(1, counting_file_system->open_count); + ASSERT_EQ(0, counting_file_system->get_file_status_count); + ASSERT_EQ(2, manifest_cache->GetCount()); + ASSERT_EQ(1, manifest_cache->SupplierCallCount()); + ASSERT_EQ(1, manifest_cache->Size()); +} + TEST_F(ManifestFileTest, TestWithNullCount) { auto pool = GetDefaultPool(); auto manifest_entries = diff --git a/src/paimon/core/manifest/manifest_list.cpp b/src/paimon/core/manifest/manifest_list.cpp index 58b9cbdf..887b8845 100644 --- a/src/paimon/core/manifest/manifest_list.cpp +++ b/src/paimon/core/manifest/manifest_list.cpp @@ -43,15 +43,16 @@ ManifestList::ManifestList(const std::shared_ptr& file_system, const std::shared_ptr& writer_builder, const std::string& compression, const std::shared_ptr& path_factory, + const std::shared_ptr& cache, const std::shared_ptr& pool) : ObjectsFile(file_system, reader_builder, writer_builder, std::make_unique(pool), compression, - std::move(path_factory), pool) {} + std::move(path_factory), cache, pool) {} Result> ManifestList::Create( const std::shared_ptr& fs, const std::shared_ptr& file_format, const std::string& compression, const std::shared_ptr& path_factory, - const std::shared_ptr& pool) { + const std::shared_ptr& cache, const std::shared_ptr& pool) { std::shared_ptr data_type = VersionedObjectSerializer::VersionType(ManifestFileMeta::DataType()); // prepare format reader builder @@ -71,7 +72,7 @@ Result> ManifestList::Create( std::shared_ptr manifest_list_path_factory = path_factory->CreateManifestListFactory(); return std::unique_ptr(new ManifestList( - fs, reader_builder, writer_builder, compression, manifest_list_path_factory, pool)); + fs, reader_builder, writer_builder, compression, manifest_list_path_factory, cache, pool)); } Result> ManifestList::Write( diff --git a/src/paimon/core/manifest/manifest_list.h b/src/paimon/core/manifest/manifest_list.h index 65b683bf..31959a5c 100644 --- a/src/paimon/core/manifest/manifest_list.h +++ b/src/paimon/core/manifest/manifest_list.h @@ -35,6 +35,7 @@ namespace paimon { +class Cache; class FileFormat; class FileSystem; class FileStorePathFactory; @@ -52,7 +53,7 @@ class ManifestList : public ObjectsFile { const std::shared_ptr& file_system, const std::shared_ptr& file_format, const std::string& compression, const std::shared_ptr& path_factory, - const std::shared_ptr& pool); + const std::shared_ptr& cache, const std::shared_ptr& pool); /// Write several `ManifestFileMeta`s into a manifest list. /// @@ -124,7 +125,7 @@ class ManifestList : public ObjectsFile { const std::shared_ptr& reader_builder, const std::shared_ptr& writer_builder, const std::string& compression, const std::shared_ptr& path_factory, - const std::shared_ptr& pool); + const std::shared_ptr& cache, const std::shared_ptr& pool); }; } // namespace paimon diff --git a/src/paimon/core/manifest/manifest_list_test.cpp b/src/paimon/core/manifest/manifest_list_test.cpp index 0f71afd9..889f18f2 100644 --- a/src/paimon/core/manifest/manifest_list_test.cpp +++ b/src/paimon/core/manifest/manifest_list_test.cpp @@ -23,6 +23,7 @@ #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/core/core_options.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/utils/file_store_path_factory.h" @@ -51,8 +52,10 @@ class ManifestListTest : public testing::Test { /*legacy_partition_name_enabled=*/true, /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, /*index_file_in_data_file_dir=*/false, pool)); - EXPECT_OK_AND_ASSIGN(auto manifest_list, ManifestList::Create(file_system, file_format, - "zstd", path_factory, pool)); + EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + EXPECT_OK_AND_ASSIGN(auto manifest_list, + ManifestList::Create(file_system, file_format, "zstd", path_factory, + options.GetCache(), pool)); return manifest_list; } diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 7f407cb5..459dcf67 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -89,7 +89,8 @@ Result> AppendOnlyFileStoreWrite::CreateFileStore PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(options_.GetFileSystem(), options_.GetManifestFormat(), - options_.GetManifestCompression(), file_store_path_factory_, pool_)); + options_.GetManifestCompression(), file_store_path_factory_, + options_.GetCache(), pool_)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_file, ManifestFile::Create(options_.GetFileSystem(), options_.GetManifestFormat(), diff --git a/src/paimon/core/operation/expire_snapshots_test.cpp b/src/paimon/core/operation/expire_snapshots_test.cpp index 5c40c442..d6b965ba 100644 --- a/src/paimon/core/operation/expire_snapshots_test.cpp +++ b/src/paimon/core/operation/expire_snapshots_test.cpp @@ -75,9 +75,10 @@ class ExpireSnapshotsTest : public testing::Test { test_data_path_ = "tmp"; path_factory_ = CreateFactory(test_data_path_); - ASSERT_OK_AND_ASSIGN(manifest_list_, ManifestList::Create(fs_, options.GetManifestFormat(), - options.GetManifestCompression(), - path_factory_, mem_pool_)); + ASSERT_OK_AND_ASSIGN( + manifest_list_, + ManifestList::Create(fs_, options.GetManifestFormat(), options.GetManifestCompression(), + path_factory_, options.GetCache(), mem_pool_)); ASSERT_OK_AND_ASSIGN( manifest_file_, diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index 4310e201..a4ce3f42 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -116,7 +116,8 @@ Result> FileStoreCommit::Create( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, ctx->GetMemoryPool())); + options.GetManifestCompression(), path_factory, options.GetCache(), + ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 26f5f27e..316002da 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -37,6 +37,7 @@ Result> InternalReadContext::Create( PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, context->GetSpecificFileSystem(), context->GetFileSystemSchemeToIdentifierMap())); + core_options.WithCache(context->GetCache()); // prepare read schema std::vector read_data_fields; if (!context->GetReadFieldIds().empty()) { diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp index 04314039..8db2cb06 100644 --- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp @@ -97,7 +97,7 @@ class KeyValueFileStoreScanTest : public testing::Test { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, pool_)); + path_factory, core_options.GetCache(), pool_)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp index 0f511993..08c5ea0c 100644 --- a/src/paimon/core/operation/key_value_file_store_write.cpp +++ b/src/paimon/core/operation/key_value_file_store_write.cpp @@ -81,7 +81,8 @@ Result> KeyValueFileStoreWrite::CreateFileStoreSc PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(options_.GetFileSystem(), options_.GetManifestFormat(), - options_.GetManifestCompression(), file_store_path_factory_, pool_)); + options_.GetManifestCompression(), file_store_path_factory_, + options_.GetCache(), pool_)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_file, ManifestFile::Create(options_.GetFileSystem(), options_.GetManifestFormat(), diff --git a/src/paimon/core/operation/orphan_files_cleaner.cpp b/src/paimon/core/operation/orphan_files_cleaner.cpp index 0b1df483..ac69d5a3 100644 --- a/src/paimon/core/operation/orphan_files_cleaner.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner.cpp @@ -197,7 +197,8 @@ Result> OrphanFilesCleaner::Create( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(options.GetFileSystem(), options.GetManifestFormat(), - options.GetManifestCompression(), path_factory, ctx->GetMemoryPool())); + options.GetManifestCompression(), path_factory, options.GetCache(), + ctx->GetMemoryPool())); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, table_schema.value()->PartitionKeys())); diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index e329a35e..bb9cca77 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -39,7 +39,7 @@ ReadContext::ReadContext( const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::map& options, PrefetchCacheMode prefetch_cache_mode, - const CacheConfig& cache_config) + const CacheConfig& cache_config, const std::shared_ptr& cache) : path_(path), branch_(branch), read_schema_(read_schema), @@ -58,7 +58,8 @@ ReadContext::ReadContext( fs_scheme_to_identifier_map_(fs_scheme_to_identifier_map), options_(options), prefetch_cache_mode_(prefetch_cache_mode), - cache_config_(cache_config) {} + cache_config_(cache_config), + cache_(cache) {} ReadContext::~ReadContext() = default; @@ -84,6 +85,7 @@ class ReadContextBuilder::Impl { executor_.reset(); specific_file_system_.reset(); cache_config_ = CacheConfig(); + cache_.reset(); } private: @@ -106,6 +108,7 @@ class ReadContextBuilder::Impl { std::shared_ptr specific_file_system_; PrefetchCacheMode prefetch_cache_mode_ = PrefetchCacheMode::ALWAYS; CacheConfig cache_config_; + std::shared_ptr cache_; }; ReadContextBuilder::ReadContextBuilder(const std::string& path) @@ -219,6 +222,11 @@ ReadContextBuilder& ReadContextBuilder::WithCacheConfig(const CacheConfig& cache return *this; } +ReadContextBuilder& ReadContextBuilder::WithCache(const std::shared_ptr& cache) { + impl_->cache_ = cache; + return *this; +} + Result> ReadContextBuilder::Finish() { PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { @@ -255,7 +263,7 @@ Result> ReadContextBuilder::Finish() { impl_->enable_multi_thread_row_to_batch_, impl_->row_to_batch_thread_number_, impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, impl_->options_, impl_->prefetch_cache_mode_, - impl_->cache_config_); + impl_->cache_config_, impl_->cache_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index fbe1a595..f1945cc0 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -21,6 +21,7 @@ #include #include "gtest/gtest.h" +#include "paimon/common/io/cache/lru_cache.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" @@ -79,6 +80,8 @@ TEST(ReadContextTest, TestSetContent) { builder.WithCacheConfig(cache_config); auto fs = std::make_shared(); builder.WithFileSystem(fs); + auto manifest_cache = std::make_shared(1024); + builder.WithCache(manifest_cache); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); // test result @@ -108,6 +111,7 @@ TEST(ReadContextTest, TestSetContent) { std::map expected_options = {{"key", "value"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); + ASSERT_TRUE(ctx->GetCache()); } TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) { diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 794480ed..684439b5 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -35,7 +35,8 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, - const std::map& options) + const std::map& options, + const std::shared_ptr& cache) : path_(path), is_streaming_mode_(is_streaming_mode), limit_(limit), @@ -44,7 +45,8 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), - options_(options) {} + options_(options), + cache_(cache) {} ScanContext::~ScanContext() = default; @@ -63,6 +65,7 @@ class ScanContextBuilder::Impl { executor_ = CreateDefaultExecutor(); specific_file_system_.reset(); options_.clear(); + cache_.reset(); } private: @@ -77,6 +80,7 @@ class ScanContextBuilder::Impl { std::shared_ptr executor_ = CreateDefaultExecutor(); std::shared_ptr specific_file_system_; std::map options_; + std::shared_ptr cache_; }; ScanContextBuilder::ScanContextBuilder(const std::string& path) @@ -145,6 +149,11 @@ ScanContextBuilder& ScanContextBuilder::WithFileSystem( return *this; } +ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr& cache) { + impl_->cache_ = cache; + return *this; +} + Result> ScanContextBuilder::Finish() { PAIMON_ASSIGN_OR_RAISE(impl_->path_, PathUtil::NormalizePath(impl_->path_)); if (impl_->path_.empty()) { @@ -155,7 +164,7 @@ Result> ScanContextBuilder::Finish() { std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_); + impl_->specific_file_system_, impl_->options_, impl_->cache_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index dd3914d6..1f74463d 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -19,6 +19,7 @@ #include "paimon/scan_context.h" #include "gtest/gtest.h" +#include "paimon/common/io/cache/lru_cache.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/global_index/bitmap_global_index_result.h" @@ -67,6 +68,8 @@ TEST(ScanContextTest, TestSetContent) { builder.WithExecutor(executor); auto fs = std::make_shared(); builder.WithFileSystem(fs); + auto manifest_cache = std::make_shared(1024); + builder.WithCache(manifest_cache); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_TRUE(ctx->IsStreamingMode()); @@ -81,6 +84,7 @@ TEST(ScanContextTest, TestSetContent) { std::map expected_options = {{"key", "value"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(fs, ctx->GetSpecificFileSystem()); + ASSERT_TRUE(ctx->GetCache()); } TEST(ScanContextTest, TestSetOptionsOverridesAddedOptions) { diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 5afe9938..15fe9ef9 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -89,7 +89,7 @@ class TableScanImpl { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr manifest_list, ManifestList::Create(fs, manifest_file_format, core_options.GetManifestCompression(), - path_factory, memory_pool)); + path_factory, core_options.GetCache(), memory_pool)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); @@ -177,7 +177,7 @@ Result> TableScan::Create(std::unique_ptrGetOptions(), - shared_context->GetSpecificFileSystem())); + shared_context->GetSpecificFileSystem(), {})); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_path, SystemTableLoader::TryParsePath(shared_context->GetPath())); if (system_table_path) { @@ -195,7 +195,7 @@ namespace { Result> NewDataTableScan(const std::shared_ptr& context) { PAIMON_ASSIGN_OR_RAISE( CoreOptions tmp_options, - CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem())); + CoreOptions::FromMap(context->GetOptions(), context->GetSpecificFileSystem(), {})); std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, @@ -215,7 +215,8 @@ Result> NewDataTableScan(const std::shared_ptrGetSpecificFileSystem())); + CoreOptions::FromMap(options, context->GetSpecificFileSystem(), {})); + core_options.WithCache(context->GetCache()); // validate options if (core_options.GetBucket() == -1) { if (!table_schema->PrimaryKeys().empty()) { diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index f5cd896d..a2581eb2 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -249,7 +249,8 @@ Result> AuditLogSystemTable::NewScan( .WithStreamingMode(context->IsStreamingMode()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) - .WithFileSystem(context->GetSpecificFileSystem()); + .WithFileSystem(context->GetSpecificFileSystem()) + .WithCache(context->GetCache()); if (scan_filter) { if (scan_filter->GetBucketFilter()) { builder.SetBucketFilter(scan_filter->GetBucketFilter().value()); @@ -298,7 +299,8 @@ Result> AuditLogSystemTable::NewChangelogRead( .EnableMultiThreadRowToBatch(context->EnableMultiThreadRowToBatch()) .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) .SetPrefetchCacheMode(context->GetPrefetchCacheMode()) - .WithCacheConfig(context->GetCacheConfig()); + .WithCacheConfig(context->GetCacheConfig()) + .WithCache(context->GetCache()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_context, builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr data_read, diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index 36a4c419..faa62ee8 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -229,10 +229,10 @@ Result> ReadDataManifests( const MetadataSystemTableContext& context, const Snapshot& snapshot, const std::shared_ptr& path_factory, const CoreOptions& core_options, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr manifest_list, - ManifestList::Create(context.fs, core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, pool)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_list, + ManifestList::Create(context.fs, core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, + core_options.GetCache(), pool)); std::vector manifests; // TODO(suxiaogang223): Align Java ReadAllManifests semantics by including changelog // manifests. ReadAllManifests currently delegates to ReadChangelogManifests, which returns diff --git a/src/paimon/core/utils/file_store_path_factory.cpp b/src/paimon/core/utils/file_store_path_factory.cpp index d17b223f..ef4aae42 100644 --- a/src/paimon/core/utils/file_store_path_factory.cpp +++ b/src/paimon/core/utils/file_store_path_factory.cpp @@ -21,7 +21,6 @@ #include #include "paimon/common/fs/external_path_provider.h" -#include "paimon/common/memory/memory_segment.h" #include "paimon/common/utils/uuid.h" #include "paimon/core/index/index_file_meta.h" #include "paimon/core/index/index_in_data_file_dir_path_factory.h" @@ -30,6 +29,7 @@ #include "paimon/core/utils/partition_path_utils.h" #include "paimon/core/utils/path_factory.h" #include "paimon/macros.h" +#include "paimon/memory/memory_segment.h" #include "paimon/status.h" namespace arrow { diff --git a/src/paimon/core/utils/objects_file.h b/src/paimon/core/utils/objects_file.h index b62bdc8b..1e334704 100644 --- a/src/paimon/core/utils/objects_file.h +++ b/src/paimon/core/utils/objects_file.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include #include @@ -26,6 +27,8 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "fmt/format.h" +#include "paimon/cache/cache.h" #include "paimon/common/data/columnar/columnar_row.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -39,6 +42,8 @@ #include "paimon/format/reader_builder.h" #include "paimon/format/writer_builder.h" #include "paimon/fs/file_system.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/memory/bytes.h" #include "paimon/record_batch.h" namespace paimon { @@ -52,7 +57,7 @@ class ObjectsFile { const std::shared_ptr& writer_builder, std::unique_ptr>&& serializer, const std::string& compression, const std::shared_ptr& path_factory, - const std::shared_ptr& pool); + const std::shared_ptr& cache, const std::shared_ptr& pool); virtual ~ObjectsFile() = default; @@ -82,6 +87,9 @@ class ObjectsFile { std::shared_ptr file_system_; std::shared_ptr reader_builder_; std::string compression_; + std::shared_ptr cache_; + + Result ReadFileSegment(const std::string& file_path) const; }; template @@ -91,6 +99,7 @@ ObjectsFile::ObjectsFile(const std::shared_ptr& file_system, std::unique_ptr>&& serializer, const std::string& compression, const std::shared_ptr& path_factory, + const std::shared_ptr& cache, const std::shared_ptr& pool) : path_factory_(path_factory), pool_(pool), @@ -98,9 +107,8 @@ ObjectsFile::ObjectsFile(const std::shared_ptr& file_system, writer_builder_(std::move(writer_builder)), file_system_(file_system), reader_builder_(std::move(reader_builder)), - compression_(compression) { - // TODO(xinyu.lxy): add cache -} + compression_(compression), + cache_(cache) {} template Status ObjectsFile::ReadIfFileExist(const std::string& file_name, @@ -119,8 +127,33 @@ Status ObjectsFile::Read(const std::string& file_name, const std::function(const T&)>& filter, std::vector* result) const { std::string file_path = path_factory_->ToPath(file_name); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_input_stream, - file_system_->Open(file_path)); + std::shared_ptr file_input_stream; + std::shared_ptr cached_bytes; + if (cache_) { + // Use a whole-file key so cache hits do not need a metadata lookup just to discover file + // length. + auto cache_key = + CacheKey::ForKind(file_path, /*position=*/0, /*length=*/-1, CacheKind::MANIFEST); + auto supplier = + [this, + &file_path](const std::shared_ptr&) -> Result> { + PAIMON_ASSIGN_OR_RAISE(MemorySegment segment, ReadFileSegment(file_path)); + return std::make_shared(segment, CacheCallback()); + }; + Result> cache_result = cache_->Get(cache_key, supplier); + if (cache_result.ok() && cache_result.value() && + cache_result.value()->GetSegment().Data() != nullptr) { + cached_bytes = cache_result.value()->GetSegment().GetOrCreateHeapMemory(pool_.get()); + file_input_stream = + std::make_shared(cached_bytes->data(), cached_bytes->size()); + } + } + if (!file_input_stream) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr unique_file_input_stream, + file_system_->Open(file_path)); + file_input_stream = std::shared_ptr(std::move(unique_file_input_stream)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, reader_builder_->Build(file_input_stream)); auto reader = std::make_unique(std::move(batch_reader), @@ -156,6 +189,24 @@ Status ObjectsFile::Read(const std::string& file_name, return Status::OK(); } +template +Result ObjectsFile::ReadFileSegment(const std::string& file_path) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr input_stream, + file_system_->Open(file_path)); + PAIMON_ASSIGN_OR_RAISE(int64_t input_length, input_stream->Length()); + + PAIMON_RETURN_NOT_OK(input_stream->Seek(0, FS_SEEK_SET)); + auto bytes = std::make_shared(input_length, pool_.get()); + PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_size, + input_stream->Read(bytes->data(), input_length)); + if (actual_read_size != input_length) { + return Status::IOError(fmt::format( + "Unexpected EOF while reading manifest file {}, expected {} bytes, got {} bytes", + file_path, input_length, actual_read_size)); + } + return MemorySegment::Wrap(bytes); +} + template Result> ObjectsFile::WriteWithoutRolling( const std::vector& records) { diff --git a/src/paimon/testing/utils/manifest_cache_test_utils.h b/src/paimon/testing/utils/manifest_cache_test_utils.h new file mode 100644 index 00000000..fa2c7d41 --- /dev/null +++ b/src/paimon/testing/utils/manifest_cache_test_utils.h @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/cache/cache.h" +#include "paimon/common/io/cache/lru_cache.h" +#include "paimon/result.h" + +namespace paimon::test { + +class CountingManifestRoutingCache : public Cache { + public: + explicit CountingManifestRoutingCache(int64_t max_weight = 64 * 1024 * 1024) { + caches_[CacheKind::MANIFEST] = std::make_shared(max_weight); + } + + Result> Get( + const std::shared_ptr& key, + std::function>(const std::shared_ptr&)> + supplier) override { + ++get_count_; + return GetCache(key)->Get( + key, + [this, supplier = std::move(supplier)](const std::shared_ptr& supplier_key) + -> Result> { + ++supplier_call_count_; + return supplier(supplier_key); + }); + } + + Status Put(const std::shared_ptr& key, + const std::shared_ptr& value) override { + return GetCache(key)->Put(key, value); + } + + void Invalidate(const std::shared_ptr& key) override { + GetCache(key)->Invalidate(key); + } + + void InvalidateAll() override { + for (const auto& [kind, cache] : caches_) { + cache->InvalidateAll(); + } + } + + size_t Size() const override { + size_t size = 0; + for (const auto& [kind, cache] : caches_) { + size += cache->Size(); + } + return size; + } + + int64_t GetCount() const { + return get_count_; + } + + int64_t SupplierCallCount() const { + return supplier_call_count_; + } + + private: + std::shared_ptr GetCache(const std::shared_ptr& key) const { + EXPECT_EQ(CacheKind::MANIFEST, key->GetKind()); + auto iter = caches_.find(key->GetKind()); + EXPECT_NE(caches_.end(), iter); + return iter == caches_.end() ? nullptr : iter->second; + } + + std::map> caches_; + int64_t get_count_ = 0; + int64_t supplier_call_count_ = 0; +}; + +} // namespace paimon::test diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 2989cf2e..0a781c5f 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -55,11 +55,24 @@ #include "paimon/table/source/startup_mode.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/manifest_cache_test_utils.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { -class ScanInteTest : public testing::Test { +enum class ManifestCacheMode { NoCache, Cache }; + +class ScanInteTest : public testing::TestWithParam { public: + Result> FinishScanContext(ScanContextBuilder& builder) { + if (GetParam() == ManifestCacheMode::Cache) { + if (!cache_) { + cache_ = std::make_shared(); + } + builder.WithCache(cache_); + } + return builder.Finish(); + } + std::vector> CollectDataSplits( const std::shared_ptr& plan) const { std::vector> result_data_splits; @@ -109,6 +122,8 @@ class ScanInteTest : public testing::Test { } private: + std::shared_ptr cache_; + std::shared_ptr pool_ = GetDefaultPool(); std::shared_ptr arrow_data_type_ = @@ -244,11 +259,38 @@ class ScanInteTest : public testing::Test { /*write_cols=*/std::nullopt); }; -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1) { +TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { + auto manifest_cache = std::make_shared(); + std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + + auto run_scan = [&]() -> Result> { + ScanContextBuilder context_builder(table_path); + context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithCache(manifest_cache); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + return table_scan->CreatePlan(); + }; + + ASSERT_OK_AND_ASSIGN(auto first_plan, run_scan()); + ASSERT_TRUE(first_plan->SnapshotId()); + ASSERT_EQ(1, first_plan->SnapshotId().value()); + ASSERT_FALSE(first_plan->Splits().empty()); + ASSERT_GT(manifest_cache->SupplierCallCount(), 0); + int64_t supplier_calls_after_first_scan = manifest_cache->SupplierCallCount(); + + ASSERT_OK_AND_ASSIGN(auto second_plan, run_scan()); + ASSERT_EQ(first_plan->SnapshotId(), second_plan->SnapshotId()); + ASSERT_EQ(first_plan->Splits().size(), second_plan->Splits().size()); + ASSERT_GT(manifest_cache->GetCount(), supplier_calls_after_first_scan); + ASSERT_EQ(supplier_calls_after_first_scan, manifest_cache->SupplierCallCount()); +} + +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -300,11 +342,11 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot3) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -355,36 +397,36 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot3) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanInvalidSnapshot) { +TEST_P(ScanInteTest, TestScanInvalidSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "100"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), "The specified scan snapshotId 100 is out of available snapshotId range [1, 5]."); } -TEST_F(ScanInteTest, TestBatchScanMultipleTimes) { +TEST_P(ScanInteTest, TestBatchScanMultipleTimes) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // batch scan multiple ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "end of scan"); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") .AddOption(Options::SOURCE_SPLIT_OPEN_FILE_COST, "1024") .AddOption(Options::SOURCE_SPLIT_TARGET_SIZE, "2048"); // open cost = 1024, and split target size is 2048, indicates at most 2 files in a split - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -447,11 +489,11 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithSplitTargetSize) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3").SetLimit(3); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -493,11 +535,11 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithRowCountLimit) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.SetBucketFilter(0).AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -535,12 +577,12 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot3WithBucketFilter) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { // from snapshot is specified std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); DataSplitImpl::Builder builder1_1( @@ -648,12 +690,12 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamWithDefaultMode) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); DataSplitImpl::Builder builder1( @@ -702,11 +744,11 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamOfLatestFullMode) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { +TEST_P(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -757,27 +799,27 @@ TEST_F(ScanInteTest, TestScanAppendWithBatchScanOfLatestMode) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamOfLatestMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::Latest().ToString()) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); std::vector> expected_snapshot_ids = {}; CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, {}); } -TEST_F(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) .AddOption(Options::SCAN_SNAPSHOT_ID, "2") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); DataSplitImpl::Builder builder2_1( @@ -842,14 +884,14 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotMode) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { +TEST_P(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshotFull().ToString()) .AddOption(Options::SCAN_SNAPSHOT_ID, "2") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_TRUE(scan_context); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); @@ -927,13 +969,13 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamOfFromSnapshotFullMode) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithInvalidOptions) { +TEST_P(ScanInteTest, TestScanAppendWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; { ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_MODE, StartupMode::FromSnapshot().ToString()) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_NOK_WITH_MSG( table_scan->CreatePlan(), @@ -942,13 +984,13 @@ TEST_F(ScanInteTest, TestScanAppendWithInvalidOptions) { { ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::BUCKET, "-2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support bucket=-2 in scan process"); } } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; std::string val("Bob"); @@ -958,7 +1000,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { ScanContextBuilder context_builder(table_path); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -982,7 +1024,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEqualPredicate) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { +TEST_P(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { // from snapshot is specified std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; @@ -1006,7 +1048,7 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { context_builder.SetPredicate(predicate) .AddOption(Options::SCAN_SNAPSHOT_ID, "1") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); DataSplitImpl::Builder builder1_2( @@ -1058,14 +1100,14 @@ TEST_F(ScanInteTest, TestScanAppendWithStreamWithAndPredicate) { CheckStreamScanResult(table_scan.get(), expected_snapshot_ids, expected_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; std::map partition_keys; partition_keys["f1"] = "10"; ScanContextBuilder context_builder(table_path); context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1104,19 +1146,19 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilter) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithInvalidPartitionFilter) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithInvalidPartitionFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; std::map partition_keys; partition_keys["invalid_partition_key"] = "10"; ScanContextBuilder context_builder(table_path); context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "field invalid_partition_key does not exist in partition keys"); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateFilter) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; // set predicate filter, f1 = 20 @@ -1130,7 +1172,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF context_builder.SetPredicate(predicate) .SetPartitionFilter({partition_keys}) .AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1169,7 +1211,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithPartitionFilterAndPredicateF CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { std::string table_path = paimon::test::GetDataDir() + "orc/multi_partition_append_table.db/multi_partition_append_table"; @@ -1180,7 +1222,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { ScanContextBuilder context_builder(table_path); context_builder.SetPartitionFilter({partition_keys}).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1221,7 +1263,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithMultiPartitionKeys) { } // test complex type ts & decimal -TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) { +TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) { std::string table_path = paimon::test::GetDataDir() + "orc/append_complex_data.db/append_complex_data"; // set predicate filter @@ -1236,7 +1278,7 @@ TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) ScanContextBuilder context_builder(table_path); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan); @@ -1282,7 +1324,7 @@ TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter) } // test complex type date & binary -TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) { +TEST_P(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_complex_data.db/append_complex_data"; // set predicate filter @@ -1297,7 +1339,7 @@ TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) ScanContextBuilder context_builder(table_path); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "4"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id @@ -1347,14 +1389,14 @@ TEST_F(ScanInteTest, TestScanAppendComplexDataWithSnapshot4WithPredicateFilter2) CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; ScanContextBuilder context_builder(table_path); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(13.0)); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1441,7 +1483,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { +TEST_P(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_10_stats_dense_store.db/append_10_stats_dense_store"; ScanContextBuilder context_builder(table_path); @@ -1452,7 +1494,7 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { auto predicate = PredicateBuilder::And({greater_than, equal}).value(); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1486,40 +1528,40 @@ TEST_F(ScanInteTest, TestScanAppendWithSnapshot1WithEnableStatsDenseStore2) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { +TEST_P(ScanInteTest, TestScanPKWithSnapshot1WithBucketStats) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").SetBucketFilter(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); ASSERT_TRUE(result_plan->Splits().empty()); } -TEST_F(ScanInteTest, TestScanPKWithInvalidOptions) { +TEST_P(ScanInteTest, TestScanPKWithInvalidOptions) { std::string table_path = paimon::test::GetDataDir() + "orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::BUCKET, "-1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(scan_context)), "do not support pk table bucket=-1 in scan process"); } -TEST_F(ScanInteTest, TestReadWithNoSnapshot) { +TEST_P(ScanInteTest, TestReadWithNoSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_nested_type.db/append_table_with_nested_type"; ScanContextBuilder context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_FALSE(result_plan->SnapshotId()); ASSERT_TRUE(result_plan->Splits().empty()); } -TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithCast) { +TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_alter_table_with_cast.db/append_table_alter_table_with_cast"; @@ -1546,7 +1588,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithCast) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2, child3})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1584,7 +1626,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithCast) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { +TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table.db/append_table_with_alter_table"; ScanContextBuilder context_builder(table_path); @@ -1596,7 +1638,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1659,7 +1701,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithNoCast) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { +TEST_P(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_alter_table_with_dense_field.db/" "append_table_with_alter_table_with_dense_field"; @@ -1672,7 +1714,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child3})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1707,7 +1749,7 @@ TEST_F(ScanInteTest, TestScanAppendWithAlterTableWithDenseField) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; ScanContextBuilder context_builder(table_path); @@ -1719,7 +1761,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1783,7 +1825,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndex) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap.db/append_with_bitmap/"; ScanContextBuilder context_builder(table_path); @@ -1795,13 +1837,13 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapEmbeddedIndexWithEmptyResult) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_no_embedding.db/append_with_bitmap_no_embedding/"; @@ -1814,7 +1856,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check data splits @@ -1848,7 +1890,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapNoEmbeddedIndex) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; @@ -1858,7 +1900,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { FieldType::INT, Literal(100)); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1916,7 +1958,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; @@ -1927,7 +1969,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1990,7 +2032,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable3) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; @@ -2003,7 +2045,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { FieldType::BIGINT, Literal(100l)); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2066,7 +2108,7 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTable2) { CheckResult(expected_data_splits, result_data_splits); } -TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { +TEST_P(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { std::string table_path = paimon::test::GetDataDir() + "orc/append_with_bitmap_alter_table.db/append_with_bitmap_alter_table/"; @@ -2081,35 +2123,36 @@ TEST_F(ScanInteTest, TestScanAppendWithBitmapAndAlterTableWithEmptyResult) { ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({child1, child2})); context_builder.SetPredicate(predicate).AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_TRUE(result_plan->Splits().empty()); } -TEST_F(ScanInteTest, TestScanAppendWithTag1) { +TEST_P(ScanInteTest, TestScanAppendWithTag1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_TAG_NAME, "1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); // check snapshot id ASSERT_EQ(1, result_plan->SnapshotId().value()); } -TEST_F(ScanInteTest, TestScanInvalidTag) { +TEST_P(ScanInteTest, TestScanInvalidTag) { std::string table_path = paimon::test::GetDataDir() + "orc/append_table_with_tag.db/append_table_with_tag"; ScanContextBuilder context_builder(table_path); context_builder.AddOption(Options::SCAN_TAG_NAME, "unknown"); - ASSERT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_NOK_WITH_MSG(table_scan->CreatePlan(), "Tag 'unknown' doesn't exist."); } -TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { +TEST_P(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { std::string table_path = GetDataDir() + "orc/append_09.db/append_09"; auto fs = std::make_shared(); @@ -2120,7 +2163,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 3); @@ -2129,7 +2172,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis() - 1)); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 2); @@ -2139,7 +2182,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(std::numeric_limits::max())); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); ASSERT_EQ(plan->SnapshotId().value(), 5); @@ -2148,14 +2191,14 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisBatchScan) { { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_NOK_WITH_MSG(scan->CreatePlan(), "There is currently no snapshot earlier than or equal to timestamp"); } } -TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { +TEST_P(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { std::string table_path = GetDataDir() + "orc/append_09.db/append_09"; auto fs = std::make_shared(); @@ -2167,7 +2210,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, "0").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); @@ -2179,7 +2222,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap2.TimeMillis() + 1)) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); @@ -2191,7 +2234,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { ScanContextBuilder builder(table_path); builder.AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(snap3.TimeMillis())) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); @@ -2205,7 +2248,7 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { .AddOption(Options::SCAN_TIMESTAMP_MILLIS, std::to_string(std::numeric_limits::max())) .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ctx, FinishScanContext(builder)); ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, TableScan::Create(std::move(ctx))); ASSERT_OK_AND_ASSIGN(std::shared_ptr plan0, scan->CreatePlan()); ASSERT_EQ(plan0->SnapshotId(), std::nullopt); @@ -2213,4 +2256,18 @@ TEST_F(ScanInteTest, TestWithAppendTimestampMillisStreamScan) { } } +std::string ManifestCacheModeName(const testing::TestParamInfo& info) { + switch (info.param) { + case ManifestCacheMode::NoCache: + return "NoCache"; + case ManifestCacheMode::Cache: + return "Cache"; + } + return "Unknown"; +} + +INSTANTIATE_TEST_SUITE_P(ManifestCacheMode, ScanInteTest, + testing::Values(ManifestCacheMode::NoCache, ManifestCacheMode::Cache), + ManifestCacheModeName); + } // namespace paimon::test From 099957aadc10e71d165ee473d5951c2a724fbc93 Mon Sep 17 00:00:00 2001 From: gripleaf <425797155@qq.com> Date: Tue, 16 Jun 2026 14:44:39 +0800 Subject: [PATCH 060/138] feat(schema): support pk table schema evolution --- .../append_only_file_store_scan_test.cpp | 6 + src/paimon/core/operation/file_store_scan.cpp | 11 +- .../operation/key_value_file_store_scan.cpp | 28 ++-- .../key_value_file_store_scan_test.cpp | 131 ++++++++++++++++++ .../core/stats/simple_stats_evolution.cpp | 1 - src/paimon/core/table/source/table_scan.cpp | 5 - .../core/table/source/table_scan_test.cpp | 8 +- test/inte/scan_and_read_inte_test.cpp | 50 +++++++ 8 files changed, 218 insertions(+), 22 deletions(-) diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index 3bdf112e..d644bec0 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -84,6 +84,12 @@ TEST(AppendOnlyFileStoreScanTest, TestReconstructPredicateWithNonCastedFields) { auto result, AppendOnlyFileStoreScan::ReconstructPredicateWithNonCastedFields(predicate, evo)); ASSERT_EQ(*result, *child4); + + auto key_predicate = + PredicateBuilder::IsNull(/*field_index=*/1, /*field_name=*/"key0", FieldType::INT); + ASSERT_OK_AND_ASSIGN(result, AppendOnlyFileStoreScan::ReconstructPredicateWithNonCastedFields( + key_predicate, evo)); + ASSERT_EQ(*result, *key_predicate); } TEST(AppendOnlyFileStoreScanTest, TestReadPartitionEntries) { diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index 6cdffe36..e49ef28f 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -72,11 +72,12 @@ Result> FileStoreScan::ReconstructPredicateWithNonCas fmt::format("field {} in predicate is not included in table schema", field_name)); } auto data_iter = id_to_data_fields.find(table_iter->second.Id()); - if (data_iter != id_to_data_fields.end()) { - // Exclude fields requiring casting to avoid false negatives in stats filtering. - if (!data_iter->second.second.Type()->Equals(table_iter->second.Type())) { - excluded_field_names.insert(field_name); - } + if (data_iter == id_to_data_fields.end()) { + continue; + } + // Exclude fields requiring casting to avoid false negatives in stats filtering. + if (!data_iter->second.second.Type()->Equals(table_iter->second.Type())) { + excluded_field_names.insert(field_name); } } return PredicateUtils::ExcludePredicateWithFields(predicate, excluded_field_names); diff --git a/src/paimon/core/operation/key_value_file_store_scan.cpp b/src/paimon/core/operation/key_value_file_store_scan.cpp index 03550e42..54197f87 100644 --- a/src/paimon/core/operation/key_value_file_store_scan.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan.cpp @@ -35,6 +35,7 @@ #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/options/merge_engine.h" +#include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/stats/simple_stats_evolution.h" @@ -191,14 +192,25 @@ Result KeyValueFileStoreScan::FilterByValueFilter(const ManifestEntry& ent const auto& meta = entry.File(); - // Primary key table currently does not support schema evolution for value filtering. - // Here we only handle `value_stats_cols` (dense stats) projection. - if (meta->schema_id != table_schema_->Id()) { - return Status::NotImplemented( - "Primary key table does not support schema evolution in FilterByValueFilter"); + std::shared_ptr data_schema = table_schema_; + std::shared_ptr trimmed_predicates = value_filter_; + int64_t data_schema_id = meta->schema_id; + if (data_schema_id != table_schema_->Id()) { + PAIMON_ASSIGN_OR_RAISE(data_schema, schema_manager_->ReadSchema(data_schema_id)); } - auto evolution = evolutions_->GetOrCreate(table_schema_); + auto evolution = evolutions_->GetOrCreate(data_schema); + if (data_schema_id != table_schema_->Id()) { + PAIMON_ASSIGN_OR_RAISE(trimmed_predicates, + ReconstructPredicateWithNonCastedFields(value_filter_, evolution)); + } + if (!trimmed_predicates) { + return true; + } + auto predicate_filter = std::dynamic_pointer_cast(trimmed_predicates); + if (!predicate_filter) { + return Status::Invalid("invalid value predicate, cannot cast to PredicateFilter"); + } PAIMON_ASSIGN_OR_RAISE( SimpleStatsEvolution::EvolutionStats new_stats, @@ -207,8 +219,8 @@ Result KeyValueFileStoreScan::FilterByValueFilter(const ManifestEntry& ent try { PAIMON_ASSIGN_OR_RAISE( bool predicate_result, - value_filter_->Test(schema_, meta->row_count, *(new_stats.min_values), - *(new_stats.max_values), *(new_stats.null_counts))); + predicate_filter->Test(schema_, meta->row_count, *(new_stats.min_values), + *(new_stats.max_values), *(new_stats.null_counts))); return predicate_result; } catch (const std::exception& e) { return Status::Invalid(fmt::format("FilterByValueFilter failed for file {}, with {} error", diff --git a/src/paimon/core/operation/key_value_file_store_scan_test.cpp b/src/paimon/core/operation/key_value_file_store_scan_test.cpp index 8db2cb06..b221715f 100644 --- a/src/paimon/core/operation/key_value_file_store_scan_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_scan_test.cpp @@ -439,4 +439,135 @@ TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithValueStatsCols) { ASSERT_OK_AND_ASSIGN(keep, scan->FilterByStats(entry_keep)); ASSERT_TRUE(keep); } + +TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithSchemaEvolution) { + std::string table_path = + paimon::test::GetDataDir() + "orc/pk_table_with_alter_table.db/pk_table_with_alter_table"; + std::vector> partition_filters = {}; + + // In schema-1, `c` is renamed from schema-0 field `b` and moves to index 3. + auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"c", + FieldType::INT, Literal(30)); + auto scan_filter = std::make_shared(/*predicate=*/greater_than, + /*partition_filters=*/partition_filters, + /*bucket_filter=*/0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + CreateFileStoreScan(table_path, scan_filter, + /*table_schema_id=*/1, /*snapshot_id=*/6)); + scan->EnableValueFilter(); + + auto pool = GetDefaultPool(); + // Build schema-0 dense stats for field `b`; after evolution they are tested against + // schema-1 field `c`. + SimpleStats value_stats = BinaryRowGenerator::GenerateStats( + /*min=*/{10}, /*max=*/{20}, /*null=*/{0}, pool.get()); + std::vector value_stats_cols = {"b"}; + ManifestEntry entry( + /*kind=*/FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(), /*bucket=*/0, + /*total_buckets=*/1, + std::make_shared( + /*file_name=*/"schema0_name", /*file_size=*/1024, /*row_count=*/10, + /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), + /*key_stats=*/SimpleStats::EmptyStats(), + /*value_stats=*/value_stats, + /*min_sequence_number=*/0, + /*max_sequence_number=*/10, + /*schema_id=*/0, + /*level=*/1, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, + /*file_source=*/FileSource::Append(), + /*value_stats_cols=*/value_stats_cols, + /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt)); + + SimpleStats value_stats_keep = BinaryRowGenerator::GenerateStats( + /*min=*/{40}, /*max=*/{50}, /*null=*/{0}, pool.get()); + ManifestEntry entry_keep( + /*kind=*/FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(), /*bucket=*/0, + /*total_buckets=*/1, + std::make_shared( + /*file_name=*/"schema0_name_keep", /*file_size=*/1024, /*row_count=*/10, + /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), + /*key_stats=*/SimpleStats::EmptyStats(), + /*value_stats=*/value_stats_keep, + /*min_sequence_number=*/0, + /*max_sequence_number=*/10, + /*schema_id=*/0, + /*level=*/1, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, + /*file_source=*/FileSource::Append(), + /*value_stats_cols=*/value_stats_cols, + /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(entry)); + ASSERT_FALSE(keep); + + ASSERT_OK_AND_ASSIGN(keep, scan->FilterByStats(entry_keep)); + ASSERT_TRUE(keep); +} + +TEST_F(KeyValueFileStoreScanTest, TestFilterByValueFilterWithNewFieldUsesNullStats) { + std::string table_path = + paimon::test::GetDataDir() + "orc/pk_table_with_alter_table.db/pk_table_with_alter_table"; + std::vector> partition_filters = {}; + + // `e` only exists in schema-1. Schema-0 files expose it to stats filtering as all NULL. + auto greater_than = PredicateBuilder::GreaterThan(/*field_index=*/7, /*field_name=*/"e", + FieldType::INT, Literal(30)); + auto scan_filter = std::make_shared(/*predicate=*/greater_than, + /*partition_filters=*/partition_filters, + /*bucket_filter=*/0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, + CreateFileStoreScan(table_path, scan_filter, + /*table_schema_id=*/1, /*snapshot_id=*/6)); + scan->EnableValueFilter(); + + auto pool = GetDefaultPool(); + SimpleStats value_stats = BinaryRowGenerator::GenerateStats( + /*min=*/{10}, /*max=*/{20}, /*null=*/{0}, pool.get()); + std::vector value_stats_cols = {"b"}; + ManifestEntry old_schema_entry( + /*kind=*/FileKind::Add(), /*partition=*/BinaryRow::EmptyRow(), /*bucket=*/0, + /*total_buckets=*/1, + std::make_shared( + /*file_name=*/"schema0_missing_e", /*file_size=*/1024, /*row_count=*/10, + /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), + /*key_stats=*/SimpleStats::EmptyStats(), + /*value_stats=*/value_stats, + /*min_sequence_number=*/0, + /*max_sequence_number=*/10, + /*schema_id=*/0, + /*level=*/1, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, + /*file_source=*/FileSource::Append(), + /*value_stats_cols=*/value_stats_cols, + /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(bool keep, scan->FilterByStats(old_schema_entry)); + ASSERT_FALSE(keep); + + auto is_null = PredicateBuilder::IsNull(/*field_index=*/7, /*field_name=*/"e", FieldType::INT); + scan_filter = std::make_shared(/*predicate=*/is_null, + /*partition_filters=*/partition_filters, + /*bucket_filter=*/0); + ASSERT_OK_AND_ASSIGN(scan, CreateFileStoreScan(table_path, scan_filter, + /*table_schema_id=*/1, /*snapshot_id=*/6)); + scan->EnableValueFilter(); + ASSERT_OK_AND_ASSIGN(keep, scan->FilterByStats(old_schema_entry)); + ASSERT_TRUE(keep); +} } // namespace paimon::test diff --git a/src/paimon/core/stats/simple_stats_evolution.cpp b/src/paimon/core/stats/simple_stats_evolution.cpp index a9356ed7..42177a0b 100644 --- a/src/paimon/core/stats/simple_stats_evolution.cpp +++ b/src/paimon/core/stats/simple_stats_evolution.cpp @@ -64,7 +64,6 @@ SimpleStatsEvolution::SimpleStatsEvolution(const std::vector& data_fi const auto& data_field = data_fields[i]; id_to_data_fields_.emplace(data_field.Id(), std::make_pair(i, data_field)); } - std::map name_to_table_fields; for (const auto& field : table_fields) { name_to_table_fields_.emplace(field.Name(), field); } diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 15fe9ef9..ba45c116 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -204,11 +204,6 @@ Result> NewDataTableScan(const std::shared_ptrId() != TableSchema::FIRST_SCHEMA_ID && - !table_schema->PrimaryKeys().empty()) { - return Status::NotImplemented( - "do not support schema evolution in pk table while scan process"); - } // merge options auto options = table_schema->Options(); for (const auto& [key, value] : context->GetOptions()) { diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index 8e43ad46..ba76de4c 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -50,14 +50,16 @@ TEST(TableScanTest, TestNonExistTable) { ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), "not found latest schema"); } -TEST(TableScanTest, TestNoSchemaEvolution) { - // do not bear schema evolution in scan +TEST(TableScanTest, TestPkSchemaEvolutionScan) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table/"; ScanContextBuilder builder(path); builder.AddOption(Options::FILE_FORMAT, "orc"); ASSERT_OK_AND_ASSIGN(auto context, builder.Finish()); - ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), "do not support schema evolution"); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(context))); + ASSERT_OK_AND_ASSIGN(auto plan, table_scan->CreatePlan()); + ASSERT_TRUE(plan->SnapshotId()); + ASSERT_FALSE(plan->Splits().empty()); } } // namespace paimon::test diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 491f4628..56715571 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -2210,6 +2210,56 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } +TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { + auto [file_format, enable_prefetch] = GetParam(); + std::string table_path = paimon::test::GetDataDir() + file_format + + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; + + auto predicate = PredicateBuilder::GreaterThan( + /*field_index=*/2, /*field_name=*/"key_2", FieldType::INT, Literal(500)); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + scan_context_builder.SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 6); + ASSERT_EQ(result_plan->Splits().size(), 1); + + size_t data_file_count = 0; + for (const auto& split : result_plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + ASSERT_TRUE(data_split); + data_file_count += data_split->data_files_.size(); + } + ASSERT_EQ(data_file_count, 1); + + ReadContextBuilder read_context_builder(table_path); + AddReadOptionsForPrefetch(&read_context_builder); + read_context_builder.SetReadSchema({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("key1", arrow::int32()), arrow::field("k", arrow::utf8()), + arrow::field("key_2", arrow::int32()), + arrow::field("c", arrow::int32()), arrow::field("d", arrow::int32()), + arrow::field("a", arrow::int32()), arrow::field("key0", arrow::int32()), + arrow::field("e", arrow::int32())}), + R"([ +[0, 1, "Two roads diverged in a wood, and I took the one less traveled by, And that has made all the difference.", 2, 4, null, 6, 0, null], +[0, 1, "Alice", 12, 94, null, 96, 0, null], +[0, 1, "Paul", 502, 504, 508, 506, 0, 509] +])") + .ValueOrDie()); + ASSERT_TRUE(expected); + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); +} + TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { auto [file_format, enable_prefetch] = GetParam(); if (file_format != "parquet") { From 99175a99ce0cc73a5c68801e1b8d05580b19493b Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Wed, 17 Jun 2026 09:37:27 +0800 Subject: [PATCH 061/138] fix: fix min/max row id serialize bug --- src/paimon/core/manifest/manifest_file_meta_serializer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/paimon/core/manifest/manifest_file_meta_serializer.cpp b/src/paimon/core/manifest/manifest_file_meta_serializer.cpp index 5018aeaa..c7042b27 100644 --- a/src/paimon/core/manifest/manifest_file_meta_serializer.cpp +++ b/src/paimon/core/manifest/manifest_file_meta_serializer.cpp @@ -72,14 +72,14 @@ Result ManifestFileMetaSerializer::ToRow(const ManifestFileMeta& reco if (!min_row_id) { writer.SetNullAt(11); } else { - writer.WriteInt(11, min_row_id.value()); + writer.WriteLong(11, min_row_id.value()); } auto max_row_id = record.MaxRowId(); if (!max_row_id) { writer.SetNullAt(12); } else { - writer.WriteInt(12, max_row_id.value()); + writer.WriteLong(12, max_row_id.value()); } writer.Complete(); return row; From d71013daf63e2c706bed1b26bbfa38bd7ccb3b1b Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:28:34 +0800 Subject: [PATCH 062/138] feat: update lumina lib to v0.3.0-rc1 --- src/paimon/global_index/lumina/lumina_api_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/global_index/lumina/lumina_api_test.cpp b/src/paimon/global_index/lumina/lumina_api_test.cpp index d92315d3..2c4954b5 100644 --- a/src/paimon/global_index/lumina/lumina_api_test.cpp +++ b/src/paimon/global_index/lumina/lumina_api_test.cpp @@ -136,7 +136,7 @@ class LuminaInterfaceTest : public ::testing::Test { ASSERT_GT(paimon_pool->MaxMemoryUsage(), 0); } - void CheckResult(const std::vector<::lumina::api::LuminaSearcher::SearchHit>& search_result, + void CheckResult(const std::vector<::lumina::api::SearchHit>& search_result, const std::vector<::lumina::core::vector_id_t>& expected_row_ids, const std::vector& expected_distances) const { ASSERT_EQ(search_result.size(), expected_row_ids.size()); From 2b01c5cd6bbf6f586af1a3d535eb23c3636cac23 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:04:25 +0800 Subject: [PATCH 063/138] feat(shredding): support shared-shredding map write --- include/paimon/format/format_writer.h | 7 + src/paimon/CMakeLists.txt | 7 + .../map_shared_shredding_batch_converter.cpp | 317 +++++ .../map_shared_shredding_batch_converter.h | 141 +++ ..._shared_shredding_batch_converter_test.cpp | 404 +++++++ .../map_shared_shredding_column_allocator.cpp | 69 ++ .../map_shared_shredding_column_allocator.h | 78 ++ ...shared_shredding_column_allocator_test.cpp | 113 ++ .../map_shared_shredding_context.cpp | 71 ++ .../shredding/map_shared_shredding_context.h | 66 ++ .../map_shared_shredding_context_test.cpp | 148 +++ .../map_shared_shredding_field_dict.h | 63 + .../map_shared_shredding_field_dict_test.cpp | 58 + .../shredding/map_shared_shredding_utils.cpp | 82 +- .../shredding/map_shared_shredding_utils.h | 48 +- .../map_shared_shredding_utils_test.cpp | 44 +- .../data/shredding/map_shredding_defs.h | 3 + src/paimon/common/utils/arrow/arrow_utils.cpp | 3 + src/paimon/common/utils/arrow/arrow_utils.h | 2 + src/paimon/core/append/append_only_writer.cpp | 81 +- src/paimon/core/append/append_only_writer.h | 30 +- .../core/append/append_only_writer_test.cpp | 1046 +++++++++++++++-- src/paimon/core/io/data_file_writer.cpp | 15 + src/paimon/core/io/data_file_writer.h | 18 +- .../core/io/key_value_data_file_writer.cpp | 16 + .../core/io/key_value_data_file_writer.h | 13 + src/paimon/core/io/single_file_writer.h | 31 + .../compact/changelog_merge_tree_rewriter.cpp | 10 +- .../compact/changelog_merge_tree_rewriter.h | 1 + .../lookup_merge_tree_compact_rewriter.cpp | 13 +- .../lookup_merge_tree_compact_rewriter.h | 1 + ...ookup_merge_tree_compact_rewriter_test.cpp | 18 +- .../compact/merge_tree_compact_rewriter.cpp | 60 +- .../compact/merge_tree_compact_rewriter.h | 6 + .../core/mergetree/merge_tree_writer.cpp | 74 +- src/paimon/core/mergetree/merge_tree_writer.h | 11 +- .../core/mergetree/merge_tree_writer_test.cpp | 263 +++++ .../append_only_file_store_write.cpp | 58 +- .../operation/append_only_file_store_write.h | 4 +- .../postpone_bucket_file_store_write.h | 9 +- .../core/postpone/postpone_bucket_writer.cpp | 95 +- .../core/postpone/postpone_bucket_writer.h | 18 +- .../postpone/postpone_bucket_writer_test.cpp | 135 ++- .../core/schema/schema_validation_test.cpp | 3 +- src/paimon/format/avro/avro_format_writer.cpp | 6 + src/paimon/format/avro/avro_format_writer.h | 4 + src/paimon/format/blob/blob_format_writer.cpp | 6 + src/paimon/format/blob/blob_format_writer.h | 4 + .../format/orc/orc_file_batch_reader.cpp | 19 + .../format/orc/orc_file_batch_reader_test.cpp | 93 ++ src/paimon/format/orc/orc_format_writer.cpp | 24 + src/paimon/format/orc/orc_format_writer.h | 2 + .../format/orc/orc_format_writer_test.cpp | 2 + src/paimon/format/orc/orc_reader_wrapper.h | 8 + .../parquet_file_batch_reader_test.cpp | 93 ++ .../format/parquet/parquet_format_writer.cpp | 16 + .../format/parquet/parquet_format_writer.h | 4 + .../parquet/parquet_format_writer_test.cpp | 1 + .../testing/mock/mock_format_writer.cpp | 5 + src/paimon/testing/mock/mock_format_writer.h | 3 + 60 files changed, 3773 insertions(+), 270 deletions(-) create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_column_allocator_test.cpp create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_context.cpp create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_context.h create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_field_dict.h create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_field_dict_test.cpp diff --git a/include/paimon/format/format_writer.h b/include/paimon/format/format_writer.h index cf570b3e..854a0a9b 100644 --- a/include/paimon/format/format_writer.h +++ b/include/paimon/format/format_writer.h @@ -18,8 +18,11 @@ #pragma once +#include #include +#include +#include "paimon/status.h" #include "paimon/type_fwd.h" struct ArrowArray; @@ -68,6 +71,10 @@ class PAIMON_EXPORT FormatWriter { /// Get metrics of the writer /// @return The accumulated writer metrics to current state. virtual std::shared_ptr GetWriterMetrics() const = 0; + + /// Adds metadata to the file footer. Values are encoded by each format writer + /// before being persisted. Must be called before Finish(). + virtual Status AddMetadata(const std::map& metadata) = 0; }; } // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index b9ef998a..125ef23b 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -141,6 +141,9 @@ set(PAIMON_COMMON_SRCS common/utils/crc32c.cpp common/utils/decimal_utils.cpp common/data/shredding/map_shared_shredding_utils.cpp + common/data/shredding/map_shared_shredding_context.cpp + common/data/shredding/map_shared_shredding_batch_converter.cpp + common/data/shredding/map_shared_shredding_column_allocator.cpp common/utils/delta_varint_compressor.cpp common/utils/fields_comparator.cpp common/utils/path_util.cpp @@ -536,6 +539,10 @@ if(PAIMON_BUILD_TESTS) common/utils/threadsafe_queue_test.cpp common/utils/generic_lru_cache_test.cpp common/data/shredding/map_shared_shredding_utils_test.cpp + common/data/shredding/map_shared_shredding_batch_converter_test.cpp + common/data/shredding/map_shared_shredding_column_allocator_test.cpp + common/data/shredding/map_shared_shredding_field_dict_test.cpp + common/data/shredding/map_shared_shredding_context_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp new file mode 100644 index 00000000..1bc89a65 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" + +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/builder.h" +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +namespace paimon { +/// Checks that a dynamic_cast result is not null, returning Status::Invalid on failure. +#define PAIMON_CHECK_NOT_NULL(ptr, msg) \ + do { \ + if (PAIMON_UNLIKELY((ptr) == nullptr)) { \ + return Status::Invalid(msg); \ + } \ + } while (false) + +Result +MapSharedShreddingBatchConverter::CreateConverter( + const std::shared_ptr& logical_schema, + const std::shared_ptr& context, + const std::shared_ptr& pool) { + ConverterBundle bundle; + if (!context) { + return bundle; + } + + std::map field_to_k = context->ComputeNextK(); + PAIMON_ASSIGN_OR_RAISE(bundle.physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_k)); + bundle.converter = std::make_shared( + logical_schema, bundle.physical_schema, field_to_k, pool); + return bundle; +} + +MapSharedShreddingBatchConverter::MapSharedShreddingBatchConverter( + const std::shared_ptr& logical_schema, + const std::shared_ptr& physical_schema, + const std::map& field_to_num_columns, + const std::shared_ptr& pool) + : logical_schema_(logical_schema), + physical_schema_(physical_schema), + pool_(GetArrowPool(pool)) { + // Iterate in schema field order (not map order) so that shredding_field_names_ + // matches the order in which shredding columns appear in the schema. + // This is critical for the sequential matching logic in Convert(). + for (int32_t i = 0; i < logical_schema->num_fields(); ++i) { + const std::string& name = logical_schema->field(i)->name(); + auto it = field_to_num_columns.find(name); + if (it != field_to_num_columns.end()) { + contexts_.emplace_back(name, it->second); + shredding_field_names_.push_back(name); + } + } +} + +Result> MapSharedShreddingBatchConverter::Convert( + ArrowArray* logical_batch) { + std::shared_ptr logical_type = arrow::struct_(logical_schema_->fields()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_array, + arrow::ImportArray(logical_batch, logical_type)); + auto logical_struct = std::dynamic_pointer_cast(logical_array); + PAIMON_CHECK_NOT_NULL(logical_struct, + "MapSharedShreddingBatchConverter: input is not a StructArray"); + + int32_t num_fields = logical_schema_->num_fields(); + arrow::ArrayVector physical_columns; + physical_columns.reserve(num_fields); + size_t context_idx = 0; + for (int32_t col = 0; col < num_fields; ++col) { + auto column = logical_struct->field(col); + const std::string& field_name = logical_schema_->field(col)->name(); + if (context_idx < shredding_field_names_.size() && + shredding_field_names_[context_idx] == field_name) { + auto physical_struct_type = physical_schema_->field(col)->type(); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr physical_column, + ConvertOneColumn(column, physical_struct_type, &contexts_[context_idx])); + physical_columns.push_back(std::move(physical_column)); + ++context_idx; + } else { + physical_columns.push_back(column); + } + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr physical_struct, + arrow::StructArray::Make(physical_columns, physical_schema_->field_names())); + + std::unique_ptr result = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*physical_struct, result.get())); + return result; +} + +Result> MapSharedShreddingBatchConverter::ConvertOneColumn( + const std::shared_ptr& map_column, + const std::shared_ptr& physical_struct_type, ColumnContext* context) const { + auto map_array = std::dynamic_pointer_cast(map_column); + PAIMON_CHECK_NOT_NULL(map_array, "MapSharedShreddingBatchConverter: column is not a MapArray"); + + int64_t num_rows = map_array->length(); + int32_t num_cols = context->num_columns; + + auto keys_array = std::dynamic_pointer_cast(map_array->keys()); + PAIMON_CHECK_NOT_NULL(keys_array, + "MapSharedShreddingBatchConverter: MAP keys are not StringArray"); + auto values_array = map_array->items(); + + // Create StructBuilder from physical struct type — it owns all child builders. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr struct_builder_base, + arrow::MakeBuilder(physical_struct_type, pool_.get())); + auto* struct_builder = dynamic_cast(struct_builder_base.get()); + PAIMON_CHECK_NOT_NULL(struct_builder, + "MapSharedShreddingBatchConverter: failed to create StructBuilder"); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Reserve(num_rows)); + + // Extract child builders: [field_mapping, col_0..K-1, overflow] + auto* field_mapping_builder = + dynamic_cast(struct_builder->field_builder(0)); + PAIMON_CHECK_NOT_NULL(field_mapping_builder, + "MapSharedShreddingBatchConverter: field_mapping is not a ListBuilder"); + auto* field_mapping_value_builder = + dynamic_cast(field_mapping_builder->value_builder()); + PAIMON_CHECK_NOT_NULL( + field_mapping_value_builder, + "MapSharedShreddingBatchConverter: field_mapping value is not Int32Builder"); + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_mapping_builder->Reserve(num_rows)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_mapping_value_builder->Reserve(num_rows * num_cols)); + + std::vector col_builders_raw; + col_builders_raw.reserve(num_cols); + for (int32_t c = 0; c < num_cols; ++c) { + arrow::ArrayBuilder* col_builder = struct_builder->field_builder(1 + c); + PAIMON_CHECK_NOT_NULL(col_builder, "MapSharedShreddingBatchConverter: col builder is null"); + PAIMON_RETURN_NOT_OK_FROM_ARROW(col_builder->Reserve(num_rows)); + col_builders_raw.push_back(col_builder); + } + + int32_t overflow_field_idx = 1 + num_cols; + auto* overflow_builder = + dynamic_cast(struct_builder->field_builder(overflow_field_idx)); + PAIMON_CHECK_NOT_NULL(overflow_builder, + "MapSharedShreddingBatchConverter: overflow is not a MapBuilder"); + auto* overflow_key_builder = + dynamic_cast(overflow_builder->key_builder()); + PAIMON_CHECK_NOT_NULL(overflow_key_builder, + "MapSharedShreddingBatchConverter: overflow key is not Int32Builder"); + arrow::ArrayBuilder* overflow_value_builder = overflow_builder->item_builder(); + PAIMON_CHECK_NOT_NULL(overflow_value_builder, + "MapSharedShreddingBatchConverter: overflow value builder is null"); + PAIMON_RETURN_NOT_OK_FROM_ARROW(overflow_builder->Reserve(num_rows)); + + // Process each row + for (int64_t row = 0; row < num_rows; ++row) { + if (map_array->IsNull(row)) { + // StructBuilder::AppendNull() auto-appends empty values to all children. + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->AppendNull()); + continue; + } + + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + + int64_t start = map_array->value_offset(row); + int64_t length = map_array->value_length(row); + + // Extract field ids and build lookup map + std::vector field_ids; + std::unordered_map field_id_to_value_index; + ExtractRowFields(keys_array, start, length, &context->dict, &field_ids, + &field_id_to_value_index); + + // Allocate columns + RowAllocation allocation = context->allocator.AllocateRow(field_ids); + + // Fill sub-columns + PAIMON_RETURN_NOT_OK(AppendFieldMapping(allocation, num_cols, field_mapping_builder, + field_mapping_value_builder)); + PAIMON_RETURN_NOT_OK(AppendColumnValues(values_array, allocation, field_id_to_value_index, + num_cols, col_builders_raw)); + PAIMON_RETURN_NOT_OK(AppendOverflow(values_array, allocation, field_id_to_value_index, + overflow_builder, overflow_key_builder, + overflow_value_builder)); + } + + // Finalize + std::shared_ptr result; + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Finish(&result)); + return result; +} + +void MapSharedShreddingBatchConverter::ExtractRowFields( + const std::shared_ptr& keys_array, int64_t start, int64_t length, + MapSharedShreddingFieldDict* dict, std::vector* field_ids_out, + std::unordered_map* field_id_to_value_index_out) const { + field_ids_out->clear(); + field_ids_out->reserve(length); + field_id_to_value_index_out->clear(); + field_id_to_value_index_out->reserve(length); + for (int64_t j = 0; j < length; ++j) { + std::string key_str = keys_array->GetString(start + j); + int32_t field_id = dict->GetOrAssign(key_str); + field_ids_out->push_back(field_id); + (*field_id_to_value_index_out)[field_id] = start + j; + } +} + +Status MapSharedShreddingBatchConverter::AppendFieldMapping( + const RowAllocation& allocation, int32_t num_cols, arrow::ListBuilder* list_builder, + arrow::Int32Builder* value_builder) const { + PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder->Append()); + for (int32_t c = 0; c < num_cols; ++c) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(allocation.col_to_field[c])); + } + return Status::OK(); +} + +Status MapSharedShreddingBatchConverter::AppendColumnValues( + const std::shared_ptr& values_array, const RowAllocation& allocation, + const std::unordered_map& field_id_to_value_index, int32_t num_cols, + const std::vector& col_builders) const { + for (int32_t c = 0; c < num_cols; ++c) { + int32_t assigned_field_id = allocation.col_to_field[c]; + if (assigned_field_id == -1) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(col_builders[c]->AppendNull()); + } else { + auto it = field_id_to_value_index.find(assigned_field_id); + if (PAIMON_UNLIKELY(it == field_id_to_value_index.end())) { + return Status::Invalid( + fmt::format("MapSharedShreddingBatchConverter: field_id {} assigned to col {} " + "but not found in current row", + assigned_field_id, c)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW( + col_builders[c]->AppendArraySlice(*values_array->data(), it->second, 1)); + } + } + return Status::OK(); +} + +Status MapSharedShreddingBatchConverter::AppendOverflow( + const std::shared_ptr& values_array, const RowAllocation& allocation, + const std::unordered_map& field_id_to_value_index, + arrow::MapBuilder* overflow_builder, arrow::Int32Builder* overflow_key_builder, + arrow::ArrayBuilder* overflow_value_builder) const { + if (allocation.overflow_fields.empty()) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(overflow_builder->AppendNull()); + return Status::OK(); + } + + PAIMON_RETURN_NOT_OK_FROM_ARROW(overflow_builder->Append()); + for (int32_t overflow_field_id : allocation.overflow_fields) { + auto it = field_id_to_value_index.find(overflow_field_id); + if (PAIMON_UNLIKELY(it == field_id_to_value_index.end())) { + return Status::Invalid(fmt::format( + "MapSharedShreddingBatchConverter: overflow field_id {} not found in current row", + overflow_field_id)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(overflow_key_builder->Append(overflow_field_id)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + overflow_value_builder->AppendArraySlice(*values_array->data(), it->second, 1)); + } + return Status::OK(); +} + +Result MapSharedShreddingBatchConverter::BuildFieldMeta( + const std::string& field_name) const { + for (const auto& context : contexts_) { + if (context.field_name == field_name) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = context.dict.GetNameToId(); + // Convert set -> vector for field_to_columns + for (const auto& [field_id, col_set] : context.allocator.GetFieldToColumns()) { + meta.field_to_columns[field_id] = + std::vector(col_set.begin(), col_set.end()); + } + meta.overflow_field_set = context.allocator.GetOverflowFieldSet(); + meta.num_columns = context.allocator.GetNumColumns(); + meta.max_row_width = context.allocator.GetMaxRowWidth(); + return meta; + } + } + return Status::Invalid(fmt::format( + "cannot find field_name '{}' in MapSharedShreddingBatchConverter contexts", field_name)); +} + +const std::vector& MapSharedShreddingBatchConverter::GetShreddingColumnNames() const { + return shredding_field_names_; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h new file mode 100644 index 00000000..8ec30007 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/memory_pool.h" +#include "arrow/type_fwd.h" +#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" +#include "paimon/common/data/shredding/map_shared_shredding_field_dict.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" +#include "paimon/status.h" + +struct ArrowArray; + +namespace paimon { + +class MapSharedShreddingContext; + +/// Converts logical batches containing MAP columns into physical batches +/// where each shared-shredding MAP column is replaced by +/// STRUCT<__field_mapping, __col_0..K-1, __overflow>. +/// +/// Non-shared-shredding columns are passed through unchanged. +/// Each shared-shredding column has its own FieldDict and ColumnAllocator. +class MapSharedShreddingBatchConverter { + public: + /// Per-column context for one shared-shredding MAP column. + struct ColumnContext { + std::string field_name; + int32_t num_columns; // K + MapSharedShreddingFieldDict dict; + MapSharedShreddingColumnAllocator allocator; + + ColumnContext(const std::string& _field_name, int32_t _num_columns) + : field_name(_field_name), num_columns(_num_columns), allocator(_num_columns) {} + }; + + struct ConverterBundle { + std::shared_ptr converter; + std::shared_ptr physical_schema; + }; + + /// Creates a converter + physical schema for one file write cycle. + /// Computes per-file K from context, builds physical schema, and constructs the converter. + /// @param logical_schema The original schema with MAP columns. + /// @param context The cross-file shared context for K adaptation. + /// @param pool Paimon memory pool for Arrow allocations. + /// @return A struct containing the converter and physical schema. + static Result CreateConverter( + const std::shared_ptr& logical_schema, + const std::shared_ptr& context, + const std::shared_ptr& pool); + + /// Constructs a converter. + /// @param logical_schema The original schema with MAP columns. + /// @param physical_schema The physical schema (MAP columns replaced with STRUCT). + /// @param field_to_num_columns Map from field name to K. + /// @param pool Paimon memory pool for Arrow allocations. + MapSharedShreddingBatchConverter(const std::shared_ptr& logical_schema, + const std::shared_ptr& physical_schema, + const std::map& field_to_num_columns, + const std::shared_ptr& pool); + + /// Converts a logical batch to a physical batch. + /// @param logical_batch Input ArrowArray (C ABI) with logical schema. Consumed on success. + /// @return Owned physical ArrowArray (C ABI) with physical schema. + Result> Convert(ArrowArray* logical_batch); + + /// Builds MapSharedShreddingFieldMeta for one shredding column (by field name). + /// Called at file close to serialize metadata. + Result BuildFieldMeta(const std::string& field_name) const; + + /// Returns all shredding column field names. + const std::vector& GetShreddingColumnNames() const; + + private: + /// Converts one MAP column to physical STRUCT for all rows. + /// @param physical_struct_type The physical struct type from physical_schema for this column. + Result> ConvertOneColumn( + const std::shared_ptr& map_column, + const std::shared_ptr& physical_struct_type, ColumnContext* context) const; + + /// Extracts field ids and builds field_id -> value_index map for one row. + void ExtractRowFields(const std::shared_ptr& keys_array, int64_t start, + int64_t length, MapSharedShreddingFieldDict* dict, + std::vector* field_ids_out, + std::unordered_map* field_id_to_value_index_out) const; + + /// Appends __field_mapping list for one row. + Status AppendFieldMapping(const RowAllocation& allocation, int32_t num_cols, + arrow::ListBuilder* list_builder, + arrow::Int32Builder* value_builder) const; + + /// Appends __col_0..K-1 values for one row. + Status AppendColumnValues(const std::shared_ptr& values_array, + const RowAllocation& allocation, + const std::unordered_map& field_id_to_value_index, + int32_t num_cols, + const std::vector& col_builders) const; + + /// Appends __overflow entries for one row. + Status AppendOverflow(const std::shared_ptr& values_array, + const RowAllocation& allocation, + const std::unordered_map& field_id_to_value_index, + arrow::MapBuilder* overflow_builder, + arrow::Int32Builder* overflow_key_builder, + arrow::ArrayBuilder* overflow_value_builder) const; + + std::shared_ptr logical_schema_; + std::shared_ptr physical_schema_; + std::vector contexts_; + std::vector shredding_field_names_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp new file mode 100644 index 00000000..643dad42 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" + +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon { + +using arrow::ipc::internal::json::ArrayFromJSON; + +class MapSharedShreddingBatchConverterTest : public ::testing::Test { + protected: + std::shared_ptr pool_ = GetDefaultPool(); + + /// Builds a logical struct array from JSON, converts it, and returns the physical result. + std::shared_ptr RunConvert(const std::shared_ptr& logical_type, + const std::string& input_json, + const std::shared_ptr& physical_type, + MapSharedShreddingBatchConverter* converter) { + auto input = ArrayFromJSON(logical_type, input_json).ValueOrDie(); + ArrowArray c_input; + EXPECT_TRUE(arrow::ExportArray(*input, &c_input).ok()); + EXPECT_OK_AND_ASSIGN(auto c_output, converter->Convert(&c_input)); + return arrow::ImportArray(c_output.get(), physical_type).ValueOrDie(); + } + + /// Asserts that two arrays are equal, printing both on failure. + void AssertArrayEquals(const std::shared_ptr& expected, + const std::shared_ptr& actual) { + ASSERT_TRUE(expected->Equals(*actual)) << "Expected:\n" + << expected->ToString() << "\nActual:\n" + << actual->ToString(); + } +}; + +TEST_F(MapSharedShreddingBatchConverterTest, BasicConversion) { + // Schema: id(INT32), tags(MAP), K=3 + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + std::map field_to_num_columns = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); + MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, + field_to_num_columns, pool_); + + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Input: 2 rows — [id, tags] + // Row0: id=100, tags={a:1, b:2} + // Row1: id=200, tags={b:3, c:4, a:5} + auto actual = RunConvert(logical_type, R"([ + [100, [["a", 1], ["b", 2]]], + [200, [["b", 3], ["c", 4], ["a", 5]]] + ])", + physical_type, &converter); + // Expected physical: [id, [mapping, col0, col1, col2, overflow]] + // Row0: a=fid0->col0, b=fid1->col1, col2 unused + // Row1: b=fid1->col0, c=fid2->col1, a=fid0->col2 + auto expected = ArrayFromJSON(physical_type, R"([ + [100, [[0, 1, -1], 1, 2, null, null]], + [200, [[1, 2, 0], 3, 4, 5, null]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"tags"}), converter.GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2, K=3, max_row_width=3, no overflow + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0, 2}}, {1, {0, 1}}, {2, {1}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter.BuildFieldMeta("tags").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, NestedValueStruct) { + // MAP>, K=2 + auto value_type = arrow::struct_({ + arrow::field("x", arrow::int32()), + arrow::field("y", arrow::float64()), + }); + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("props", arrow::map(arrow::utf8(), value_type)), + }); + std::map field_to_num_columns = {{"props", 2}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); + MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, + field_to_num_columns, pool_); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: props={a:[1,1.5], b:[null,2.5]} → a=fid0->col0, b=fid1->col1; b.x is null + // Row1: props={c:[3,3.5]} → c=fid2->col0 + // Row2: props={a:[null,null], c:[5,5.5], b:[6,6.5]} → 3 fields K=2: overflow b; a has all-null + // struct Row3: props=null → null row + auto actual = RunConvert(logical_type, R"([ + [1, [["a", [1, 1.5]], ["b", [null, 2.5]]]], + [2, [["c", [3, 3.5]]]], + [3, [["a", [null, null]], ["c", [5, 5.5]], ["b", [6, 6.5]]]], + [4, null] + ])", + physical_type, &converter); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], [1, 1.5], [null, 2.5], null]], + [2, [[2, -1], [3, 3.5], null, null]], + [3, [[0, 2], [null, null], [5, 5.5], [[1, [6, 6.5]]]]], + [4, null] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"props"}), converter.GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, b overflowed in row2 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0, 1}}}; + expected_meta.overflow_field_set = {1}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter.BuildFieldMeta("props").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, NestedValueList) { + // MAP>, K=2 + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::list(arrow::int32()))), + }); + std::map field_to_num_columns = {{"tags", 2}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); + MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, + field_to_num_columns, pool_); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: tags={a:[1,null,2], b:[3]} → a=fid0->col0, b=fid1->col1; a has null element + // Row1: tags={a:[null]} → a=fid0->col0; single null element list + // Row2: tags={c:[5,6,7]} → c=fid2->col0 + // Row3: tags={b:[8], a:[9,10], c:[null]} → 3 fields K=2: overflow c; c has null element + auto actual = RunConvert(logical_type, R"([ + [1, [["a", [1, null, 2]], ["b", [3]]]], + [2, [["a", [null]]]], + [3, [["c", [5, 6, 7]]]], + [4, [["b", [8]], ["a", [9, 10]], ["c", [null]]]] + ])", + physical_type, &converter); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], [1, null, 2], [3], null]], + [2, [[0, -1], [null], null, null]], + [3, [[2, -1], [5, 6, 7], null, null]], + [4, [[1, 0], [8], [9, 10], [[2, [null]]]]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"tags"}), converter.GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, c overflowed in row3 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0, 1}}, {1, {0, 1}}, {2, {0}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter.BuildFieldMeta("tags").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, NestedValueMap) { + // MAP>, K=2 + auto inner_map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("nested", arrow::map(arrow::utf8(), inner_map_type)), + }); + std::map field_to_num_columns = {{"nested", 2}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); + MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, + field_to_num_columns, pool_); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: nested={a:{x:1,y:null}, b:{z:3}} → a=fid0->col0, b=fid1->col1; a has null value + // Row1: nested={c:{p:null}} → c=fid2->col0; inner map value all null + // Row2: nested=null → null row + // Row3: nested={a:{m:7}, b:{n:8}, c:{o:9}} → 3 fields K=2: overflow c + auto actual = RunConvert(logical_type, R"([ + [1, [["a", [["x", 1], ["y", null]]], ["b", [["z", 3]]]]], + [2, [["c", [["p", null]]]]], + [3, null], + [4, [["a", [["m", 7]]], ["b", [["n", 8]]], ["c", [["o", 9]]]]] + ])", + physical_type, &converter); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], [["x", 1], ["y", null]], [["z", 3]], null]], + [2, [[2, -1], [["p", null]], null, null]], + [3, null], + [4, [[0, 1], [["m", 7]], [["n", 8]], [[2, [["o", 9]]]]]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"nested"}), converter.GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, c overflowed in row3 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter.BuildFieldMeta("nested").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, NestedComplex) { + // MAP, meta:MAP>>, K=2 + auto value_type = arrow::struct_({ + arrow::field("score", arrow::int32()), + arrow::field("tags", arrow::list(arrow::utf8())), + arrow::field("meta", arrow::map(arrow::utf8(), arrow::int32())), + }); + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("data", arrow::map(arrow::utf8(), value_type)), + }); + std::map field_to_num_columns = {{"data", 2}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); + MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, + field_to_num_columns, pool_); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: a=[10,["t1","t2"],{x:1}], b=[20,["t3"],{y:2,z:3}] → a=fid0->col0, b=fid1->col1 + // Row1: c=[null,null,{p:null}] → c=fid2->col0; nulls inside + // struct Row2: a=[30,[null,"t4"],{}], b=[null,[],{q:5}], c=[40,["t5"],{r:6}] → overflow c Row3: + // null → null row + auto actual = RunConvert(logical_type, R"([ + [1, [["a", [10, ["t1", "t2"], [["x", 1]]]], ["b", [20, ["t3"], [["y", 2], ["z", 3]]]]]], + [2, [["c", [null, null, [["p", null]]]]]], + [3, [["a", [30, [null, "t4"], []]], ["b", [null, [], [["q", 5]]]], ["c", [40, ["t5"], [["r", 6]]]]]], + [4, null] + ])", + physical_type, &converter); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], [10, ["t1", "t2"], [["x", 1]]], [20, ["t3"], [["y", 2], ["z", 3]]], null]], + [2, [[2, -1], [null, null, [["p", null]]], null, null]], + [3, [[0, 1], [30, [null, "t4"], []], [null, [], [["q", 5]]], [[2, [40, ["t5"], [["r", 6]]]]]]], + [4, null] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames + ASSERT_EQ(std::vector({"data"}), converter.GetShreddingColumnNames()); + + // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, c overflowed in row2 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter.BuildFieldMeta("data").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, MultipleMapFields) { + // Schema: id(INT32), tags(MAP) K=2, attrs(MAP) K=3 + // Tests that two shared-shredding MAP columns in the same schema are independently converted. + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::float64())), + }); + std::map field_to_num_columns = {{"tags", 2}, {"attrs", 3}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); + + MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, + field_to_num_columns, pool_); + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + // Row0: id=1, tags={a:10, b:20}, attrs={x:1.1, y:2.2} + // tags: a=fid0->col0, b=fid1->col1 (fits K=2) + // attrs: x=fid0->col0, y=fid1->col1, col2 unused (fits K=3) + // Row1: id=2, tags={c:30, a:40, b:50}, attrs={z:3.3} + // tags: c=fid2->col0, a=fid0->col1; b overflows (K=2) + // attrs: z=fid2->col0, col1/col2 unused + // Row2: id=3, tags=null, attrs={x:4.4, y:5.5, z:6.6, w:7.7} + // tags: null + // attrs: x=fid0->col0, y=fid1->col1, z=fid2->col2; w overflows (K=3) + auto actual = RunConvert(logical_type, R"([ + [1, [["a", 10], ["b", 20]], [["x", 1.1], ["y", 2.2]]], + [2, [["c", 30], ["a", 40], ["b", 50]], [["z", 3.3]]], + [3, null, [["x", 4.4], ["y", 5.5], ["z", 6.6], ["w", 7.7]]] + ])", + physical_type, &converter); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1], 10, 20, null], [[0, 1, -1], 1.1, 2.2, null, null]], + [2, [[2, 0], 30, 40, [[1, 50]]], [[2, -1, -1], 3.3, null, null, null]], + [3, null, [[0, 1, 2], 4.4, 5.5, 6.6, [[3, 7.7]]]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + // Verify GetShreddingColumnNames returns both columns in order + ASSERT_EQ(std::vector({"tags", "attrs"}), converter.GetShreddingColumnNames()); + + // Verify BuildFieldMeta for tags: a=0,b=1,c=2; K=2, max_row_width=3 + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + tags_meta.field_to_columns = {{0, {0, 1}}, {1, {1}}, {2, {0}}}; + tags_meta.overflow_field_set = {1}; + tags_meta.num_columns = 2; + tags_meta.max_row_width = 3; + ASSERT_EQ(tags_meta, converter.BuildFieldMeta("tags").value()); + + // Verify BuildFieldMeta for attrs: x=0,y=1,z=2,w=3; K=3, max_row_width=4 + MapSharedShreddingFieldMeta attrs_meta; + attrs_meta.name_to_id = {{"x", 0}, {"y", 1}, {"z", 2}, {"w", 3}}; + attrs_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0, 2}}}; + attrs_meta.overflow_field_set = {3}; + attrs_meta.num_columns = 3; + attrs_meta.max_row_width = 4; + ASSERT_EQ(attrs_meta, converter.BuildFieldMeta("attrs").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, BuildFieldMetaInvalidFieldName) { + // Schema: id(INT32), tags(MAP), K=3 + // Only "tags" is a shredding field + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + std::map field_to_num_columns = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); + MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, + field_to_num_columns, pool_); + + // Valid case: "tags" exists + ASSERT_OK_AND_ASSIGN([[maybe_unused]] auto meta, converter.BuildFieldMeta("tags")); + + // Invalid case: "id" is not a shredding field + ASSERT_NOK_WITH_MSG(converter.BuildFieldMeta("id"), "cannot find field_name 'id'"); + + // Invalid case: nonexistent field name + ASSERT_NOK_WITH_MSG(converter.BuildFieldMeta("nonexistent"), + "cannot find field_name 'nonexistent'"); +} +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp new file mode 100644 index 00000000..b0bedfe5 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" + +#include + +namespace paimon { + +MapSharedShreddingColumnAllocator::MapSharedShreddingColumnAllocator(int32_t num_columns) + : num_columns_(num_columns) {} + +RowAllocation MapSharedShreddingColumnAllocator::AllocateRow( + const std::vector& field_ids) { + max_row_width_ = std::max(max_row_width_, static_cast(field_ids.size())); + + RowAllocation result; + result.col_to_field.assign(num_columns_, -1); + int32_t assign_limit = std::min(static_cast(field_ids.size()), num_columns_); + + for (int32_t i = 0; i < assign_limit; ++i) { + int32_t field_id = field_ids[i]; + result.col_to_field[i] = field_id; + field_to_columns_[field_id].insert(i); + } + + for (int32_t i = assign_limit; i < static_cast(field_ids.size()); ++i) { + int32_t field_id = field_ids[i]; + result.overflow_fields.push_back(field_id); + overflow_field_set_.insert(field_id); + } + + return result; +} + +const std::map>& MapSharedShreddingColumnAllocator::GetFieldToColumns() + const { + return field_to_columns_; +} + +const std::set& MapSharedShreddingColumnAllocator::GetOverflowFieldSet() const { + return overflow_field_set_; +} + +int32_t MapSharedShreddingColumnAllocator::GetMaxRowWidth() const { + return max_row_width_; +} + +int32_t MapSharedShreddingColumnAllocator::GetNumColumns() const { + return num_columns_; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h new file mode 100644 index 00000000..1e115a77 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace paimon { + +/// Per-row allocation result produced by MapSharedShreddingColumnAllocator. +struct RowAllocation { + /// Physical column assignments: col_to_field[col_index] = field_id. + /// Length is always K. Unused columns have value -1. + std::vector col_to_field; + + /// Field ids that overflowed (more fields than K physical columns). + std::vector overflow_fields; +}; + +/// Allocates MAP field ids to K physical columns on a per-row basis, +/// and accumulates field-level metadata (field_to_columns, overflow_field_set, max_row_width). +/// +/// This is a trivial implementation: each row simply assigns columns 0..min(N,K)-1 +/// in order, with no LRU eviction. +/// TODO(jinli.zjw): support LRU +class MapSharedShreddingColumnAllocator { + public: + /// @param num_columns Number of physical columns K for this shared-shredding MAP column. + explicit MapSharedShreddingColumnAllocator(int32_t num_columns); + + /// Allocates physical columns for one row's field ids. + /// @param field_ids The field ids present in this row (order matters for fake impl). + /// @return Allocation result with column assignments and overflow list. + RowAllocation AllocateRow(const std::vector& field_ids); + + /// Returns accumulated field_id -> set of column indices (for MapSharedShreddingFileMeta). + const std::map>& GetFieldToColumns() const; + + /// Returns accumulated overflow field id set (for MapSharedShreddingFileMeta). + const std::set& GetOverflowFieldSet() const; + + /// Returns the maximum row width observed so far. + int32_t GetMaxRowWidth() const; + + /// Returns the number of physical columns K. + int32_t GetNumColumns() const; + + private: + int32_t num_columns_; + + // ---- Accumulated field-level metadata ---- + std::map> field_to_columns_; + std::set overflow_field_set_; + int32_t max_row_width_ = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator_test.cpp new file mode 100644 index 00000000..b83c56e3 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator_test.cpp @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" + +#include "gtest/gtest.h" + +namespace paimon { + +TEST(MapSharedShreddingColumnAllocatorTest, BasicAllocation) { + MapSharedShreddingColumnAllocator allocator(3); + + // 2 fields, K=3 -> all fit, no overflow + auto result = allocator.AllocateRow({10, 20}); + ASSERT_EQ(std::vector({10, 20, -1}), result.col_to_field); + ASSERT_TRUE(result.overflow_fields.empty()); +} + +TEST(MapSharedShreddingColumnAllocatorTest, ExactlyKFields) { + MapSharedShreddingColumnAllocator allocator(3); + + auto result = allocator.AllocateRow({0, 1, 2}); + ASSERT_EQ(std::vector({0, 1, 2}), result.col_to_field); + ASSERT_TRUE(result.overflow_fields.empty()); +} + +TEST(MapSharedShreddingColumnAllocatorTest, OverflowWhenExceedK) { + MapSharedShreddingColumnAllocator allocator(2); + + // 4 fields, K=2 -> first 2 assigned, last 2 overflow + auto result = allocator.AllocateRow({10, 20, 30, 40}); + ASSERT_EQ(std::vector({10, 20}), result.col_to_field); + ASSERT_EQ(std::vector({30, 40}), result.overflow_fields); +} + +TEST(MapSharedShreddingColumnAllocatorTest, EmptyRow) { + MapSharedShreddingColumnAllocator allocator(3); + + auto result = allocator.AllocateRow({}); + ASSERT_EQ(std::vector({-1, -1, -1}), result.col_to_field); + ASSERT_TRUE(result.overflow_fields.empty()); +} + +TEST(MapSharedShreddingColumnAllocatorTest, MaxRowWidthTracked) { + MapSharedShreddingColumnAllocator allocator(3); + + allocator.AllocateRow({1, 2}); + ASSERT_EQ(2, allocator.GetMaxRowWidth()); + + allocator.AllocateRow({1, 2, 3, 4, 5}); + ASSERT_EQ(5, allocator.GetMaxRowWidth()); + + allocator.AllocateRow({1}); + ASSERT_EQ(5, allocator.GetMaxRowWidth()); +} + +TEST(MapSharedShreddingColumnAllocatorTest, FieldToColumnsAccumulated) { + MapSharedShreddingColumnAllocator allocator(3); + + allocator.AllocateRow({10, 20, 30}); + allocator.AllocateRow({20, 40}); + + auto field_to_cols = allocator.GetFieldToColumns(); + // field 10 -> {0} + ASSERT_EQ(std::set({0}), field_to_cols.at(10)); + // field 20 -> {1, 0} (col 1 in row 0, col 0 in row 1) + ASSERT_EQ(std::set({0, 1}), field_to_cols.at(20)); + // field 30 -> {2} + ASSERT_EQ(std::set({2}), field_to_cols.at(30)); + // field 40 -> {1} + ASSERT_EQ(std::set({1}), field_to_cols.at(40)); +} + +TEST(MapSharedShreddingColumnAllocatorTest, OverflowFieldSetAccumulated) { + MapSharedShreddingColumnAllocator allocator(2); + + allocator.AllocateRow({1, 2, 3}); // 3 overflows + allocator.AllocateRow({4, 5, 6, 7}); // 6, 7 overflow + + auto overflow_set = allocator.GetOverflowFieldSet(); + ASSERT_EQ(std::set({3, 6, 7}), overflow_set); +} + +TEST(MapSharedShreddingColumnAllocatorTest, GetNumColumns) { + MapSharedShreddingColumnAllocator allocator(5); + ASSERT_EQ(5, allocator.GetNumColumns()); +} + +TEST(MapSharedShreddingColumnAllocatorTest, SingleColumnAllocator) { + MapSharedShreddingColumnAllocator allocator(1); + + auto result = allocator.AllocateRow({10, 20, 30}); + ASSERT_EQ(std::vector({10}), result.col_to_field); + ASSERT_EQ(std::vector({20, 30}), result.overflow_fields); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context.cpp b/src/paimon/common/data/shredding/map_shared_shredding_context.cpp new file mode 100644 index 00000000..9d5356f5 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_context.cpp @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_context.h" + +#include + +namespace paimon { + +MapSharedShreddingContext::MapSharedShreddingContext( + const std::map& column_to_k_max) + : column_to_k_max_(column_to_k_max) {} + +std::map MapSharedShreddingContext::ComputeNextK() const { + std::map result; + for (const auto& [field_name, k_max] : column_to_k_max_) { + auto it = recent_max_row_widths_.find(field_name); + if (it == recent_max_row_widths_.end() || it->second.empty()) { + // First file — no history, use K_max. + result[field_name] = k_max; + } else { + int32_t window_max = ComputeWindowMax(it->second); + result[field_name] = std::max(1, std::min(window_max, k_max)); + } + } + return result; +} + +void MapSharedShreddingContext::ReportFileStats(const std::string& field_name, + int32_t max_row_width) { + auto& window = recent_max_row_widths_[field_name]; + window.push_back(max_row_width); + if (static_cast(window.size()) > kWindowSize) { + window.erase(window.begin()); + } +} + +std::vector MapSharedShreddingContext::GetShreddingColumnNames() const { + std::vector names; + names.reserve(column_to_k_max_.size()); + for (const auto& [field_name, _] : column_to_k_max_) { + names.push_back(field_name); + } + return names; +} + +int32_t MapSharedShreddingContext::ComputeWindowMax(const std::vector& values) { + if (values.empty()) { + return 0; + } + // TODO(xinyu.lxy): support P99 + return *std::max_element(values.begin(), values.end()); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context.h b/src/paimon/common/data/shredding/map_shared_shredding_context.h new file mode 100644 index 00000000..54466941 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_context.h @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace paimon { + +/// Cross-file shared context for shared-shredding MAP columns. +/// +/// Lifetime: same as the owning writer (e.g. AppendOnlyWriter). +/// Holds per-column K_max and a sliding window of recent max_row_width +/// values to support adaptive K sizing across files. +/// +/// - First file: K = K_max (no history). +/// - Subsequent files: K = min(max(recent_max_row_widths), K_max). +class MapSharedShreddingContext { + public: + /// @param column_to_k_max Map from field name to its K_max (from options). + explicit MapSharedShreddingContext(const std::map& column_to_k_max); + + /// Returns the K to use for each shared-shredding column in the next file. + /// First file returns K_max for all columns; subsequent files adapt + /// based on recent max_row_width observations. + std::map ComputeNextK() const; + + /// Reports the max row width observed in a completed file, for K adaptation. + /// @param field_name Field name of the shared-shredding MAP column. + /// @param max_row_width The maximum number of MAP keys in any single row of this file. + void ReportFileStats(const std::string& field_name, int32_t max_row_width); + + /// Returns the set of shared-shredding field names. + std::vector GetShreddingColumnNames() const; + + private: + static constexpr int32_t kWindowSize = 100; + + static int32_t ComputeWindowMax(const std::vector& values); + + /// K_max per shared-shredding field, from options. + std::map column_to_k_max_; + /// Sliding window of recent max_row_width per field, for K adaptation. + std::map> recent_max_row_widths_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp new file mode 100644 index 00000000..655f8de6 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_context.h" + +#include +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(MapSharedShreddingContextTest, FirstFileUsesKMax) { + // No history — ComputeNextK should return K_max for every column. + std::map field_to_k_max = {{"tags", 8}, {"metrics", 4}}; + MapSharedShreddingContext context(field_to_k_max); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(2, next_k.size()); + ASSERT_EQ(8, next_k.at("tags")); + ASSERT_EQ(4, next_k.at("metrics")); +} + +TEST(MapSharedShreddingContextTest, AdaptKAfterOneFile) { + // After reporting stats from one file, K should adapt to + // min(max_row_width, K_max). + std::map field_to_k_max = {{"m", 10}}; + MapSharedShreddingContext context(field_to_k_max); + + // First file uses K_max=10. + auto k1 = context.ComputeNextK(); + ASSERT_EQ(10, k1.at("m")); + + // Report: file had max_row_width=3 for field "m". + context.ReportFileStats("m", 3); + + // Second file: K = min(3, 10) = 3. + auto k2 = context.ComputeNextK(); + ASSERT_EQ(3, k2.at("m")); +} + +TEST(MapSharedShreddingContextTest, AdaptKCappedByKMax) { + // Even if max_row_width > K_max, K should be capped at K_max. + std::map field_to_k_max = {{"m", 5}}; + MapSharedShreddingContext context(field_to_k_max); + + context.ReportFileStats("m", 100); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(5, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, WindowMaxTracksLargest) { + // K should use the max of all recent max_row_widths within the window. + std::map field_to_k_max = {{"m", 20}}; + MapSharedShreddingContext context(field_to_k_max); + + context.ReportFileStats("m", 3); + context.ReportFileStats("m", 7); + context.ReportFileStats("m", 5); + + // max of {3, 7, 5} = 7, capped by K_max=20 → K=7. + auto next_k = context.ComputeNextK(); + ASSERT_EQ(7, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, MultipleColumnsIndependent) { + // Each field adapts independently. + std::map field_to_k_max = {{"tags", 10}, {"attrs", 6}}; + MapSharedShreddingContext context(field_to_k_max); + + // First file. + auto k1 = context.ComputeNextK(); + ASSERT_EQ(10, k1.at("tags")); + ASSERT_EQ(6, k1.at("attrs")); + + // Report: tags had width 4, attrs had width 2. + context.ReportFileStats("tags", 4); + context.ReportFileStats("attrs", 2); + + auto k2 = context.ComputeNextK(); + ASSERT_EQ(4, k2.at("tags")); + ASSERT_EQ(2, k2.at("attrs")); + + // Report: tags had width 8, attrs had width 6. + context.ReportFileStats("tags", 8); + context.ReportFileStats("attrs", 6); + + auto k3 = context.ComputeNextK(); + // tags: max(4,8)=8, capped by 10 → 8 + // attrs: max(2,6)=6, capped by 6 → 6 + ASSERT_EQ(8, k3.at("tags")); + ASSERT_EQ(6, k3.at("attrs")); +} + +TEST(MapSharedShreddingContextTest, GetShreddingColumnNames) { + std::map field_to_k_max = {{"tags", 8}, {"metrics", 4}, {"props", 16}}; + MapSharedShreddingContext context(field_to_k_max); + + auto names = context.GetShreddingColumnNames(); + ASSERT_EQ(names, std::vector({"metrics", "props", "tags"})); +} + +TEST(MapSharedShreddingContextTest, SlidingWindowEvictsOldEntries) { + // The window size is 100. After filling 100 entries, adding one more + // should evict the oldest. Verify that the evicted value no longer + // affects ComputeNextK. + std::map field_to_k_max = {{"m", 256}}; + MapSharedShreddingContext context(field_to_k_max); + + // Insert a large value as the first entry. + context.ReportFileStats("m", 200); + + // Fill the remaining 99 slots with small values. + for (int i = 0; i < 99; ++i) { + context.ReportFileStats("m", 3); + } + + // Window = [200, 3, 3, ..., 3] (100 entries). Max = 200. + auto k_before = context.ComputeNextK(); + ASSERT_EQ(200, k_before.at("m")); + + // Push one more — evicts the 200. + context.ReportFileStats("m", 5); + + // Window = [3, 3, ..., 3, 5] (100 entries). Max = 5. + auto k_after = context.ComputeNextK(); + ASSERT_EQ(5, k_after.at("m")); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/map_shared_shredding_field_dict.h b/src/paimon/common/data/shredding/map_shared_shredding_field_dict.h new file mode 100644 index 00000000..d6c6e4be --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_field_dict.h @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace paimon { + +/// File-level field name <-> field_id dictionary for shared-shredding MAP. +/// Assigns monotonically increasing ids to new field names within one file. +class MapSharedShreddingFieldDict { + public: + MapSharedShreddingFieldDict() = default; + + /// Looks up or assigns a field_id for the given field name. + /// New names get the next available id. + int32_t GetOrAssign(const std::string& name) { + auto iterator = name_to_id_.find(name); + if (iterator != name_to_id_.end()) { + return iterator->second; + } + int32_t field_id = next_id_++; + name_to_id_[name] = field_id; + return field_id; + } + + /// Returns the complete name -> field_id dictionary. + /// Used to populate MapSharedShreddingFieldMeta::name_to_id at file close. + const std::map& GetNameToId() const { + return name_to_id_; + } + + /// Returns the number of distinct field names seen so far. + int32_t Size() const { + return next_id_; + } + + private: + std::map name_to_id_; + int32_t next_id_ = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_field_dict_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_field_dict_test.cpp new file mode 100644 index 00000000..65c14051 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_field_dict_test.cpp @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_field_dict.h" + +#include "gtest/gtest.h" + +namespace paimon { + +TEST(MapSharedShreddingFieldDictTest, AssignMonotonicallyIncreasingIds) { + MapSharedShreddingFieldDict dict; + ASSERT_EQ(0, dict.GetOrAssign("cpu_usage")); + ASSERT_EQ(1, dict.GetOrAssign("mem_load")); + ASSERT_EQ(2, dict.GetOrAssign("disk_io")); + ASSERT_EQ(3, dict.Size()); +} + +TEST(MapSharedShreddingFieldDictTest, LookupReturnsExistingId) { + MapSharedShreddingFieldDict dict; + int32_t id = dict.GetOrAssign("alpha"); + ASSERT_EQ(id, dict.GetOrAssign("alpha")); + ASSERT_EQ(1, dict.Size()); +} + +TEST(MapSharedShreddingFieldDictTest, GetNameToId) { + MapSharedShreddingFieldDict dict; + dict.GetOrAssign("b_field"); + dict.GetOrAssign("a_field"); + + auto name_to_id = dict.GetNameToId(); + ASSERT_EQ(2u, name_to_id.size()); + ASSERT_EQ(1, name_to_id.at("a_field")); + ASSERT_EQ(0, name_to_id.at("b_field")); +} + +TEST(MapSharedShreddingFieldDictTest, EmptyDict) { + MapSharedShreddingFieldDict dict; + ASSERT_EQ(0, dict.Size()); + ASSERT_TRUE(dict.GetNameToId().empty()); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp index 6725cb8b..15cc549b 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp @@ -28,6 +28,8 @@ #include "paimon/common/compression/block_compression_factory.h" #include "paimon/common/compression/block_compressor.h" #include "paimon/common/compression/block_decompressor.h" +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/options/map_storage_layout.h" @@ -47,9 +49,9 @@ bool MapSharedShreddingUtils::IsShreddingKeyMap( return map_type->key_type()->id() == arrow::Type::STRING; } -Result> MapSharedShreddingUtils::DetectShreddingColumns( +Result> MapSharedShreddingUtils::DetectShreddingColumns( const std::shared_ptr& schema, const CoreOptions& options) { - std::vector indices; + std::vector field_names; for (int32_t i = 0; i < schema->num_fields(); ++i) { const auto& field = schema->field(i); if (!IsShreddingKeyMap(field->type())) { @@ -57,10 +59,22 @@ Result> MapSharedShreddingUtils::DetectShreddingColumns( } PAIMON_ASSIGN_OR_RAISE(MapStorageLayout layout, options.GetMapStorageLayout(field->name())); if (layout == MapStorageLayout::SHARED_SHREDDING) { - indices.push_back(i); + field_names.push_back(field->name()); } } - return indices; + return field_names; +} + +Result> MapSharedShreddingUtils::CreateShreddingContext( + const std::shared_ptr& schema, const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::vector shredding_field_names, + DetectShreddingColumns(schema, options)); + if (shredding_field_names.empty()) { + return std::shared_ptr(); + } + std::map field_to_k_max; + PAIMON_ASSIGN_OR_RAISE(field_to_k_max, BuildColumnToNumColumns(shredding_field_names, options)); + return std::make_shared(field_to_k_max); } // ---- Schema conversion ---- @@ -71,7 +85,7 @@ std::shared_ptr MapSharedShreddingUtils::BuildPhysicalStructTyp struct_fields.reserve(num_columns + 2); struct_fields.push_back( - arrow::field(MapSharedShreddingDefine::kFieldMapping, arrow::list(arrow::int32()), false)); + arrow::field(MapSharedShreddingDefine::kFieldMapping, arrow::list(arrow::int32()), true)); for (int32_t i = 0; i < num_columns; ++i) { struct_fields.push_back(arrow::field(MapSharedShreddingDefine::PhysicalColumnName(i), @@ -87,20 +101,20 @@ std::shared_ptr MapSharedShreddingUtils::BuildPhysicalStructTyp Result> MapSharedShreddingUtils::LogicalToPhysicalSchema( const std::shared_ptr& logical_schema, - const std::map& column_to_num_columns) { + const std::map& field_to_num_columns) { arrow::FieldVector physical_fields; physical_fields.reserve(logical_schema->num_fields()); for (int32_t i = 0; i < logical_schema->num_fields(); ++i) { const auto& field = logical_schema->field(i); - auto it = column_to_num_columns.find(i); - if (it != column_to_num_columns.end()) { + auto it = field_to_num_columns.find(field->name()); + if (it != field_to_num_columns.end()) { auto map_type = std::static_pointer_cast(field->type()); auto value_type = map_type->item_type(); bool value_nullable = map_type->item_field()->nullable(); auto physical_type = BuildPhysicalStructType(value_type, it->second, value_nullable); - physical_fields.push_back( - arrow::field(field->name(), physical_type, field->nullable())); + auto physical_field = arrow::field(field->name(), physical_type, field->nullable()); + physical_fields.push_back(physical_field); } else { physical_fields.push_back(field); } @@ -109,17 +123,15 @@ Result> MapSharedShreddingUtils::LogicalToPhysica return arrow::schema(std::move(physical_fields)); } -Result> MapSharedShreddingUtils::BuildColumnToNumColumns( - const std::vector& shredding_column_indices, - const std::shared_ptr& schema, const CoreOptions& options) { - std::map column_to_num_columns; - for (int32_t col_index : shredding_column_indices) { - const std::string& field_name = schema->field(col_index)->name(); +Result> MapSharedShreddingUtils::BuildColumnToNumColumns( + const std::vector& shredding_field_names, const CoreOptions& options) { + std::map field_to_num_columns; + for (const std::string& field_name : shredding_field_names) { PAIMON_ASSIGN_OR_RAISE(int32_t max_columns, options.GetMapSharedShreddingMaxColumns(field_name)); - column_to_num_columns[col_index] = max_columns; + field_to_num_columns[field_name] = max_columns; } - return column_to_num_columns; + return field_to_num_columns; } // ---- Metadata serialization helpers ---- @@ -129,7 +141,7 @@ namespace { std::string JsonEncodeObject( std::function builder) { rapidjson::Document doc(rapidjson::kObjectType); - auto allocator = doc.GetAllocator(); + auto& allocator = doc.GetAllocator(); builder(&doc, &allocator); rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -140,7 +152,7 @@ std::string JsonEncodeObject( std::string JsonEncodeArray( std::function builder) { rapidjson::Document doc(rapidjson::kArrayType); - auto allocator = doc.GetAllocator(); + auto& allocator = doc.GetAllocator(); builder(&doc, &allocator); rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -188,7 +200,7 @@ Result DecompressString(const std::string& input, int32_t original_ } Result GetRequiredValue(const std::shared_ptr& metadata, - const char* key) { + const std::string& key) { int32_t index = metadata->FindKey(key); if (index < 0) { return Status::Invalid(fmt::format("missing shredding metadata key: {}", key)); @@ -197,7 +209,7 @@ Result GetRequiredValue(const std::shared_ptr GetRequiredInt32(const std::shared_ptr& metadata, - const char* key) { + const std::string& key) { PAIMON_ASSIGN_OR_RAISE(std::string value, GetRequiredValue(metadata, key)); std::optional parsed = StringUtils::StringToValue(value); if (!parsed.has_value()) { @@ -391,4 +403,30 @@ bool MapSharedShreddingUtils::HasShreddingMetadata( return metadata->value(index) == MapShreddingDefine::kStorageLayoutSharedShredding; } +std::function>()> +MapSharedShreddingUtils::BuildMetadataFinalizer( + const std::shared_ptr& converter, + const std::string& compression, const std::shared_ptr& context, + const std::shared_ptr& physical_schema) { + return [converter, compression, context, + physical_schema]() -> Result> { + const std::vector& shredding_field_names = + converter->GetShreddingColumnNames(); + arrow::FieldVector updated_fields = physical_schema->fields(); + for (const std::string& field_name : shredding_field_names) { + int32_t col_index = physical_schema->GetFieldIndex(field_name); + const auto& field = physical_schema->field(col_index); + auto metadata = field->metadata() ? field->metadata()->Copy() + : std::make_shared(); + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingFieldMeta file_meta, + converter->BuildFieldMeta(field_name)); + PAIMON_RETURN_NOT_OK( + MapSharedShreddingUtils::SerializeMetadata(file_meta, compression, metadata.get())); + updated_fields[col_index] = field->WithMetadata(metadata); + context->ReportFileStats(field_name, file_meta.max_row_width); + } + return arrow::schema(std::move(updated_fields)); + }; +} + } // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.h b/src/paimon/common/data/shredding/map_shared_shredding_utils.h index 30e2a857..fd5d76d4 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -38,6 +39,8 @@ class Schema; namespace paimon { class CoreOptions; +class MapSharedShreddingBatchConverter; +class MapSharedShreddingContext; /// Utility functions for shared-shredding MAP storage layout. class MapSharedShreddingUtils { @@ -52,35 +55,40 @@ class MapSharedShreddingUtils { /// @return true if the type is MAP. static bool IsShreddingKeyMap(const std::shared_ptr& arrow_type); - /// Finds all shredding MAP column indices in a schema by checking per-column config + /// Finds all shredding MAP field names in a schema by checking per-column config /// via CoreOptions. /// @param schema The logical Arrow schema. /// @param options CoreOptions containing per-column configuration. - /// @return Vector of column indices whose map.storage-layout is "shared-shredding", or error + /// @return Vector of field names whose map.storage-layout is "shared-shredding", or error /// if validation fails. - static Result> DetectShreddingColumns( + static Result> DetectShreddingColumns( const std::shared_ptr& schema, const CoreOptions& options); + /// Creates a MapSharedShreddingContext for the given schema and options. + /// Returns nullptr if no shredding MAP columns are detected. + /// @param schema The logical Arrow schema. + /// @param options CoreOptions containing per-column configuration. + /// @return Shared context, or nullptr if no shredding columns. + static Result> CreateShreddingContext( + const std::shared_ptr& schema, const CoreOptions& options); // ---- Schema conversion ---- /// Converts a logical schema to a physical schema by replacing shredding MAP columns /// with their physical Struct representation. /// @param logical_schema The original schema with MAP columns. - /// @param column_to_num_columns Map from column index to its physical column count K. + /// @param field_to_num_columns Map from field name to its physical column count K. /// Each shredding column can have its own width. /// @return The physical schema for file writing. static Result> LogicalToPhysicalSchema( const std::shared_ptr& logical_schema, - const std::map& column_to_num_columns); + const std::map& field_to_num_columns); - /// Builds column_to_num_columns map from DetectShreddingColumns result and CoreOptions. - /// @param shredding_column_indices Indices returned by DetectShreddingColumns. - /// @param schema The logical Arrow schema (used to get field names). + /// Builds field_to_num_columns map from DetectShreddingColumns result and CoreOptions. + /// @param shredding_field_names Field names returned by DetectShreddingColumns. /// @param options CoreOptions containing per-column max-columns config. - /// @return Map from column index to K (max physical columns for that column). - static Result> BuildColumnToNumColumns( - const std::vector& shredding_column_indices, - const std::shared_ptr& schema, const CoreOptions& options); + /// @return Map from field name to K (max physical columns for that field). + static Result> BuildColumnToNumColumns( + const std::vector& shredding_field_names, const CoreOptions& options); // ---- Metadata serialization ---- @@ -102,6 +110,22 @@ class MapSharedShreddingUtils { /// Checks whether a KeyValueMetadata contains shredding MAP metadata. static bool HasShreddingMetadata(const std::shared_ptr& metadata); + // ---- Writer helpers ---- + + /// Builds a MetadataFinalizer that serializes shredding metadata into per-field + /// KeyValueMetadata and reports file stats back to context for K adaptation. + /// Shared by DataFileWriter (append-only) and KeyValueDataFileWriter (PK table). + /// @param converter The batch converter that holds field-dict state for BuildFieldMeta. + /// @param compression Compression codec name for field_dict serialization (e.g. "zstd"). + /// @param context The cross-file shared context for K adaptation. + /// @param physical_schema The physical schema used for writing. + /// @return A callable that produces the updated schema with shredding metadata + /// and reports file stats to context. + static std::function>()> BuildMetadataFinalizer( + const std::shared_ptr& converter, + const std::string& compression, const std::shared_ptr& context, + const std::shared_ptr& physical_schema); + private: /// Builds the physical Arrow type for one shredding MAP column. /// @param value_type The value type of the original MAP. diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp index 87a0f089..9eb65879 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp @@ -60,11 +60,11 @@ TEST(MapSharedShreddingUtilsTest, DetectShreddingColumnsBasic) { CoreOptions::FromMap({{"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.metrics.map.storage-layout", "shared-shredding"}})); - ASSERT_OK_AND_ASSIGN(auto indices, + ASSERT_OK_AND_ASSIGN(auto field_names, MapSharedShreddingUtils::DetectShreddingColumns(schema, options)); - ASSERT_EQ(indices.size(), 2); - ASSERT_EQ(indices[0], 1); - ASSERT_EQ(indices[1], 2); + ASSERT_EQ(field_names.size(), 2); + ASSERT_EQ(field_names[0], "tags"); + ASSERT_EQ(field_names[1], "metrics"); } TEST(MapSharedShreddingUtilsTest, DetectShreddingColumnsNoShredding) { @@ -74,9 +74,9 @@ TEST(MapSharedShreddingUtilsTest, DetectShreddingColumnsNoShredding) { }); ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); - ASSERT_OK_AND_ASSIGN(auto indices, + ASSERT_OK_AND_ASSIGN(auto field_names, MapSharedShreddingUtils::DetectShreddingColumns(schema, options)); - ASSERT_TRUE(indices.empty()); + ASSERT_TRUE(field_names.empty()); } // ---- LogicalToPhysicalSchema ---- @@ -88,13 +88,13 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaBasic) { arrow::field("name", arrow::utf8()), }); - std::map column_to_num_columns = {{1, 4}}; + std::map field_to_num_columns = {{"tags", 4}}; ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - schema, column_to_num_columns)); + schema, field_to_num_columns)); // Build expected schema for comparison auto expected_struct = arrow::struct_({ - arrow::field("__field_mapping", arrow::list(arrow::int32()), false), + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), arrow::field("__col_0", arrow::utf8(), true), arrow::field("__col_1", arrow::utf8(), true), arrow::field("__col_2", arrow::utf8(), true), @@ -116,12 +116,12 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNestedValue) { auto map_type = arrow::map(arrow::utf8(), nested_value); auto schema = arrow::schema({arrow::field("data", map_type)}); - std::map column_to_num_columns = {{0, 2}}; + std::map field_to_num_columns = {{"data", 2}}; ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - schema, column_to_num_columns)); + schema, field_to_num_columns)); auto expected_struct = arrow::struct_({ - arrow::field("__field_mapping", arrow::list(arrow::int32()), false), + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), arrow::field("__col_0", nested_value, true), arrow::field("__col_1", nested_value, true), arrow::field("__overflow", arrow::map(arrow::int32(), nested_value), true), @@ -134,7 +134,7 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNullable) { // MAP value is nullable auto nullable_map = arrow::map(arrow::utf8(), arrow::field("item", arrow::int64(), true)); auto schema_nullable = arrow::schema({arrow::field("m", nullable_map)}); - std::map col_map = {{0, 2}}; + std::map col_map = {{"m", 2}}; ASSERT_OK_AND_ASSIGN( auto physical, MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_nullable, col_map)); @@ -159,7 +159,7 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNoShreddingColumns) { arrow::field("name", arrow::utf8()), }); - std::map empty_map; + std::map empty_map; ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, empty_map)); ASSERT_TRUE(physical_schema->Equals(schema)); @@ -179,13 +179,13 @@ TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumns) { CoreOptions::FromMap({{"fields.tags.map.shared-shredding.max-columns", "128"}, {"fields.metrics.map.shared-shredding.max-columns", "64"}})); - std::vector shredding_indices = {1, 2}; + std::vector shredding_field_names = {"tags", "metrics"}; ASSERT_OK_AND_ASSIGN(auto result, MapSharedShreddingUtils::BuildColumnToNumColumns( - shredding_indices, schema, options)); + shredding_field_names, options)); ASSERT_EQ(result.size(), 2); - ASSERT_EQ(result[1], 128); - ASSERT_EQ(result[2], 64); + ASSERT_EQ(result.at("tags"), 128); + ASSERT_EQ(result.at("metrics"), 64); } TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumnsDefault) { @@ -195,10 +195,10 @@ TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumnsDefault) { // No explicit max-columns config -> default 256 ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); - std::vector shredding_indices = {0}; + std::vector shredding_field_names = {"tags"}; ASSERT_OK_AND_ASSIGN(auto result, MapSharedShreddingUtils::BuildColumnToNumColumns( - shredding_indices, schema, options)); - ASSERT_EQ(result[0], 256); + shredding_field_names, options)); + ASSERT_EQ(result.at("tags"), 256); } // ---- SerializeMetadata / DeserializeMetadata roundtrip ---- @@ -214,7 +214,7 @@ TEST(MapSharedShreddingUtilsTest, MetadataRoundtripNoneCompression) { auto metadata = std::make_shared(); ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata(original, "none", metadata.get())); - // Verify raw KV strings to get intuition of what's stored + // Verify raw KV strings auto find_value = [&](const char* key) -> std::string { int32_t idx = metadata->FindKey(key); EXPECT_GE(idx, 0); diff --git a/src/paimon/common/data/shredding/map_shredding_defs.h b/src/paimon/common/data/shredding/map_shredding_defs.h index 3cb028e0..ecc993d1 100644 --- a/src/paimon/common/data/shredding/map_shredding_defs.h +++ b/src/paimon/common/data/shredding/map_shredding_defs.h @@ -65,6 +65,9 @@ struct MapSharedShreddingDefine { /// Overflow column name. static constexpr const char* kOverflow = "__overflow"; + /// Default compression codec for field_dict serialization. + static constexpr const char* kDefaultDictCompression = "zstd"; + /// Returns the name of the i-th physical column: "__col_0", "__col_1", etc. static std::string PhysicalColumnName(int32_t index) { return "__col_" + std::to_string(index); diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp index 88623b3f..e9f4f1a9 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils.cpp @@ -26,6 +26,9 @@ #include "paimon/common/utils/string_utils.h" namespace paimon { + +const char* ArrowUtils::kArrowSchemaMetadataKey = "ARROW:schema"; + Result> ArrowUtils::DataTypeToSchema( const std::shared_ptr& data_type) { if (data_type->id() != arrow::Type::STRUCT) { diff --git a/src/paimon/common/utils/arrow/arrow_utils.h b/src/paimon/common/utils/arrow/arrow_utils.h index f6dea3d1..52c521d2 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.h +++ b/src/paimon/common/utils/arrow/arrow_utils.h @@ -33,6 +33,8 @@ class PAIMON_EXPORT ArrowUtils { ArrowUtils() = delete; ~ArrowUtils() = delete; + static const char* kArrowSchemaMetadataKey; + static Result> DataTypeToSchema( const std::shared_ptr& data_type); diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 599c3078..c5d81093 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -19,6 +19,7 @@ #include "paimon/core/append/append_only_writer.h" #include +#include #include #include @@ -26,6 +27,11 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/types/row_kind.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -55,13 +61,29 @@ namespace paimon { class MemoryPool; class FormatStatsExtractor; -AppendOnlyWriter::AppendOnlyWriter(const CoreOptions& options, int64_t schema_id, - const std::shared_ptr& write_schema, - const std::optional>& write_cols, - int64_t max_sequence_number, - const std::shared_ptr& path_factory, - const std::shared_ptr& compact_manager, - const std::shared_ptr& memory_pool) +Result> AppendOnlyWriter::Create( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, int64_t max_sequence_number, + const std::shared_ptr& path_factory, + const std::shared_ptr& compact_manager, + const std::shared_ptr& memory_pool) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); + + return std::unique_ptr( + new AppendOnlyWriter(options, schema_id, write_schema, write_cols, max_sequence_number, + path_factory, compact_manager, shredding_context, memory_pool)); +} + +AppendOnlyWriter::AppendOnlyWriter( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, int64_t max_sequence_number, + const std::shared_ptr& path_factory, + const std::shared_ptr& compact_manager, + const std::shared_ptr& shredding_context, + const std::shared_ptr& memory_pool) : options_(options), schema_id_(schema_id), write_schema_(write_schema), @@ -70,7 +92,8 @@ AppendOnlyWriter::AppendOnlyWriter(const CoreOptions& options, int64_t schema_id path_factory_(path_factory), compact_manager_(compact_manager), memory_pool_(memory_pool), - metrics_(std::make_shared()) {} + metrics_(std::make_shared()), + shredding_context_(shredding_context) {} AppendOnlyWriter::~AppendOnlyWriter() = default; @@ -116,6 +139,7 @@ Status AppendOnlyWriter::Write(std::unique_ptr&& batch) { PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*struct_array, &c_array)); return writer_->Write(&c_array); } + return writer_->Write(batch->GetData()); } @@ -216,7 +240,9 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWrit auto schemas = BlobUtils::SeparateBlobSchema(write_schema_, blob_context->GetInlineFields()); return CreateRollingBlobWriter(schemas, blob_context->GetInlineFields()); - } else if (!blob_context) { + } + + if (!blob_context) { // No BLOB fields at all -> plain rolling writer return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/false), @@ -237,24 +263,51 @@ AppendOnlyWriter::SingleFileWriterCreator AppendOnlyWriter::GetDataFileWriterCre [this, schema, write_cols]() -> Result< std::unique_ptr>>> { + // Determine the schema to use for file writing. + // When shared-shredding map is active, compute per-file K and build a physical schema. + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, + MapSharedShreddingBatchConverter::CreateConverter( + schema, shredding_context_, memory_pool_)); + std::shared_ptr file_schema = + bundle.physical_schema ? bundle.physical_schema : schema; + ::ArrowSchema arrow_schema; ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); auto format = options_.GetFileFormat(); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr writer_builder, format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); writer_builder->WithMemoryPool(memory_pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, format->CreateStatsExtractor(&arrow_schema)); + // Build the converter that transforms logical batches to physical batches. + // When shredding is active, it performs MAP→STRUCT conversion first. + std::function batch_converter; + if (bundle.converter) { + auto converter = bundle.converter; + batch_converter = [converter](ArrowArray* input, ArrowArray* output) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, + converter->Convert(input)); + ArrowArrayMove(physical.get(), output); + return Status::OK(); + }; + } + auto writer = std::make_unique( - options_.GetFileCompression(), std::function(), - schema_id_, seq_num_counter_, FileSource::Append(), stats_extractor, - path_factory_->IsExternalPath(), write_cols, memory_pool_); + options_.GetFileCompression(), batch_converter, schema_id_, seq_num_counter_, + FileSource::Append(), stats_extractor, path_factory_->IsExternalPath(), write_cols, + memory_pool_); PAIMON_RETURN_NOT_OK( writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder)); + + if (bundle.converter) { + writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( + bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, + shredding_context_, file_schema)); + } return writer; }; } diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index 69b8c845..1a78d16f 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -46,6 +47,7 @@ namespace paimon { class CommitIncrement; class ExternalStorageBlobWriter; +class MapSharedShreddingContext; class RecordBatch; template class RollingFileWriter; @@ -58,13 +60,13 @@ class WriterBuilder; class AppendOnlyWriter : public BatchWriter { public: - AppendOnlyWriter(const CoreOptions& options, int64_t schema_id, - const std::shared_ptr& write_schema, - const std::optional>& write_cols, - int64_t max_sequence_number, - const std::shared_ptr& path_factory, - const std::shared_ptr& compact_manager, - const std::shared_ptr& memory_pool); + static Result> Create( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, int64_t max_sequence_number, + const std::shared_ptr& path_factory, + const std::shared_ptr& compact_manager, + const std::shared_ptr& memory_pool); ~AppendOnlyWriter() override; @@ -98,6 +100,15 @@ class AppendOnlyWriter : public BatchWriter { using RollingFileWriterResult = Result>>>; + AppendOnlyWriter(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, + int64_t max_sequence_number, + const std::shared_ptr& path_factory, + const std::shared_ptr& compact_manager, + const std::shared_ptr& shredding_context, + const std::shared_ptr& memory_pool); + RollingFileWriterResult CreateRollingRowWriter(); RollingFileWriterResult CreateRollingBlobWriter( const BlobUtils::SeparatedSchemas& schemas, @@ -138,6 +149,11 @@ class AppendOnlyWriter : public BatchWriter { std::unique_ptr external_storage_writer_; std::set inline_descriptor_fields_; std::set inline_view_fields_; + + // ---- Shared-shredding MAP support ---- + /// Cross-file context for K adaptation and shredding column tracking. + /// nullptr when no shared-shredding MAP columns are configured. + std::shared_ptr shredding_context_; }; } // namespace paimon diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp index 32743f8b..f6c87dea 100644 --- a/src/paimon/core/append/append_only_writer_test.cpp +++ b/src/paimon/core/append/append_only_writer_test.cpp @@ -33,12 +33,17 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/ipc/json_simple.h" #include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" #include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/fs/external_path_provider.h" +#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/core/compact/compact_deletion_file.h" #include "paimon/core/compact/compact_result.h" #include "paimon/core/compact/noop_compact_manager.h" @@ -50,10 +55,13 @@ #include "paimon/core/stats/simple_stats.h" #include "paimon/core/utils/commit_increment.h" #include "paimon/defs.h" +#include "paimon/format/file_format_factory.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/record_batch.h" +#include "paimon/testing/utils/binary_row_generator.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" namespace arrow { @@ -239,7 +247,71 @@ class AppendOnlyWriterTest : public testing::Test { return batch_builder.Finish().value(); } - private: + /// Creates a RecordBatch from a JSON string matching the given schema. + std::unique_ptr CreateBatch(const std::shared_ptr& schema, + const std::string& json) const { + auto struct_type = arrow::struct_(schema->fields()); + auto array = arrow::ipc::internal::json::ArrayFromJSON(struct_type, json).ValueOrDie(); + ::ArrowArray arrow_array; + EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + return batch_builder.Finish().value(); + } + + /// Opens a file using the specified format and returns a reader. + std::unique_ptr OpenFormatReader(const std::string& file_path, + const std::string& format) const { + auto fs = std::make_shared(); + EXPECT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(file_path)); + EXPECT_TRUE(input_stream); + EXPECT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get(format, /*options=*/{})); + EXPECT_OK_AND_ASSIGN(auto reader_builder, + file_format->CreateReaderBuilder(/*batch_size=*/10)); + return reader_builder->Build(input_stream).value(); + } + + /// Reads a file's content and compares it to expected_array. + void CheckFileContent(const std::string& file_path, const std::string& format, + const std::shared_ptr& expected_array) const { + auto reader = OpenFormatReader(file_path, format); + auto c_file_schema = reader->GetFileSchema().value(); + ASSERT_OK(reader->SetReadSchema(c_file_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); + ASSERT_TRUE(expected_array->Equals(result_array)) + << "Expected:\n" + << expected_array->ToString() << "\nActual:\n" + << result_array->ToString(); + } + + /// Reads a file's schema, compares structure against expected physical schema + /// (ignoring metadata), then verifies shared-shredding map metadata on the given field. + void CheckShreddingFileSchema(const std::string& file_path, const std::string& format, + const std::shared_ptr& expected_physical_schema, + int32_t field_index, + const MapSharedShreddingFieldMeta& expected_meta, + const std::string& compression) const { + auto reader = OpenFormatReader(file_path, format); + auto c_file_schema = reader->GetFileSchema().value(); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + + // Compare schema structure (types + field names), ignoring metadata. + ASSERT_TRUE(file_schema->Equals(*expected_physical_schema, /*check_metadata=*/false)) + << "Expected schema:\n" + << expected_physical_schema->ToString() << "\nActual schema:\n" + << file_schema->ToString(); + + // Deserialize and compare the per-field shared-shredding map metadata. + auto metadata = file_schema->field(field_index)->metadata(); + ASSERT_NE(nullptr, metadata); + ASSERT_OK_AND_ASSIGN( + auto deserialized_meta, + MapSharedShreddingUtils::DeserializeMetadata( + metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression)); + ASSERT_EQ(expected_meta, deserialized_meta); + } + + protected: std::shared_ptr memory_pool_; std::shared_ptr compact_manager_; }; @@ -263,11 +335,12 @@ TEST_F(AppendOnlyWriterTest, TestEmptyCommits) { ASSERT_TRUE(dir); ASSERT_OK(path_factory->Init(dir->Str(), "mock_format", options.DataFilePrefix(), nullptr)); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); for (int32_t i = 0; i < 3; i++) { - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(true)); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(true)); ASSERT_TRUE(inc.GetNewFilesIncrement().IsEmpty()); ASSERT_TRUE(inc.GetCompactIncrement().IsEmpty()); } @@ -292,9 +365,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), "mock_format", options.DataFilePrefix(), nullptr)); - AppendOnlyWriter writer(options, /*schema_id=*/2, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/2, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); arrow::StringBuilder builder; for (size_t j = 0; j < 100; j++) { ASSERT_TRUE(builder.Append(std::to_string(j)).ok()); @@ -304,9 +378,9 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) { ASSERT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); RecordBatchBuilder batch_builder(&arrow_array); ASSERT_OK_AND_ASSIGN(auto record_batch, batch_builder.Finish()); - ASSERT_OK(writer.Write(std::move(record_batch))); + ASSERT_OK(writer->Write(std::move(record_batch))); ASSERT_TRUE(ArrowArrayIsReleased(&arrow_array)); - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(true)); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(true)); ASSERT_FALSE(inc.GetNewFilesIncrement().IsEmpty()); const auto& data_increment = inc.GetNewFilesIncrement(); const auto& data_file_metas = data_increment.NewFiles(); @@ -316,7 +390,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) { std::string path = path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); ASSERT_OK_AND_ASSIGN(bool exist, options.GetFileSystem()->Exists(path)); ASSERT_TRUE(exist); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestWriteAndClose) { @@ -334,9 +408,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndClose) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); - AppendOnlyWriter writer(options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); auto struct_type = arrow::struct_(fields); arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), {std::make_shared()}); @@ -353,9 +428,9 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndClose) { RecordBatchBuilder batch_builder(&arrow_array); ASSERT_OK_AND_ASSIGN(auto record_batch, batch_builder.Finish()); - ASSERT_OK(writer.Write(std::move(record_batch))); + ASSERT_OK(writer->Write(std::move(record_batch))); ASSERT_TRUE(ArrowArrayIsReleased(&arrow_array)); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); auto file_system = std::make_shared(); std::vector> file_status_list; @@ -378,9 +453,10 @@ TEST_F(AppendOnlyWriterTest, TestInvalidRowKind) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); - AppendOnlyWriter writer(options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); auto struct_type = arrow::struct_(fields); arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), {std::make_shared()}); @@ -396,10 +472,10 @@ TEST_F(AppendOnlyWriterTest, TestInvalidRowKind) { RecordBatchBuilder batch_builder(&arrow_array); ASSERT_OK_AND_ASSIGN(auto record_batch, batch_builder.SetRowKinds({RecordBatch::RowKind::DELETE}).Finish()); - ASSERT_NOK_WITH_MSG(writer.Write(std::move(record_batch)), + ASSERT_NOK_WITH_MSG(writer->Write(std::move(record_batch)), "Append only writer can not accept record batch with RowKind DELETE"); ASSERT_TRUE(ArrowArrayIsReleased(&arrow_array)); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); auto file_system = std::make_shared(); std::vector> file_status_list; @@ -416,17 +492,18 @@ TEST_F(AppendOnlyWriterTest, TestPrepareCommitWaitCompactionUsesBlockingGetResul arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Write(CreateSingleStringBatch({"a", "b"}))); - ASSERT_OK(writer.PrepareCommit(/*wait_compaction=*/true).status()); + ASSERT_OK(writer->Write(CreateSingleStringBatch({"a", "b"}))); + ASSERT_OK(writer->PrepareCommit(/*wait_compaction=*/true).status()); ASSERT_EQ(compact_manager->get_result_blocking_calls.size(), 2); ASSERT_FALSE(compact_manager->get_result_blocking_calls[0]); ASSERT_TRUE(compact_manager->get_result_blocking_calls[1]); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestPrepareCommitForceCompactUsesBlockingGetResult) { @@ -438,17 +515,18 @@ TEST_F(AppendOnlyWriterTest, TestPrepareCommitForceCompactUsesBlockingGetResult) arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Write(CreateSingleStringBatch({"a"}))); - ASSERT_OK(writer.PrepareCommit(/*wait_compaction=*/false).status()); + ASSERT_OK(writer->Write(CreateSingleStringBatch({"a"}))); + ASSERT_OK(writer->PrepareCommit(/*wait_compaction=*/false).status()); ASSERT_EQ(compact_manager->get_result_blocking_calls.size(), 2); ASSERT_FALSE(compact_manager->get_result_blocking_calls[0]); ASSERT_TRUE(compact_manager->get_result_blocking_calls[1]); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, @@ -481,13 +559,14 @@ TEST_F(AppendOnlyWriterTest, arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Sync()); - ASSERT_OK(writer.Sync()); - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(writer->Sync()); + ASSERT_OK(writer->Sync()); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/false)); ASSERT_EQ(inc.GetCompactIncrement().CompactBefore().size(), 2); ASSERT_EQ(inc.GetCompactIncrement().CompactAfter().size(), 2); @@ -500,7 +579,7 @@ TEST_F(AppendOnlyWriterTest, ASSERT_TRUE(merged); ASSERT_EQ(merged->Id(), "d2"); ASSERT_EQ(merged->MergedOld(), deletion_file1); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestCloseDeletesCompactAfterFiles) { @@ -524,13 +603,14 @@ TEST_F(AppendOnlyWriterTest, TestCloseDeletesCompactAfterFiles) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Sync()); + ASSERT_OK(writer->Sync()); ASSERT_TRUE(options.GetFileSystem()->Exists(compact_after_path).value()); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); ASSERT_FALSE(options.GetFileSystem()->Exists(compact_after_path).value()); ASSERT_TRUE(compact_manager->request_cancel_called); ASSERT_TRUE(compact_manager->wait_called); @@ -556,15 +636,16 @@ TEST_F(AppendOnlyWriterTest, TestCloseCleansDeletionFile) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); // Sync to consume the compaction result and populate compact_deletion_file_. - ASSERT_OK(writer.Sync()); + ASSERT_OK(writer->Sync()); ASSERT_FALSE(deletion_file->Cleaned()); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); ASSERT_TRUE(deletion_file->Cleaned()); } @@ -578,14 +659,15 @@ TEST_F(AppendOnlyWriterTest, TestCompactNotCompletedTriggersCompaction) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK_AND_ASSIGN(bool not_completed, writer.CompactNotCompleted()); + ASSERT_OK_AND_ASSIGN(bool not_completed, writer->CompactNotCompleted()); ASSERT_TRUE(not_completed); ASSERT_EQ(compact_manager->trigger_calls, std::vector({false})); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestCompactPassesFullCompactionFlag) { @@ -597,14 +679,15 @@ TEST_F(AppendOnlyWriterTest, TestCompactPassesFullCompactionFlag) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); - ASSERT_OK(writer.Compact(/*full_compaction=*/true)); - ASSERT_OK(writer.Compact(/*full_compaction=*/false)); + ASSERT_OK(writer->Compact(/*full_compaction=*/true)); + ASSERT_OK(writer->Compact(/*full_compaction=*/false)); ASSERT_EQ(compact_manager->trigger_calls, std::vector({true, false})); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { @@ -618,9 +701,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { auto blob_field = BlobUtils::ToArrowField("blob", false); auto schema = arrow::schema({int_field, blob_field}); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); arrow::Int32Builder int_builder; ASSERT_TRUE(int_builder.AppendValues({1, 2}).ok()); @@ -630,8 +714,8 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { ASSERT_TRUE(blob_builder.Append("bb", 2).ok()); auto blob_array = blob_builder.Finish().ValueOrDie(); - ASSERT_OK(writer.Write(CreateStructBatch(schema, {int_array, blob_array}))); - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Write(CreateStructBatch(schema, {int_array, blob_array}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); ASSERT_EQ(inc.GetNewFilesIncrement().NewFiles().size(), 2); const auto& main_file = inc.GetNewFilesIncrement().NewFiles()[0]; @@ -640,7 +724,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { options.GetFileSystem()->Exists(path_factory->ToPath(main_file->file_name)).value()); ASSERT_TRUE( options.GetFileSystem()->Exists(path_factory->ToPath(blob_file->file_name)).value()); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { @@ -653,9 +737,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { auto schema = arrow::schema({arrow::field("id", arrow::int32()), BlobUtils::ToArrowField("blob1", false), BlobUtils::ToArrowField("blob2", false)}); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); arrow::Int32Builder int_builder; ASSERT_TRUE(int_builder.AppendValues({1}).ok()); @@ -667,8 +752,8 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { ASSERT_TRUE(blob_builder2.Append("b", 1).ok()); auto blob_array2 = blob_builder2.Finish().ValueOrDie(); - ASSERT_OK(writer.Write(CreateStructBatch(schema, {int_array, blob_array1, blob_array2}))); - ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer.PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Write(CreateStructBatch(schema, {int_array, blob_array1, blob_array2}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); ASSERT_EQ(inc.GetNewFilesIncrement().NewFiles().size(), 3); const auto& main_file = inc.GetNewFilesIncrement().NewFiles()[0]; @@ -680,7 +765,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { options.GetFileSystem()->Exists(path_factory->ToPath(blob_file1->file_name)).value()); ASSERT_TRUE( options.GetFileSystem()->Exists(path_factory->ToPath(blob_file2->file_name)).value()); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) { @@ -691,14 +776,15 @@ TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); - ASSERT_OK(writer.Write(CreateSingleStringBatch({"a", "b", "c"}))); - ASSERT_OK_AND_ASSIGN(CommitIncrement first, writer.PrepareCommit(/*wait_compaction=*/false)); - ASSERT_OK(writer.Write(CreateSingleStringBatch({"d", "e"}))); - ASSERT_OK_AND_ASSIGN(CommitIncrement second, writer.PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(writer->Write(CreateSingleStringBatch({"a", "b", "c"}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement first, writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(writer->Write(CreateSingleStringBatch({"d", "e"}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement second, writer->PrepareCommit(/*wait_compaction=*/false)); ASSERT_EQ(first.GetNewFilesIncrement().NewFiles().size(), 1); ASSERT_EQ(second.GetNewFilesIncrement().NewFiles().size(), 1); @@ -706,7 +792,7 @@ TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) { ASSERT_EQ(first.GetNewFilesIncrement().NewFiles()[0]->max_sequence_number, 2); ASSERT_EQ(second.GetNewFilesIncrement().NewFiles()[0]->min_sequence_number, 3); ASSERT_EQ(second.GetNewFilesIncrement().NewFiles()[0]->max_sequence_number, 4); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) { @@ -719,9 +805,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) { auto schema = arrow::schema({arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); // Build f0 column arrow::Int32Builder int_builder; @@ -739,10 +826,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) { ASSERT_TRUE(view_builder.Append(view_bytes_1->data(), view_bytes_1->size()).ok()); auto view_array = view_builder.Finish().ValueOrDie(); - ASSERT_OK(writer.Write(CreateStructBatch(schema, {int_array, view_array}))); - ASSERT_OK_AND_ASSIGN(auto inc, writer.PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Write(CreateStructBatch(schema, {int_array, view_array}))); + ASSERT_OK_AND_ASSIGN(auto inc, writer->PrepareCommit(/*wait_compaction=*/true)); ASSERT_FALSE(inc.GetNewFilesIncrement().NewFiles().empty()); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); } TEST_F(AppendOnlyWriterTest, TestWriteInvalidBlobViewFieldRejected) { @@ -755,9 +842,10 @@ TEST_F(AppendOnlyWriterTest, TestWriteInvalidBlobViewFieldRejected) { auto schema = arrow::schema({arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}); - AppendOnlyWriter writer(options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager_, - memory_pool_); + ASSERT_OK_AND_ASSIGN( + auto writer, AppendOnlyWriter::Create( + options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); // Build f0 column arrow::Int32Builder int_builder; @@ -769,9 +857,783 @@ TEST_F(AppendOnlyWriterTest, TestWriteInvalidBlobViewFieldRejected) { ASSERT_TRUE(view_builder.Append("not_a_valid_blob_view_or_descriptor").ok()); auto view_array = view_builder.Finish().ValueOrDie(); - ASSERT_NOK_WITH_MSG(writer.Write(CreateStructBatch(schema, {int_array, view_array})), + ASSERT_NOK_WITH_MSG(writer->Write(CreateStructBatch(schema, {int_array, view_array})), "BLOB inline field view require values to be set as corresponding type."); - ASSERT_OK(writer.Close()); + ASSERT_OK(writer->Close()); +} + +/// Parameterized test class for shared-shredding tests, parameterized by file format. +class AppendOnlyWriterShreddingTest : public AppendOnlyWriterTest, + public ::testing::WithParamInterface { + public: + std::string GetFormat() const { + return GetParam(); + } +}; + +INSTANTIATE_TEST_SUITE_P(FileFormats, AppendOnlyWriterShreddingTest, + ::testing::Values("parquet", "orc")); + +TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) { + std::string format = GetFormat(); + // Configure with shared-shredding map on "tags" field, K=3. + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {Options::WRITE_ONLY, "true"}, + }); + + // Logical schema: id(INT32), tags(MAP) + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + // Write a batch with MAP data using the logical schema. + // Row0: id=1, tags={a:10, b:20} → fits K=3 + // Row1: id=2, tags={c:30, a:40, b:50} → fits K=3 + // Row2: id=3, tags={a:60} → fits K=3 + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30], ["a", 40], ["b", 50]]], + [3, [["a", 60]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + // Verify we got one data file. + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + // Check shared-shredding map metadata: a=0, b=1, c=2; K=3, max_row_width=3, no overflow. + std::map column_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0, 1}}, {1, {1, 2}}, {2, {0}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 3; + std::string compression = options.GetFileCompression(); + CheckShreddingFileSchema(data_file_path, format, expected_physical_schema, /*field_index=*/1, + expected_meta, compression); + + auto physical_type = arrow::struct_(expected_physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1, -1], 10, 20, null, null]], + [2, [[2, 0, 1], 30, 40, 50, null]], + [3, [[0, -1, -1], 60, null, null, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(data_file_path, format, expected_array); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllEmptyFirstFile) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + auto batch = CreateBatch(logical_schema, R"([ + [1, []], + [2, []] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map first_file_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto first_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, first_file_k)); + MapSharedShreddingFieldMeta empty_meta; + empty_meta.num_columns = 3; + empty_meta.max_row_width = 0; + CheckShreddingFileSchema(data_file_path, format, first_schema, /*field_index=*/1, empty_meta, + options.GetFileCompression()); + + auto physical_type = arrow::struct_(first_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[-1, -1, -1], null, null, null, null]], + [2, [[-1, -1, -1], null, null, null, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(data_file_path, format, expected_array); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyFiles) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + auto null_batch = CreateBatch(logical_schema, R"([ + [1, null], + [2, null] + ])"); + ASSERT_OK(writer->Write(std::move(null_batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement null_inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, null_inc.GetNewFilesIncrement().NewFiles().size()); + std::string null_file_path = + path_factory->ToPath(null_inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map first_file_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto first_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, first_file_k)); + MapSharedShreddingFieldMeta empty_meta; + empty_meta.num_columns = 3; + empty_meta.max_row_width = 0; + CheckShreddingFileSchema(null_file_path, format, first_schema, /*field_index=*/1, empty_meta, + options.GetFileCompression()); + + auto first_physical_type = arrow::struct_(first_schema->fields()); + std::shared_ptr expected_null_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(first_physical_type, {R"([ + [1, null], + [2, null] + ])"}, + &expected_null_array) + .ok()); + CheckFileContent(null_file_path, format, expected_null_array); + + auto empty_batch = CreateBatch(logical_schema, R"([ + [3, []], + [4, []] + ])"); + ASSERT_OK(writer->Write(std::move(empty_batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement empty_inc, + writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, empty_inc.GetNewFilesIncrement().NewFiles().size()); + std::string empty_file_path = + path_factory->ToPath(empty_inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + // Previous file observed max_row_width=0, but the next file must still keep at least one + // physical value column so shared-shredding never produces a K=0 schema. + std::map second_file_k = {{"tags", 1}}; + ASSERT_OK_AND_ASSIGN(auto second_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, second_file_k)); + empty_meta.num_columns = 1; + CheckShreddingFileSchema(empty_file_path, format, second_schema, /*field_index=*/1, empty_meta, + options.GetFileCompression()); + + auto second_physical_type = arrow::struct_(second_schema->fields()); + std::shared_ptr expected_empty_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(second_physical_type, {R"([ + [3, [[-1], null, null]], + [4, [[-1], null, null]] + ])"}, + &expected_empty_array) + .ok()); + CheckFileContent(empty_file_path, format, expected_empty_array); + + auto null_value_batch = CreateBatch(logical_schema, R"([ + [5, [["a", null]]], + [6, [["b", null]]], + [7, [["c", 7], ["d", null]]] + ])"); + ASSERT_OK(writer->Write(std::move(null_value_batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement null_value_inc, + writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, null_value_inc.GetNewFilesIncrement().NewFiles().size()); + std::string null_value_file_path = + path_factory->ToPath(null_value_inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + MapSharedShreddingFieldMeta null_value_meta; + null_value_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}}; + null_value_meta.field_to_columns = {{0, {0}}, {1, {0}}, {2, {0}}}; + null_value_meta.overflow_field_set = {3}; + null_value_meta.num_columns = 1; + null_value_meta.max_row_width = 2; + CheckShreddingFileSchema(null_value_file_path, format, second_schema, /*field_index=*/1, + null_value_meta, options.GetFileCompression()); + + std::shared_ptr expected_null_value_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(second_physical_type, {R"([ + [5, [[0], null, null]], + [6, [[1], null, null]], + [7, [[2], 7, [[3, null]]]] + ])"}, + &expected_null_value_array) + .ok()); + CheckFileContent(null_value_file_path, format, expected_null_value_array); + + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithOverflow) { + std::string format = GetFormat(); + // K=2, write rows with 3+ keys to trigger overflow. + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + // Row0: {a:1, b:2} → fits K=2 + // Row1: {c:3, a:4, b:5} → 3 keys, K=2: c→col0, a→col1, b→overflow + // Row2: {d:6, e:7, f:8, a:9} → 4 keys, K=2: d→col0, e→col1, f+a→overflow + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 1], ["b", 2]]], + [2, [["c", 3], ["a", 4], ["b", 5]]], + [3, [["d", 6], ["e", 7], ["f", 8], ["a", 9]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map column_to_k = {{"tags", 2}}; + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); + std::string compression = options.GetFileCompression(); + + // Verify metadata: a=0,b=1,c=2,d=3,e=4,f=5; K=2, max_row_width=4 + // b overflows in row1, f and a overflow in row2 + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}, {"e", 4}, {"f", 5}}; + expected_meta.field_to_columns = {{0, {0, 1}}, {1, {1}}, {2, {0}}, {3, {0}}, {4, {1}}}; + expected_meta.overflow_field_set = {0, 1, 5}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 4; + CheckShreddingFileSchema(data_file_path, format, expected_physical_schema, /*field_index=*/1, + expected_meta, compression); + + // Verify data content. + auto physical_type = arrow::struct_(expected_physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1], 1, 2, null]], + [2, [[2, 0], 3, 4, [[1, 5]]]], + [3, [[3, 4], 6, 7, [[5, 8], [0, 9]]]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(data_file_path, format, expected_array); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFiles) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "10"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + // --- File 1: max_row_width = 3, K = K_max = 10 (first file, no history) --- + auto batch1 = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30], ["a", 40], ["b", 50]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch1))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc1, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc1.GetNewFilesIncrement().NewFiles().size()); + + std::string file1_path = + path_factory->ToPath(inc1.GetNewFilesIncrement().NewFiles()[0]->file_name); + + // File 1 should have K=10 (first file uses K_max). + std::map column_to_k_file1 = {{"tags", 10}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k_file1)); + // Verify file1 physical schema has 10 columns. + auto struct_type1 = std::static_pointer_cast(phys_schema1->field(1)->type()); + ASSERT_EQ(12, struct_type1->num_fields()); // mapping + 10 cols + overflow + + MapSharedShreddingFieldMeta meta1; + meta1.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + meta1.field_to_columns = {{0, {0, 1}}, {1, {1, 2}}, {2, {0}}}; + meta1.num_columns = 10; + meta1.max_row_width = 3; + std::string compression = options.GetFileCompression(); + CheckShreddingFileSchema(file1_path, format, phys_schema1, /*field_index=*/1, meta1, + compression); + + // --- File 2: K should adapt to min(max_window=3, K_max=10) = 3 --- + // Write 5 keys → 3 fit in columns, 2 overflow. + auto batch2 = CreateBatch(logical_schema, R"([ + [3, [["x", 100], ["y", 200], ["z", 300], ["w", 400], ["v", 500]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch2))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc2, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc2.GetNewFilesIncrement().NewFiles().size()); + + std::string file2_path = + path_factory->ToPath(inc2.GetNewFilesIncrement().NewFiles()[0]->file_name); + + // File 2 should have K=3 (adapted from file1's max_row_width=3). + std::map column_to_k_file2 = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k_file2)); + auto struct_type2 = std::static_pointer_cast(phys_schema2->field(1)->type()); + ASSERT_EQ(5, struct_type2->num_fields()); // mapping + 3 cols + overflow + + MapSharedShreddingFieldMeta meta2; + meta2.name_to_id = {{"x", 0}, {"y", 1}, {"z", 2}, {"w", 3}, {"v", 4}}; + meta2.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}}; + meta2.overflow_field_set = {3, 4}; + meta2.num_columns = 3; + meta2.max_row_width = 5; + CheckShreddingFileSchema(file2_path, format, phys_schema2, /*field_index=*/1, meta2, + compression); + + // Verify data: 5 keys, K=3, so w and v overflow. + auto physical_type2 = arrow::struct_(phys_schema2->fields()); + std::shared_ptr expected_array2; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type2, {R"([ + [3, [[0, 1, 2], 100, 200, 300, [[3, 400], [4, 500]]]] + ])"}, + &expected_array2) + .ok()); + CheckFileContent(file2_path, format, expected_array2); + + // --- File 3: K should adapt to min(max_window=max(3,5)=5, K_max=10) = 5 --- + // File2 reported max_row_width=5, so window now has [3, 5], max=5. + // Write 4 keys → all fit in K=5, no overflow. + auto batch3 = CreateBatch(logical_schema, R"([ + [4, [["p", 1000], ["q", 2000], ["r", 3000], ["s", 4000]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch3))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc3, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc3.GetNewFilesIncrement().NewFiles().size()); + + std::string file3_path = + path_factory->ToPath(inc3.GetNewFilesIncrement().NewFiles()[0]->file_name); + + // File 3 should have K=5 (window max grew from file2's max_row_width=5). + std::map column_to_k_file3 = {{"tags", 5}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k_file3)); + auto struct_type3 = std::static_pointer_cast(phys_schema3->field(1)->type()); + ASSERT_EQ(7, struct_type3->num_fields()); // mapping + 5 cols + overflow + + MapSharedShreddingFieldMeta meta3; + meta3.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}, {"s", 3}}; + meta3.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}, {3, {3}}}; + meta3.num_columns = 5; + meta3.max_row_width = 4; + CheckShreddingFileSchema(file3_path, format, phys_schema3, /*field_index=*/1, meta3, + compression); + + // Verify data: 4 keys fit in K=5, col4 unused, no overflow. + auto physical_type3 = arrow::struct_(phys_schema3->fields()); + std::shared_ptr expected_array3; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type3, {R"([ + [4, [[0, 1, 2, 3, -1], 1000, 2000, 3000, 4000, null, null]] + ])"}, + &expected_array3) + .ok()); + CheckFileContent(file3_path, format, expected_array3); + + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKAdaptation) { + std::string format = GetFormat(); + // Two shared-shredding MAP fields with different initial K: tags(K=8), attrs(K=4). + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "8"}, + {"fields.attrs.map.storage-layout", "shared-shredding"}, + {"fields.attrs.map.shared-shredding.max-columns", "4"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::utf8())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + std::string compression = options.GetFileCompression(); + + // --- File 1: first file, tags K=8, attrs K=4 --- + // tags: max_row_width=2, attrs: max_row_width=1 + auto batch1 = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20]], [["x", "v1"]]], + [2, [["a", 30]], [["x", "v2"]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch1))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc1, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc1.GetNewFilesIncrement().NewFiles().size()); + + std::string file1_path = + path_factory->ToPath(inc1.GetNewFilesIncrement().NewFiles()[0]->file_name); + + // Verify file1: tags K=8, attrs K=4 (first file uses K_max). + std::map col_to_k_file1 = {{"tags", 8}, {"attrs", 4}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k_file1)); + + MapSharedShreddingFieldMeta meta1_tags; + meta1_tags.name_to_id = {{"a", 0}, {"b", 1}}; + meta1_tags.field_to_columns = {{0, {0}}, {1, {1}}}; + meta1_tags.num_columns = 8; + meta1_tags.max_row_width = 2; + CheckShreddingFileSchema(file1_path, format, phys_schema1, /*field_index=*/1, meta1_tags, + compression); + + MapSharedShreddingFieldMeta meta1_attrs; + meta1_attrs.name_to_id = {{"x", 0}}; + meta1_attrs.field_to_columns = {{0, {0}}}; + meta1_attrs.num_columns = 4; + meta1_attrs.max_row_width = 1; + CheckShreddingFileSchema(file1_path, format, phys_schema1, /*field_index=*/2, meta1_attrs, + compression); + + // --- File 2: tags K=min(2,8)=2, attrs K=min(1,4)=1 --- + // tags: 3 keys → 1 overflow; attrs: 3 keys → 2 overflow + auto batch2 = CreateBatch(logical_schema, R"([ + [3, [["c", 100], ["d", 200], ["e", 300]], [["p", "a1"], ["q", "a2"], ["r", "a3"]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch2))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc2, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc2.GetNewFilesIncrement().NewFiles().size()); + + std::string file2_path = + path_factory->ToPath(inc2.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map col_to_k_file2 = {{"tags", 2}, {"attrs", 1}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k_file2)); + + MapSharedShreddingFieldMeta meta2_tags; + meta2_tags.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}}; + meta2_tags.field_to_columns = {{0, {0}}, {1, {1}}}; + meta2_tags.overflow_field_set = {2}; + meta2_tags.num_columns = 2; + meta2_tags.max_row_width = 3; + CheckShreddingFileSchema(file2_path, format, phys_schema2, /*field_index=*/1, meta2_tags, + compression); + + MapSharedShreddingFieldMeta meta2_attrs; + meta2_attrs.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}}; + meta2_attrs.field_to_columns = {{0, {0}}}; + meta2_attrs.overflow_field_set = {1, 2}; + meta2_attrs.num_columns = 1; + meta2_attrs.max_row_width = 3; + CheckShreddingFileSchema(file2_path, format, phys_schema2, /*field_index=*/2, meta2_attrs, + compression); + + // --- File 3: tags K=min(max(2,3),8)=3, attrs K=min(max(1,3),4)=3 --- + // tags: 2 keys, fits; attrs: 2 keys, fits. + auto batch3 = CreateBatch(logical_schema, R"([ + [4, [["f", 400], ["g", 500]], [["s", "b1"], ["t", "b2"]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch3))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc3, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc3.GetNewFilesIncrement().NewFiles().size()); + + std::string file3_path = + path_factory->ToPath(inc3.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map col_to_k_file3 = {{"tags", 3}, {"attrs", 3}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k_file3)); + + MapSharedShreddingFieldMeta meta3_tags; + meta3_tags.name_to_id = {{"f", 0}, {"g", 1}}; + meta3_tags.field_to_columns = {{0, {0}}, {1, {1}}}; + meta3_tags.num_columns = 3; + meta3_tags.max_row_width = 2; + CheckShreddingFileSchema(file3_path, format, phys_schema3, /*field_index=*/1, meta3_tags, + compression); + + MapSharedShreddingFieldMeta meta3_attrs; + meta3_attrs.name_to_id = {{"s", 0}, {"t", 1}}; + meta3_attrs.field_to_columns = {{0, {0}}, {1, {1}}}; + meta3_attrs.num_columns = 3; + meta3_attrs.max_row_width = 2; + CheckShreddingFileSchema(file3_path, format, phys_schema3, /*field_index=*/2, meta3_attrs, + compression); + + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapDataFileMetaInfo) { + std::string format = GetFormat(); + // Verify PrepareCommit returns correct DataFileMeta for shared-shredding map files. + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + AppendOnlyWriter::Create(options, /*schema_id=*/5, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/9, path_factory, + compact_manager_, memory_pool_)); + + // Write 3 rows. + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30]]], + [3, [["a", 40], ["b", 50], ["c", 60]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + + auto actual_meta = inc.GetNewFilesIncrement().NewFiles()[0]; + + // Construct expected value_stats independently. + // Physical schema has 2 top-level fields: id(INT32), tags(STRUCT). + // id: min=1, max=3, null_count=0; tags is nested: NullType(), null_count=null. + int32_t map_null_count = (format == "parquet" ? -1 : 0); + auto expected_value_stats = BinaryRowGenerator::GenerateStats( + {1, NullType()}, {3, NullType()}, {0, map_null_count}, memory_pool_.get()); + + // Build expected DataFileMeta with fake file_name/file_size (TEST_Equal ignores them). + auto expected_meta = DataFileMeta::ForAppend( + /*file_name=*/"fake-file.parquet", /*file_size=*/999, /*row_count=*/3, + expected_value_stats, + /*min_sequence_number=*/10, /*max_sequence_number=*/12, + /*schema_id=*/5, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt) + .value(); + + ASSERT_TRUE(expected_meta->TEST_Equal(*actual_meta)); + + // Verify the written file has correct shared-shredding map content. + std::string file_path = path_factory->ToPath(actual_meta->file_name); + std::map col_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto phys_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k)); + auto physical_type = arrow::struct_(phys_schema->fields()); + + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1, -1], 10, 20, null, null]], + [2, [[2, -1, -1], 30, null, null, null]], + [3, [[0, 1, 2], 40, 50, 60, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(file_path, format, expected_array); + + ASSERT_OK(writer->Close()); +} + +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapWithBlobSeparation) { + std::string format = GetFormat(); + // Schema: id(INT32), blob_data(BLOB), tags(MAP) + // BLOB field will be separated into a .blob file; MAP field uses shared-shredding. + // This tests that blob separation + shredding works correctly together. + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + BlobUtils::ToArrowField("blob_data", false), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + // Write rows with id, blob_data, and tags. + // Row0: id=1, blob="hello", tags={a:10, b:20} + // Row1: id=2, blob="world", tags={c:30} + auto batch = CreateBatch(logical_schema, R"([ + [1, "hello", [["a", 10], ["b", 20]]], + [2, "world", [["c", 30]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + // Verify: should produce 2 files — one main data file and one .blob file. + const auto& new_files = inc.GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(2, new_files.size()); + + // Identify main vs blob file. + std::string main_file_path, blob_file_path; + for (const auto& file_meta : new_files) { + std::string file_path = path_factory->ToPath(file_meta->file_name); + if (BlobUtils::IsBlobFile(file_meta->file_name)) { + blob_file_path = file_path; + } else { + main_file_path = file_path; + } + } + ASSERT_FALSE(main_file_path.empty()) << "Main data file not found"; + ASSERT_FALSE(blob_file_path.empty()) << "Blob file not found"; + + // Verify both files exist on disk. + auto fs = options.GetFileSystem(); + ASSERT_TRUE(fs->Exists(main_file_path).value()); + ASSERT_TRUE(fs->Exists(blob_file_path).value()); + + // Verify main file schema: should have id(INT32) + tags(shredded STRUCT), no blob_data. + // Build expected physical schema for the main schema (id + tags). + auto main_logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + std::map col_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(main_logical_schema, col_to_k)); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 2; + + CheckShreddingFileSchema(main_file_path, format, expected_physical_schema, + /*field_index=*/1, expected_meta, options.GetFileCompression()); + + // Verify main file content: id + shredded tags. + auto physical_type = arrow::struct_(expected_physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1, -1], 10, 20, null, null]], + [2, [[2, -1, -1], 30, null, null, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(main_file_path, format, expected_array); + + // Verify blob file exists and can be read (blob_data column). + auto blob_reader = OpenFormatReader(blob_file_path, "blob"); + ASSERT_NE(blob_reader, nullptr); } } // namespace paimon::test diff --git a/src/paimon/core/io/data_file_writer.cpp b/src/paimon/core/io/data_file_writer.cpp index 6f8bb3bb..4ed3e040 100644 --- a/src/paimon/core/io/data_file_writer.cpp +++ b/src/paimon/core/io/data_file_writer.cpp @@ -45,6 +45,10 @@ DataFileWriter::DataFileWriter( stats_extractor_(stats_extractor), write_cols_(write_cols) {} +void DataFileWriter::SetMetadataFinalizer(MetadataFinalizer finalizer) { + metadata_finalizer_ = std::move(finalizer); +} + Status DataFileWriter::Write(ArrowArray* batch) { int64_t record_count = batch->length; PAIMON_RETURN_NOT_OK(SingleFileWriter::Write(batch)); @@ -52,6 +56,17 @@ Status DataFileWriter::Write(ArrowArray* batch) { return Status::OK(); } +Status DataFileWriter::BeforeFinish() { + if (metadata_finalizer_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr updated_schema, + metadata_finalizer_()); + if (updated_schema) { + PAIMON_RETURN_NOT_OK(UpdateSchema(updated_schema)); + } + } + return Status::OK(); +} + Result> DataFileWriter::GetResult() { PAIMON_ASSIGN_OR_RAISE(std::vector> field_stats, GetFieldStats()); PAIMON_ASSIGN_OR_RAISE(SimpleStats stats, diff --git a/src/paimon/core/io/data_file_writer.h b/src/paimon/core/io/data_file_writer.h index 097c9b95..60cc808a 100644 --- a/src/paimon/core/io/data_file_writer.h +++ b/src/paimon/core/io/data_file_writer.h @@ -33,6 +33,10 @@ #include "paimon/result.h" #include "paimon/status.h" +namespace arrow { +class Schema; +} // namespace arrow + namespace paimon { class ColumnStats; @@ -42,6 +46,11 @@ class MemoryPool; class DataFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr> { public: + /// Callback invoked during BeforeFinish() to finalize file metadata. + /// Produces an updated schema with per-field metadata (e.g. shredding metadata) + /// and may perform other finalization work (e.g. reporting stats to cross-file context). + using MetadataFinalizer = std::function>()>; + DataFileWriter(const std::string& compression, std::function converter, int64_t schema_id, const std::shared_ptr& seq_num_counter, FileSource file_source, @@ -49,14 +58,20 @@ class DataFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr>& write_cols, const std::shared_ptr& pool); + /// Sets the metadata finalizer. Called during BeforeFinish() to produce an updated + /// schema and perform finalization callbacks. Must be set before Close(). + void SetMetadataFinalizer(MetadataFinalizer finalizer); + Status Write(::ArrowArray* batch) override; Result> GetResult() override; + protected: + Status BeforeFinish() override; + private: Result>> GetFieldStats(); - private: std::shared_ptr pool_; int64_t schema_id_; bool is_external_path_; @@ -65,6 +80,7 @@ class DataFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr stats_extractor_; std::optional> write_cols_; + MetadataFinalizer metadata_finalizer_; }; } // namespace paimon diff --git a/src/paimon/core/io/key_value_data_file_writer.cpp b/src/paimon/core/io/key_value_data_file_writer.cpp index 82bbba92..9393c7c3 100644 --- a/src/paimon/core/io/key_value_data_file_writer.cpp +++ b/src/paimon/core/io/key_value_data_file_writer.cpp @@ -64,6 +64,10 @@ KeyValueDataFileWriter::KeyValueDataFileWriter( is_external_path_(is_external_path), disable_stats_(stats_extractor == nullptr) {} +void KeyValueDataFileWriter::SetMetadataFinalizer(MetadataFinalizer finalizer) { + metadata_finalizer_ = std::move(finalizer); +} + Status KeyValueDataFileWriter::Write(KeyValueBatch batch) { // update min and max key if (!min_key_) { @@ -75,10 +79,22 @@ Status KeyValueDataFileWriter::Write(KeyValueBatch batch) { max_sequence_number_ = std::max(max_sequence_number_, batch.max_sequence_number); // update delete row count delete_row_count_ += batch.delete_row_count; + PAIMON_RETURN_NOT_OK(SingleFileWriter::Write(std::move(batch))); return Status::OK(); } +Status KeyValueDataFileWriter::BeforeFinish() { + if (metadata_finalizer_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr updated_schema, + metadata_finalizer_()); + if (updated_schema) { + PAIMON_RETURN_NOT_OK(UpdateSchema(updated_schema)); + } + } + return Status::OK(); +} + Result> KeyValueDataFileWriter::GetResult() { PAIMON_ASSIGN_OR_RAISE(std::vector> field_stats, GetFieldStats()); if (!disable_stats_ && field_stats.size() != static_cast(write_schema_->num_fields())) { diff --git a/src/paimon/core/io/key_value_data_file_writer.h b/src/paimon/core/io/key_value_data_file_writer.h index f62bf64c..e1e3fd92 100644 --- a/src/paimon/core/io/key_value_data_file_writer.h +++ b/src/paimon/core/io/key_value_data_file_writer.h @@ -47,6 +47,11 @@ class SimpleStats; class KeyValueDataFileWriter : public SingleFileWriter> { public: + /// Callback invoked during BeforeFinish() to finalize file metadata. + /// Produces an updated schema with per-field metadata (e.g. shredding metadata) + /// and may perform other finalization work (e.g. reporting stats to cross-file context). + using MetadataFinalizer = std::function>()>; + KeyValueDataFileWriter(const std::string& compression, std::function converter, int64_t schema_id, int32_t level, FileSource file_source, @@ -55,10 +60,17 @@ class KeyValueDataFileWriter const std::shared_ptr& write_schema, bool is_external_path, const std::shared_ptr& pool); + /// Sets the metadata finalizer. Called during BeforeFinish() to produce an updated + /// schema and perform finalization callbacks. Must be set before Close(). + void SetMetadataFinalizer(MetadataFinalizer finalizer); + Status Write(KeyValueBatch batch) override; Result> GetResult() override; + protected: + Status BeforeFinish() override; + private: Result>> GetFieldStats(); @@ -84,6 +96,7 @@ class KeyValueDataFileWriter int64_t max_sequence_number_ = std::numeric_limits::min(); std::shared_ptr min_key_; std::shared_ptr max_key_; + MetadataFinalizer metadata_finalizer_; }; } // namespace paimon diff --git a/src/paimon/core/io/single_file_writer.h b/src/paimon/core/io/single_file_writer.h index 2ead8aec..c085729c 100644 --- a/src/paimon/core/io/single_file_writer.h +++ b/src/paimon/core/io/single_file_writer.h @@ -21,12 +21,15 @@ #include #include #include +#include #include #include #include #include "arrow/c/abi.h" +#include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/ipc/api.h" #include "fmt/format.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/scope_guard.h" @@ -41,6 +44,10 @@ #include "paimon/result.h" #include "paimon/status.h" +namespace arrow { +class Schema; +} // namespace arrow + namespace paimon { class RecordBatch; @@ -113,6 +120,15 @@ class SingleFileWriter : public FileWriter { } protected: + /// Hook called after Flush() and before Finish() during Close(). + /// Subclasses can override to update per-field metadata before the file is finalized. + virtual Status BeforeFinish() { + return Status::OK(); + } + + /// Serializes schema and forwards it as file metadata to FormatWriter. + Status UpdateSchema(const std::shared_ptr& schema); + int64_t output_bytes_ = -1; std::string compression_; std::function converter_; @@ -201,6 +217,7 @@ Status SingleFileWriter::Close() { path_.c_str()); }); PAIMON_RETURN_NOT_OK(writer_->Flush()); + PAIMON_RETURN_NOT_OK(BeforeFinish()); PAIMON_RETURN_NOT_OK(writer_->Finish()); if (out_) { PAIMON_RETURN_NOT_OK(out_->Flush()); @@ -220,6 +237,20 @@ Result SingleFileWriter::ReachTargetSize(bool suggested_check, int64 return writer_->ReachTargetSize(suggested_check, target_size); } +template +Status SingleFileWriter::UpdateSchema(const std::shared_ptr& schema) { + if (!writer_) { + return Status::Invalid("Cannot update schema: format writer is not initialized."); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr serialized, + arrow::ipc::SerializeSchema(*schema)); + std::map metadata; + metadata.emplace(ArrowUtils::kArrowSchemaMetadataKey, + std::string(reinterpret_cast(serialized->data()), + static_cast(serialized->size()))); + return writer_->AddMetadata(metadata); +} + template void SingleFileWriter::Abort() { if (out_) { diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp index ecd2af7e..bb415bb0 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp @@ -27,11 +27,13 @@ ChangelogMergeTreeRewriter::ChangelogMergeTreeRewriter( std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, const std::shared_ptr& cancellation_controller, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool) - : MergeTreeCompactRewriter( - partition, bucket, schema_id, trimmed_primary_keys, options, data_schema, write_schema, - std::move(dv_factory), path_factory_cache, std::move(merge_file_split_read), - std::move(merge_function_wrapper_factory), cancellation_controller, pool), + : MergeTreeCompactRewriter(partition, bucket, schema_id, trimmed_primary_keys, options, + data_schema, write_schema, std::move(dv_factory), path_factory_cache, + std::move(merge_file_split_read), + std::move(merge_function_wrapper_factory), cancellation_controller, + shredding_context, pool), max_level_(max_level), force_drop_delete_(force_drop_delete) {} diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h index c5d8e891..1e6b9034 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h @@ -43,6 +43,7 @@ class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter { std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, const std::shared_ptr& cancellation_controller, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool); struct UpgradeStrategy { diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp index c8c0b182..19cfe0e4 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp @@ -18,6 +18,10 @@ #include "paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h" +#include + +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/table/special_fields.h" #include "paimon/core/mergetree/compact/first_row_merge_function_wrapper.h" #include "paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h" @@ -38,13 +42,14 @@ LookupMergeTreeCompactRewriter::LookupMergeTreeCompactRewriter( MergeFunctionWrapperFactory merge_function_wrapper_factory, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool) : ChangelogMergeTreeRewriter( max_level, /*force_drop_delete=*/dv_maintainer != nullptr, partition, bucket, schema_id, trimmed_primary_keys, options, data_schema, write_schema, DeletionVector::CreateFactory(dv_maintainer), path_factory_cache, std::move(merge_file_split_read), std::move(merge_function_wrapper_factory), - cancellation_controller, pool), + cancellation_controller, shredding_context, pool), lookup_levels_(std::move(lookup_levels)), dv_maintainer_(dv_maintainer), remote_lookup_file_manager_(remote_lookup_file_manager) {} @@ -90,11 +95,15 @@ LookupMergeTreeCompactRewriter::Create( PAIMON_ASSIGN_OR_RAISE( std::unique_ptr merge_file_split_read, MergeFileSplitRead::Create(path_factory, internal_context, pool, CreateDefaultExecutor())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); + return std::unique_ptr(new LookupMergeTreeCompactRewriter( std::move(lookup_levels), dv_maintainer, max_level, partition, bucket, table_schema->Id(), trimmed_primary_keys, options, data_schema, write_schema, path_factory_cache, std::move(merge_file_split_read), std::move(merge_function_wrapper_factory), - cancellation_controller, remote_lookup_file_manager, pool)); + cancellation_controller, remote_lookup_file_manager, shredding_context, pool)); } template diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h index 4aaf704f..7890ed25 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h @@ -71,6 +71,7 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { MergeFunctionWrapperFactory merge_function_wrapper_factory, const std::shared_ptr& cancellation_controller, const std::shared_ptr& remote_lookup_file_manager, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool); bool RewriteChangelog(int32_t output_level, bool drop_delete, diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index d5836d09..662426ed 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -1078,7 +1078,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + cancellation_controller, /*remote_lookup_file_manager=*/nullptr, + /*shredding_context=*/nullptr, pool_); auto file = create_meta(/*level=*/1, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::NoChangelogNoRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); @@ -1093,7 +1094,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + cancellation_controller, /*remote_lookup_file_manager=*/nullptr, + /*shredding_context=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file)); @@ -1112,7 +1114,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + cancellation_controller, /*remote_lookup_file_manager=*/nullptr, + /*shredding_context=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/1); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); @@ -1127,7 +1130,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + cancellation_controller, /*remote_lookup_file_manager=*/nullptr, + /*shredding_context=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file)); @@ -1142,7 +1146,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + cancellation_controller, /*remote_lookup_file_manager=*/nullptr, + /*shredding_context=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); @@ -1158,7 +1163,8 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) { /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr, /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr, /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr, - cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_); + cancellation_controller, /*remote_lookup_file_manager=*/nullptr, + /*shredding_context=*/nullptr, pool_); auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt); ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(), rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file)); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index 519c1701..6c16b0f1 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -18,9 +18,14 @@ #include "paimon/core/mergetree/compact/merge_tree_compact_rewriter.h" #include +#include #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/io/key_value_data_file_writer.h" @@ -43,6 +48,7 @@ MergeTreeCompactRewriter::MergeTreeCompactRewriter( std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, const std::shared_ptr& cancellation_controller, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool) : options_(options), merge_file_split_read_(std::move(merge_file_split_read)), @@ -56,7 +62,8 @@ MergeTreeCompactRewriter::MergeTreeCompactRewriter( dv_factory_(std::move(dv_factory)), path_factory_cache_(path_factory_cache), merge_function_wrapper_factory_(std::move(merge_function_wrapper_factory)), - cancellation_controller_(cancellation_controller) { + cancellation_controller_(cancellation_controller), + shredding_context_(shredding_context) { assert(cancellation_controller_ != nullptr); } @@ -100,10 +107,14 @@ Result> MergeTreeCompactRewriter::Crea [](int32_t output_level) -> Result>> { return std::shared_ptr>(); }; + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); + return std::unique_ptr(new MergeTreeCompactRewriter( partition, bucket, table_schema->Id(), trimmed_primary_keys, options, data_schema, write_schema, std::move(dv_factory), path_factory_cache, std::move(merge_file_split_read), - merge_function_wrapper_factory, cancellation_controller, pool)); + merge_function_wrapper_factory, cancellation_controller, shredding_context, pool)); } Result MergeTreeCompactRewriter::Upgrade(int32_t output_level, @@ -134,30 +145,59 @@ std::unique_ptr MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) { auto create_file_writer = [this, level]() -> Result>>> { + // Determine file-level schema. When shredding is active, compute per-file K + // and build a physical schema with MAP columns replaced by STRUCT. + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, + MapSharedShreddingBatchConverter::CreateConverter( + write_schema_, shredding_context_, pool_)); + std::shared_ptr file_schema = + bundle.physical_schema ? bundle.physical_schema : write_schema_; + ::ArrowSchema arrow_schema{}; ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); auto format = options_.GetWriteFileFormat(level); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr writer_builder, format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); writer_builder->WithMemoryPool(pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, format->CreateStatsExtractor(&arrow_schema)); - auto converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { - ArrowArrayMove(key_value_batch.batch.get(), array); - return Status::OK(); - }; + // Build the converter that transforms KeyValueBatch to ArrowArray. + // When shredding is active, it performs MAP→STRUCT conversion on the batch data. + std::function kv_converter; + if (bundle.converter) { + auto shredding_converter = bundle.converter; + kv_converter = [shredding_converter](KeyValueBatch key_value_batch, + ArrowArray* array) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, + shredding_converter->Convert(key_value_batch.batch.get())); + ArrowArrayMove(physical.get(), array); + return Status::OK(); + }; + } else { + kv_converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { + ArrowArrayMove(key_value_batch.batch.get(), array); + return Status::OK(); + }; + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, CreateDataFilePathFactory(format->Identifier())); auto writer = std::make_unique( - options_.GetWriteFileCompression(level), converter, schema_id_, level, - FileSource::Compact(), trimmed_primary_keys_, stats_extractor, write_schema_, + options_.GetWriteFileCompression(level), kv_converter, schema_id_, level, + FileSource::Compact(), trimmed_primary_keys_, stats_extractor, file_schema, data_file_path_factory->IsExternalPath(), pool_); PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), data_file_path_factory->NewPath(), writer_builder)); + + if (bundle.converter) { + writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( + bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, + shredding_context_, file_schema)); + } + return writer; }; return std::make_unique( diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h index 3c7ab00b..a42e26e9 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h @@ -32,6 +32,8 @@ #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/file_store_path_factory_cache.h" namespace paimon { +class MapSharedShreddingContext; + /// Default `CompactRewriter` for merge trees. class MergeTreeCompactRewriter : public CompactRewriter { public: @@ -81,6 +83,7 @@ class MergeTreeCompactRewriter : public CompactRewriter { std::unique_ptr&& merge_file_split_read, MergeFunctionWrapperFactory merge_function_wrapper_factory, const std::shared_ptr& cancellation_controller, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool); using KeyValueRollingFileWriter = @@ -121,6 +124,9 @@ class MergeTreeCompactRewriter : public CompactRewriter { std::shared_ptr path_factory_cache_; MergeFunctionWrapperFactory merge_function_wrapper_factory_; std::shared_ptr cancellation_controller_; + + /// Cross-file shared context for shared-shredding MAP columns (nullable). + std::shared_ptr shredding_context_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp index dbca52e0..459a0055 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer.cpp @@ -20,12 +20,17 @@ #include #include +#include #include #include #include "arrow/api.h" #include "arrow/c/abi.h" #include "arrow/c/helpers.h" +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -61,27 +66,33 @@ Result> MergeTreeWriter::Create( const std::shared_ptr& io_manager, bool enable_multi_thread_spill, const std::shared_ptr& pool) { auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); + PAIMON_ASSIGN_OR_RAISE( std::unique_ptr write_buffer, WriteBuffer::Create(last_sequence_number, value_schema, trimmed_primary_keys, options.GetSequenceField(), key_comparator, user_defined_seq_comparator, merge_function_wrapper, options, io_manager, enable_multi_thread_spill, pool)); - return std::shared_ptr( - new MergeTreeWriter(pool, trimmed_primary_keys, options, path_factory, key_comparator, - user_defined_seq_comparator, merge_function_wrapper, schema_id, - write_schema, compact_manager, std::move(write_buffer))); + return std::shared_ptr(new MergeTreeWriter( + trimmed_primary_keys, options, path_factory, key_comparator, user_defined_seq_comparator, + merge_function_wrapper, schema_id, write_schema, compact_manager, std::move(write_buffer), + shredding_context, pool)); } MergeTreeWriter::MergeTreeWriter( - const std::shared_ptr& pool, const std::vector& trimmed_primary_keys, - const CoreOptions& options, const std::shared_ptr& path_factory, + const std::vector& trimmed_primary_keys, const CoreOptions& options, + const std::shared_ptr& path_factory, const std::shared_ptr& key_comparator, const std::shared_ptr& user_defined_seq_comparator, const std::shared_ptr>& merge_function_wrapper, int64_t schema_id, const std::shared_ptr& write_schema, const std::shared_ptr& compact_manager, - std::unique_ptr&& write_buffer) + std::unique_ptr&& write_buffer, + const std::shared_ptr& shredding_context, + const std::shared_ptr& pool) : pool_(pool), trimmed_primary_keys_(trimmed_primary_keys), options_(options), @@ -93,7 +104,8 @@ MergeTreeWriter::MergeTreeWriter( write_schema_(write_schema), compact_manager_(compact_manager), write_buffer_(std::move(write_buffer)), - metrics_(std::make_shared()) {} + metrics_(std::make_shared()), + shredding_context_(shredding_context) {} Status MergeTreeWriter::DoClose() { // Request cancellation and wait for running compaction to exit. @@ -319,29 +331,57 @@ Result MergeTreeWriter::DrainIncrement() { std::unique_ptr>> MergeTreeWriter::CreateRollingRowWriter() const { - auto create_file_writer = [&]() + auto create_file_writer = [this]() -> Result>>> { + // Determine file-level schema. When shredding is active, compute per-file K + // and build a physical schema with MAP columns replaced by STRUCT. + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, + MapSharedShreddingBatchConverter::CreateConverter( + write_schema_, shredding_context_, pool_)); + std::shared_ptr file_schema = + bundle.physical_schema ? bundle.physical_schema : write_schema_; + ::ArrowSchema arrow_schema; ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); auto format = options_.GetWriteFileFormat(/*level=*/0); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr writer_builder, format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); writer_builder->WithMemoryPool(pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, format->CreateStatsExtractor(&arrow_schema)); - auto converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { - ArrowArrayMove(key_value_batch.batch.get(), array); - return Status::OK(); - }; + // Build the converter that transforms KeyValueBatch to ArrowArray. + // When shredding is active, it performs MAP→STRUCT conversion on the batch data. + std::function kv_converter; + if (bundle.converter) { + auto converter = bundle.converter; + kv_converter = [converter](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, + converter->Convert(key_value_batch.batch.get())); + ArrowArrayMove(physical.get(), array); + return Status::OK(); + }; + } else { + kv_converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { + ArrowArrayMove(key_value_batch.batch.get(), array); + return Status::OK(); + }; + } auto writer = std::make_unique( - options_.GetWriteFileCompression(0), converter, schema_id_, /*level=*/0, - FileSource::Append(), trimmed_primary_keys_, stats_extractor, write_schema_, + options_.GetWriteFileCompression(0), kv_converter, schema_id_, /*level=*/0, + FileSource::Append(), trimmed_primary_keys_, stats_extractor, file_schema, path_factory_->IsExternalPath(), pool_); PAIMON_RETURN_NOT_OK( writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder)); + + if (bundle.converter) { + writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( + bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, + shredding_context_, file_schema)); + } + return writer; }; return std::make_unique>>( diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h index bc1a1b49..b437e6ad 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.h +++ b/src/paimon/core/mergetree/merge_tree_writer.h @@ -49,6 +49,7 @@ namespace paimon { class DataFilePathFactory; class IOManager; class FieldsComparator; +class MapSharedShreddingContext; class MemoryPool; class Metrics; template @@ -105,8 +106,7 @@ class MergeTreeWriter : public BatchWriter { Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file); private: - MergeTreeWriter(const std::shared_ptr& pool, - const std::vector& trimmed_primary_keys, + MergeTreeWriter(const std::vector& trimmed_primary_keys, const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& key_comparator, @@ -114,7 +114,9 @@ class MergeTreeWriter : public BatchWriter { const std::shared_ptr>& merge_function_wrapper, int64_t schema_id, const std::shared_ptr& write_schema, const std::shared_ptr& compact_manager, - std::unique_ptr&& write_buffer); + std::unique_ptr&& write_buffer, + const std::shared_ptr& shredding_context, + const std::shared_ptr& pool); std::shared_ptr pool_; std::vector trimmed_primary_keys_; @@ -139,5 +141,8 @@ class MergeTreeWriter : public BatchWriter { std::vector> compact_after_; std::shared_ptr compact_deletion_file_; + + /// Cross-file shared context for shared-shredding MAP columns (nullable). + std::shared_ptr shredding_context_; }; } // namespace paimon diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp index b4abb514..cce77330 100644 --- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp @@ -31,6 +31,7 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" @@ -156,6 +157,34 @@ class MergeTreeWriterTest : public ::testing::TestWithParam { ASSERT_TRUE(expected_array->Equals(result_array)) << result_array->ToString(); } + void CheckShreddingFileSchema(const std::string& data_file_name, + const std::shared_ptr& expected_physical_schema, + int32_t field_index, + const MapSharedShreddingFieldMeta& expected_meta) const { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(data_file_name)); + ASSERT_TRUE(input_stream); + ASSERT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get("orc", /*options=*/{})); + ASSERT_OK_AND_ASSIGN(auto reader_builder, + file_format->CreateReaderBuilder(/*batch_size=*/10)); + ASSERT_OK_AND_ASSIGN(auto orc_batch_reader, reader_builder->Build(input_stream)); + ASSERT_OK_AND_ASSIGN(auto c_file_schema, orc_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + + ASSERT_TRUE(file_schema->Equals(*expected_physical_schema, /*check_metadata=*/false)) + << "Expected schema:\n" + << expected_physical_schema->ToString() << "\nActual schema:\n" + << file_schema->ToString(); + + auto metadata = file_schema->field(field_index)->metadata(); + ASSERT_NE(nullptr, metadata); + ASSERT_OK_AND_ASSIGN( + auto deserialized_meta, + MapSharedShreddingUtils::DeserializeMetadata( + metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression)); + ASSERT_EQ(expected_meta, deserialized_meta); + } + std::shared_ptr CreateMeta(const std::string& name, int32_t level) const { return std::make_shared( name, /*file_size=*/100, /*row_count=*/1, DataFileMeta::EmptyMinKey(), @@ -349,6 +378,240 @@ TEST_P(MergeTreeWriterTest, TestWriteMultiBatch) { ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); } +TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({ + {Options::FILE_FORMAT, "orc"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {Options::WRITE_ONLY, "true"}, + })); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + std::string uuid = path_factory->uuid_; + + std::vector value_fields = { + DataField(0, arrow::field("id", arrow::int32())), + DataField(1, arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64()))), + }; + auto value_schema = DataField::ConvertDataFieldsToArrowSchema(value_fields); + auto value_type = DataField::ConvertDataFieldsToArrowStructType(value_fields); + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator, + FieldsComparator::Create({value_fields[0]}, + /*is_ascending_order=*/true)); + + ASSERT_OK_AND_ASSIGN( + auto merge_writer, + MergeTreeWriter::Create( + /*last_sequence_number=*/9, /*trimmed_primary_keys=*/{"id"}, path_factory, + key_comparator, + /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/5, + value_schema, options, noop_compact_manager_, + GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr, + /*enable_multi_thread_spill=*/false, pool_)); + + // Each batch contains duplicated primary keys. DeduplicateMergeFunction should keep the + // latest sequence number for each key across and within batches. + auto array1 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30]]], + [1, [["a", 11], ["c", 31]]] + ])") + .ValueOrDie(); + WriteBatch(array1, /*row_kinds=*/{}, merge_writer.get()); + + auto array2 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([ + [2, [["b", 40]]], + [1, [["c", 50], ["d", 60]]], + [2, [["a", 70], ["b", 80], ["c", 90]]] + ])") + .ValueOrDie(); + WriteBatch(array2, /*row_kinds=*/{}, merge_writer.get()); + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(merge_writer->Close()); + + ASSERT_TRUE(commit_increment.GetCompactIncrement().IsEmpty()); + ASSERT_EQ(1, commit_increment.GetNewFilesIncrement().NewFiles().size()); + auto actual_meta = commit_increment.GetNewFilesIncrement().NewFiles()[0]; + + std::string expected_data_file_name = "data-" + uuid + "-0.orc"; + std::string expected_data_file_path = dir->Str() + "/" + expected_data_file_name; + ASSERT_OK_AND_ASSIGN(std::unique_ptr data_file_status, + options.GetFileSystem()->GetFileStatus(expected_data_file_path)); + + auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema); + std::map column_to_k = {{"tags", 3}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k)); + auto physical_type = arrow::struct_(physical_schema->fields()); + + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [14, 0, 1, [[0, 1, -1], 50, 60, null, null]], + [15, 0, 2, [[2, 3, 0], 70, 80, 90, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(expected_data_file_path, expected_array); + + MapSharedShreddingFieldMeta expected_shredding_meta; + expected_shredding_meta.name_to_id = {{"a", 2}, {"b", 3}, {"c", 0}, {"d", 1}}; + expected_shredding_meta.field_to_columns = {{0, {0, 2}}, {1, {1}}, {2, {0}}, {3, {1}}}; + expected_shredding_meta.num_columns = 3; + expected_shredding_meta.max_row_width = 3; + CheckShreddingFileSchema(expected_data_file_path, physical_schema, /*field_index=*/3, + expected_shredding_meta); + + auto expected_data_file_meta = std::make_shared( + expected_data_file_name, /*file_size=*/data_file_status->GetLen(), /*row_count=*/2, + /*min_key=*/BinaryRowGenerator::GenerateRow({1}, pool_.get()), + /*max_key=*/BinaryRowGenerator::GenerateRow({2}, pool_.get()), + /*key_stats=*/ + BinaryRowGenerator::GenerateStats({1}, {2}, {0}, pool_.get()), + /*value_stats=*/ + BinaryRowGenerator::GenerateStats({1, NullType()}, {2, NullType()}, {0, 0}, pool_.get()), + /*min_sequence_number=*/14, /*max_sequence_number=*/15, /*schema_id=*/5, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/actual_meta->creation_time, /*delete_row_count=*/0, + /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); + ASSERT_TRUE(expected_data_file_meta->TEST_Equal(*actual_meta)); +} + +TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({ + {Options::FILE_FORMAT, "orc"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "8"}, + {"fields.attrs.map.storage-layout", "shared-shredding"}, + {"fields.attrs.map.shared-shredding.max-columns", "4"}, + {Options::WRITE_ONLY, "true"}, + })); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); + + std::vector value_fields = { + DataField(0, arrow::field("id", arrow::int32())), + DataField(1, arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64()))), + DataField(2, arrow::field("attrs", arrow::map(arrow::utf8(), arrow::utf8()))), + }; + auto value_schema = DataField::ConvertDataFieldsToArrowSchema(value_fields); + auto value_type = DataField::ConvertDataFieldsToArrowStructType(value_fields); + auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema); + ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator, + FieldsComparator::Create({value_fields[0]}, + /*is_ascending_order=*/true)); + + ASSERT_OK_AND_ASSIGN( + auto merge_writer, + MergeTreeWriter::Create( + /*last_sequence_number=*/-1, /*trimmed_primary_keys=*/{"id"}, path_factory, + key_comparator, + /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0, + value_schema, options, noop_compact_manager_, + GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr, + /*enable_multi_thread_spill=*/false, pool_)); + + auto array1 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([ + [1, [["a", 10], ["b", 20]], [["x", "v1"]]], + [2, [["a", 30]], [["x", "v2"]]] + ])") + .ValueOrDie(); + WriteBatch(array1, /*row_kinds=*/{}, merge_writer.get()); + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment1, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_EQ(1, commit_increment1.GetNewFilesIncrement().NewFiles().size()); + std::string file1_path = + path_factory->ToPath(commit_increment1.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map column_to_k_file1 = {{"tags", 8}, {"attrs", 4}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k_file1)); + MapSharedShreddingFieldMeta tags_meta1; + tags_meta1.name_to_id = {{"a", 0}, {"b", 1}}; + tags_meta1.field_to_columns = {{0, {0}}, {1, {1}}}; + tags_meta1.num_columns = 8; + tags_meta1.max_row_width = 2; + CheckShreddingFileSchema(file1_path, physical_schema1, /*field_index=*/3, tags_meta1); + + MapSharedShreddingFieldMeta attrs_meta1; + attrs_meta1.name_to_id = {{"x", 0}}; + attrs_meta1.field_to_columns = {{0, {0}}}; + attrs_meta1.num_columns = 4; + attrs_meta1.max_row_width = 1; + CheckShreddingFileSchema(file1_path, physical_schema1, /*field_index=*/4, attrs_meta1); + + auto array2 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([ + [3, [["c", 100], ["d", 200], ["e", 300]], [["p", "a1"], ["q", "a2"], ["r", "a3"]]] + ])") + .ValueOrDie(); + WriteBatch(array2, /*row_kinds=*/{}, merge_writer.get()); + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment2, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_EQ(1, commit_increment2.GetNewFilesIncrement().NewFiles().size()); + std::string file2_path = + path_factory->ToPath(commit_increment2.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map column_to_k_file2 = {{"tags", 2}, {"attrs", 1}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k_file2)); + MapSharedShreddingFieldMeta tags_meta2; + tags_meta2.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}}; + tags_meta2.field_to_columns = {{0, {0}}, {1, {1}}}; + tags_meta2.overflow_field_set = {2}; + tags_meta2.num_columns = 2; + tags_meta2.max_row_width = 3; + CheckShreddingFileSchema(file2_path, physical_schema2, /*field_index=*/3, tags_meta2); + + MapSharedShreddingFieldMeta attrs_meta2; + attrs_meta2.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}}; + attrs_meta2.field_to_columns = {{0, {0}}}; + attrs_meta2.overflow_field_set = {1, 2}; + attrs_meta2.num_columns = 1; + attrs_meta2.max_row_width = 3; + CheckShreddingFileSchema(file2_path, physical_schema2, /*field_index=*/4, attrs_meta2); + + auto array3 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([ + [4, [["f", 400], ["g", 500]], [["s", "b1"], ["t", "b2"]]] + ])") + .ValueOrDie(); + WriteBatch(array3, /*row_kinds=*/{}, merge_writer.get()); + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment3, + merge_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_EQ(1, commit_increment3.GetNewFilesIncrement().NewFiles().size()); + std::string file3_path = + path_factory->ToPath(commit_increment3.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map column_to_k_file3 = {{"tags", 3}, {"attrs", 3}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k_file3)); + MapSharedShreddingFieldMeta tags_meta3; + tags_meta3.name_to_id = {{"f", 0}, {"g", 1}}; + tags_meta3.field_to_columns = {{0, {0}}, {1, {1}}}; + tags_meta3.num_columns = 3; + tags_meta3.max_row_width = 2; + CheckShreddingFileSchema(file3_path, physical_schema3, /*field_index=*/3, tags_meta3); + + MapSharedShreddingFieldMeta attrs_meta3; + attrs_meta3.name_to_id = {{"s", 0}, {"t", 1}}; + attrs_meta3.field_to_columns = {{0, {0}}, {1, {1}}}; + attrs_meta3.num_columns = 3; + attrs_meta3.max_row_width = 2; + CheckShreddingFileSchema(file3_path, physical_schema3, /*field_index=*/4, attrs_meta3); + + ASSERT_OK(merge_writer->Close()); +} + TEST_P(MergeTreeWriterTest, TestWriteWithDeleteRow) { ASSERT_OK_AND_ASSIGN( CoreOptions options, diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index 459dcf67..c7b6355a 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -18,9 +18,15 @@ #include "paimon/core/operation/append_only_file_store_write.h" +#include #include +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" #include "paimon/common/data/binary_row.h" +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/core/append/append_only_writer.h" @@ -114,10 +120,14 @@ Result>> AppendOnlyFileStoreWrite::Com PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, CreateFilesReader(partition, bucket, dv_factory, to_compact)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema_, options_)); auto rewriter = std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterCreator(partition, bucket, write_schema_, write_cols_, to_compact)); + GetDataFileWriterCreator(partition, bucket, write_schema_, write_cols_, to_compact, + shredding_context)); ScopeGuard reader_guard([&]() { if (reader) { @@ -201,44 +211,70 @@ Result> AppendOnlyFileStoreWrite::CreateWriter( compaction_metrics_->CreateReporter(partition, bucket), cancellation_controller); } - auto writer = std::make_shared( - options_, table_schema_->Id(), write_schema_, write_cols_, restore_max_seq_number, - data_file_path_factory, compact_manager, pool_); - return std::shared_ptr(writer); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr writer, + AppendOnlyWriter::Create(options_, table_schema_->Id(), write_schema_, write_cols_, + restore_max_seq_number, data_file_path_factory, compact_manager, + pool_)); + return std::shared_ptr(std::move(writer)); } AppendOnlyFileStoreWrite::SingleFileWriterCreator AppendOnlyFileStoreWrite::GetDataFileWriterCreator( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& schema, const std::optional>& write_cols, - const std::vector>& to_compact) const { + const std::vector>& to_compact, + const std::shared_ptr& shredding_context) const { return - [this, partition, bucket, schema, write_cols, to_compact]() + [this, partition, bucket, schema, write_cols, to_compact, shredding_context]() -> Result< std::unique_ptr>>> { + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, + MapSharedShreddingBatchConverter::CreateConverter( + schema, shredding_context, pool_)); + std::shared_ptr file_schema = + bundle.physical_schema ? bundle.physical_schema : schema; + ::ArrowSchema arrow_schema; ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); auto format = options_.GetFileFormat(); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr writer_builder, format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); writer_builder->WithMemoryPool(pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &arrow_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, format->CreateStatsExtractor(&arrow_schema)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr data_file_path_factory, file_store_path_factory_->CreateDataFilePathFactory(partition, bucket)); + + std::function batch_converter; + if (bundle.converter) { + auto converter = bundle.converter; + batch_converter = [converter](ArrowArray* input, ArrowArray* output) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, + converter->Convert(input)); + ArrowArrayMove(physical.get(), output); + return Status::OK(); + }; + } + auto writer = std::make_unique( - options_.GetFileCompression(), std::function(), - table_schema_->Id(), + options_.GetFileCompression(), batch_converter, table_schema_->Id(), std::make_shared(to_compact[0]->min_sequence_number), FileSource::Compact(), stats_extractor, data_file_path_factory->IsExternalPath(), write_cols, pool_); PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), data_file_path_factory->NewPath(), writer_builder)); + + if (bundle.converter) { + writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( + bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, + shredding_context, file_schema)); + } return writer; }; } diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h index e3b729ec..7fb75fe4 100644 --- a/src/paimon/core/operation/append_only_file_store_write.h +++ b/src/paimon/core/operation/append_only_file_store_write.h @@ -60,6 +60,7 @@ class BinaryRow; class CoreOptions; class Executor; class Logger; +class MapSharedShreddingContext; class MemoryPool; class SchemaManager; class TableSchema; @@ -110,7 +111,8 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { SingleFileWriterCreator GetDataFileWriterCreator( const BinaryRow& partition, int32_t bucket, const std::shared_ptr& schema, const std::optional>& write_cols, - const std::vector>& to_compact) const; + const std::vector>& to_compact, + const std::shared_ptr& shredding_context) const; Result> CreateFilesReader( const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory, diff --git a/src/paimon/core/postpone/postpone_bucket_file_store_write.h b/src/paimon/core/postpone/postpone_bucket_file_store_write.h index d1c2ef41..1ca65d28 100644 --- a/src/paimon/core/postpone/postpone_bucket_file_store_write.h +++ b/src/paimon/core/postpone/postpone_bucket_file_store_write.h @@ -127,10 +127,11 @@ class PostponeBucketFileStoreWrite : public AbstractFileStoreWrite { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr data_file_path_factory, file_store_path_factory_->CreateDataFilePathFactory(partition, bucket)); - auto writer = - std::make_shared(trimmed_primary_keys, data_file_path_factory, - table_schema_->Id(), schema_, options_, pool_); - return std::shared_ptr(writer); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr writer, + PostponeBucketWriter::Create(trimmed_primary_keys, data_file_path_factory, + table_schema_->Id(), schema_, options_, pool_)); + return writer; } Result> CreateFileStoreScan( diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp index 543f83d7..9de632e2 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp @@ -20,6 +20,7 @@ #include #include +#include #include "arrow/api.h" #include "arrow/array/array_base.h" @@ -32,6 +33,9 @@ #include "arrow/scalar.h" #include "arrow/util/checked_cast.h" #include "fmt/format.h" +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" @@ -54,29 +58,52 @@ namespace paimon { class InternalRow; class MemoryPool; -PostponeBucketWriter::PostponeBucketWriter(const std::vector& trimmed_primary_keys, - const std::shared_ptr& path_factory, - int64_t schema_id, - const std::shared_ptr& value_schema, - const CoreOptions& options, - const std::shared_ptr& pool) - : pool_(pool), - arrow_pool_(GetArrowPool(pool)), - trimmed_primary_keys_(trimmed_primary_keys), - options_(options), - path_factory_(path_factory), - schema_id_(schema_id), - value_type_(arrow::struct_(value_schema->fields())), - metrics_(std::make_shared()) { +namespace { + +std::shared_ptr BuildPostponeBucketWriteSchema( + const std::shared_ptr& value_schema) { arrow::FieldVector target_fields; target_fields.push_back( DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber())); target_fields.push_back(DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind())); target_fields.insert(target_fields.end(), value_schema->fields().begin(), value_schema->fields().end()); - write_schema_ = arrow::schema(target_fields); + return arrow::schema(target_fields); +} + +} // namespace + +Result> PostponeBucketWriter::Create( + const std::vector& trimmed_primary_keys, + const std::shared_ptr& path_factory, int64_t schema_id, + const std::shared_ptr& value_schema, const CoreOptions& options, + const std::shared_ptr& pool) { + auto write_schema = BuildPostponeBucketWriteSchema(value_schema); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); + return std::unique_ptr( + new PostponeBucketWriter(trimmed_primary_keys, path_factory, schema_id, value_schema, + write_schema, options, pool, shredding_context)); } +PostponeBucketWriter::PostponeBucketWriter( + const std::vector& trimmed_primary_keys, + const std::shared_ptr& path_factory, int64_t schema_id, + const std::shared_ptr& value_schema, + const std::shared_ptr& write_schema, const CoreOptions& options, + const std::shared_ptr& pool, + const std::shared_ptr& shredding_context) + : pool_(pool), + arrow_pool_(GetArrowPool(pool)), + trimmed_primary_keys_(trimmed_primary_keys), + options_(options), + path_factory_(path_factory), + schema_id_(schema_id), + value_type_(arrow::struct_(value_schema->fields())), + write_schema_(write_schema), + shredding_context_(shredding_context), + metrics_(std::make_shared()) {} + Status PostponeBucketWriter::Write(std::unique_ptr&& moved_batch) { if (moved_batch->GetData()->length == 0) { return Status::OK(); @@ -240,26 +267,50 @@ PostponeBucketWriter::PrepareMinMaxKey( std::unique_ptr>> PostponeBucketWriter::CreateRollingRowWriter() const { - auto create_file_writer = [&]() + auto shredding_context = shredding_context_; + auto create_file_writer = [this, shredding_context]() -> Result>>> { + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, + MapSharedShreddingBatchConverter::CreateConverter( + write_schema_, shredding_context, pool_)); + std::shared_ptr file_schema = + bundle.physical_schema ? bundle.physical_schema : write_schema_; + ::ArrowSchema arrow_schema; ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*write_schema_, &arrow_schema)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); auto format = options_.GetWriteFileFormat(/*level=*/0); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr writer_builder, format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); writer_builder->WithMemoryPool(pool_); - auto converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { - ArrowArrayMove(key_value_batch.batch.get(), array); - return Status::OK(); - }; + std::function converter; + if (bundle.converter) { + auto shredding_converter = bundle.converter; + converter = [shredding_converter](KeyValueBatch key_value_batch, + ArrowArray* array) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, + shredding_converter->Convert(key_value_batch.batch.get())); + ArrowArrayMove(physical.get(), array); + return Status::OK(); + }; + } else { + converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { + ArrowArrayMove(key_value_batch.batch.get(), array); + return Status::OK(); + }; + } auto writer = std::make_unique( options_.GetWriteFileCompression(0), converter, schema_id_, /*level=*/0, - FileSource::Append(), trimmed_primary_keys_, /*stats_extractor=*/nullptr, write_schema_, + FileSource::Append(), trimmed_primary_keys_, /*stats_extractor=*/nullptr, file_schema, path_factory_->IsExternalPath(), pool_); PAIMON_RETURN_NOT_OK( writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder)); + if (bundle.converter) { + writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( + bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, + shredding_context, file_schema)); + } return writer; }; return std::make_unique>>( diff --git a/src/paimon/core/postpone/postpone_bucket_writer.h b/src/paimon/core/postpone/postpone_bucket_writer.h index 9da8f32c..78a43ad8 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer.h +++ b/src/paimon/core/postpone/postpone_bucket_writer.h @@ -46,15 +46,17 @@ struct ArrowArray; namespace paimon { class DataFilePathFactory; +class MapSharedShreddingContext; class MemoryPool; class Metrics; class PostponeBucketWriter : public BatchWriter { public: - PostponeBucketWriter(const std::vector& trimmed_primary_keys, - const std::shared_ptr& path_factory, - int64_t schema_id, const std::shared_ptr& value_schema, - const CoreOptions& options, const std::shared_ptr& pool); + static Result> Create( + const std::vector& trimmed_primary_keys, + const std::shared_ptr& path_factory, int64_t schema_id, + const std::shared_ptr& value_schema, const CoreOptions& options, + const std::shared_ptr& pool); ~PostponeBucketWriter() override { [[maybe_unused]] auto status = DoClose(); @@ -124,6 +126,13 @@ class PostponeBucketWriter : public BatchWriter { std::unique_ptr>> CreateRollingRowWriter() const; + PostponeBucketWriter(const std::vector& trimmed_primary_keys, + const std::shared_ptr& path_factory, + int64_t schema_id, const std::shared_ptr& value_schema, + const std::shared_ptr& write_schema, + const CoreOptions& options, const std::shared_ptr& pool, + const std::shared_ptr& shredding_context); + private: std::shared_ptr pool_; std::unique_ptr arrow_pool_; @@ -134,6 +143,7 @@ class PostponeBucketWriter : public BatchWriter { // write_schema = value_schema + special fields std::shared_ptr value_type_; std::shared_ptr write_schema_; + std::shared_ptr shredding_context_; std::shared_ptr metrics_; std::vector> new_files_; std::unique_ptr>> writer_; diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp index 4e2d211a..7679e67e 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp @@ -29,6 +29,8 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/data/data_define.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/fs/external_path_provider.h" #include "paimon/common/table/special_fields.h" @@ -110,6 +112,34 @@ class PostponeBucketWriterTest : public ::testing::Test, << expected_array->ToString(); } + void CheckShreddingFileSchema(const std::string& file_format_str, + const std::string& data_file_name, + const std::shared_ptr& expected_schema, + int32_t field_index, + const MapSharedShreddingFieldMeta& expected_meta) const { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(data_file_name)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr file_format, + FileFormatFactory::Get(file_format_str, {})); + ASSERT_OK_AND_ASSIGN(auto reader_builder, + file_format->CreateReaderBuilder(/*batch_size=*/10)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, reader_builder->Build(input_stream)); + auto c_file_schema = batch_reader->GetFileSchema().value(); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + ASSERT_TRUE(file_schema->Equals(*expected_schema, /*check_metadata=*/false)) + << "Expected schema:\n" + << expected_schema->ToString() << "\nActual schema:\n" + << file_schema->ToString(); + + auto metadata = file_schema->field(field_index)->metadata(); + ASSERT_NE(nullptr, metadata); + ASSERT_OK_AND_ASSIGN( + auto actual_meta, + MapSharedShreddingUtils::DeserializeMetadata( + metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression)); + ASSERT_EQ(expected_meta, actual_meta); + } + private: std::shared_ptr pool_; std::shared_ptr file_system_; @@ -145,8 +175,9 @@ TEST_P(PostponeBucketWriterTest, TestSimple) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - auto postpone_bucket_writer = std::make_shared( - primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_); + ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, + value_schema_, options, pool_)); // write batch std::shared_ptr array1 = @@ -221,9 +252,10 @@ TEST_P(PostponeBucketWriterTest, TestNestedType) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - auto postpone_bucket_writer = std::make_shared( - std::vector{"key"}, path_factory, /*schema_id=*/1, arrow::schema(fields), - options, pool_); + ASSERT_OK_AND_ASSIGN( + auto postpone_bucket_writer, + PostponeBucketWriter::Create(std::vector{"key"}, path_factory, /*schema_id=*/1, + arrow::schema(fields), options, pool_)); // write batch auto array1 = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -284,6 +316,73 @@ TEST_P(PostponeBucketWriterTest, TestNestedType) { ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement()); } +TEST_F(PostponeBucketWriterTest, TestSharedShreddingMap) { + const std::string file_format = "parquet"; + arrow::FieldVector fields = {arrow::field("key", arrow::utf8()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int32()))}; + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{Options::FILE_FORMAT, file_format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}})); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = std::make_shared(); + ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); + std::string uuid = path_factory->uuid_; + + ASSERT_OK_AND_ASSIGN( + auto postpone_bucket_writer, + PostponeBucketWriter::Create(std::vector{"key"}, path_factory, /*schema_id=*/1, + arrow::schema(fields), options, pool_)); + + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + ["Lucy", [["a", 1], ["b", 2]]], + ["Bob", [["c", 3], ["a", 4]]] + ])") + .ValueOrDie(); + ASSERT_TRUE(array); + WriteBatch(array, /*row_kinds=*/{}, postpone_bucket_writer.get()); + + ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, + postpone_bucket_writer->PrepareCommit(/*wait_compaction=*/false)); + ASSERT_OK(postpone_bucket_writer->Close()); + ASSERT_TRUE(commit_increment.GetCompactIncrement().IsEmpty()); + ASSERT_EQ(1, commit_increment.GetNewFilesIncrement().NewFiles().size()); + + std::string data_file_name = "data-" + uuid + "-0." + file_format; + std::string data_file_path = dir->Str() + "/" + data_file_name; + ASSERT_OK_AND_ASSIGN(std::unique_ptr data_file_status, + options.GetFileSystem()->GetFileStatus(data_file_path)); + ASSERT_GT(data_file_status->GetLen(), 0); + + arrow::FieldVector write_fields = {arrow::field("_SEQUENCE_NUMBER", arrow::int64()), + arrow::field("_VALUE_KIND", arrow::int8())}; + write_fields.insert(write_fields.end(), fields.begin(), fields.end()); + ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + arrow::schema(write_fields), {{"tags", 3}})); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0, 1}}, {1, {1}}, {2, {0}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 2; + + CheckShreddingFileSchema(file_format, data_file_path, expected_schema, /*field_index=*/3, + expected_meta); + + auto physical_type = arrow::struct_(expected_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [-1, 0, "Lucy", [[0, 1, -1], 1, 2, null, null]], + [-1, 0, "Bob", [[2, 0, -1], 3, 4, null, null]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(file_format, data_file_path, physical_type, expected_array); +} + TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) { auto file_format = GetParam(); ASSERT_OK_AND_ASSIGN(CoreOptions options, @@ -295,8 +394,9 @@ TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - auto postpone_bucket_writer = std::make_shared( - primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_); + ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, + value_schema_, options, pool_)); // write batch 1, batch size = 3 std::shared_ptr array1 = @@ -392,8 +492,9 @@ TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - auto postpone_bucket_writer = std::make_shared( - primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_); + ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, + value_schema_, options, pool_)); // write batch 1, batch size = 3 std::shared_ptr array1 = @@ -521,8 +622,9 @@ TEST_P(PostponeBucketWriterTest, TestPrepareCommitForEmptyData) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - auto postpone_bucket_writer = std::make_shared( - primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_); + ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, + value_schema_, options, pool_)); // prepare commit, without write ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, @@ -561,8 +663,9 @@ TEST_P(PostponeBucketWriterTest, TestCloseBeforePrepareCommit) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - auto postpone_bucket_writer = std::make_shared( - primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_); + ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, + value_schema_, options, pool_)); // write batch std::shared_ptr array1 = @@ -593,8 +696,10 @@ TEST_P(PostponeBucketWriterTest, TestIOException) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - auto postpone_bucket_writer = std::make_shared( - primary_keys_, path_factory, /*schema_id=*/1, value_schema_, options, pool_); + ASSERT_OK_AND_ASSIGN( + auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, + value_schema_, options, pool_)); // write batch std::shared_ptr array = diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index afabc2f9..4cb52f41 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -823,13 +823,14 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) { "Data evolution config must disabled with deletion-vectors.enabled"); } } + TEST(SchemaValidationTest, TestMapStorageLayout) { auto f0 = arrow::field("f0", arrow::utf8()); auto f1 = arrow::field("f1", arrow::int32()); auto f2 = arrow::field("f2", arrow::map(arrow::utf8(), arrow::int64())); auto f3 = arrow::field("f3", arrow::map(arrow::int32(), arrow::utf8())); - // Valid: extend on MAP column + // Valid: shared-shredding on MAP column { arrow::FieldVector fields = {f0, f1, f2}; auto schema = arrow::schema(fields); diff --git a/src/paimon/format/avro/avro_format_writer.cpp b/src/paimon/format/avro/avro_format_writer.cpp index e4ad454a..21905644 100644 --- a/src/paimon/format/avro/avro_format_writer.cpp +++ b/src/paimon/format/avro/avro_format_writer.cpp @@ -20,7 +20,9 @@ #include #include +#include #include +#include #include #include "arrow/api.h" @@ -111,6 +113,10 @@ Result AvroFormatWriter::ReachTargetSize(bool suggested_check, int64_t tar return false; } +Status AvroFormatWriter::AddMetadata(const std::map& /*metadata*/) { + return Status::NotImplemented("AddMetadata is not supported by avro format writer."); +} + Status AvroFormatWriter::AddBatch(ArrowArray* batch) { assert(batch); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, diff --git a/src/paimon/format/avro/avro_format_writer.h b/src/paimon/format/avro/avro_format_writer.h index 64427c31..23baffa9 100644 --- a/src/paimon/format/avro/avro_format_writer.h +++ b/src/paimon/format/avro/avro_format_writer.h @@ -20,8 +20,10 @@ #include #include +#include #include #include +#include #include "arrow/api.h" #include "avro/DataFile.hh" @@ -66,6 +68,8 @@ class AvroFormatWriter : public FormatWriter { return metrics_; } + Status AddMetadata(const std::map& metadata) override; + private: static constexpr size_t DEFAULT_SYNC_INTERVAL = 64 * 1024; diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index aa8dd248..6331f94e 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -19,6 +19,8 @@ #include "paimon/format/blob/blob_format_writer.h" #include +#include +#include #include "arrow/api.h" #include "arrow/c/bridge.h" @@ -230,6 +232,10 @@ Result BlobFormatWriter::ReachTargetSize(bool suggested_check, int64_t tar return current_pos >= target_size; } +Status BlobFormatWriter::AddMetadata(const std::map& /*metadata*/) { + return Status::NotImplemented("AddMetadata is not supported by blob format writer."); +} + template PAIMON_UNIQUE_PTR BlobFormatWriter::IntegerToLittleEndian( T value, const std::shared_ptr& pool) { diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index b586e0d4..0e734a44 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -20,7 +20,9 @@ #include #include +#include #include +#include #include #include @@ -73,6 +75,8 @@ class BlobFormatWriter : public FormatWriter { return metrics_; } + Status AddMetadata(const std::map& metadata) override; + private: BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, diff --git a/src/paimon/format/orc/orc_file_batch_reader.cpp b/src/paimon/format/orc/orc_file_batch_reader.cpp index 7836c06c..c96b4e5f 100644 --- a/src/paimon/format/orc/orc_file_batch_reader.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader.cpp @@ -26,9 +26,13 @@ #include #include "arrow/c/bridge.h" +#include "arrow/io/memory.h" +#include "arrow/ipc/api.h" +#include "arrow/util/base64.h" #include "fmt/format.h" #include "orc/OrcFile.hh" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" @@ -107,6 +111,21 @@ Result> OrcFileBatchReader::Create( Result> OrcFileBatchReader::GetFileSchema() const { assert(reader_); + + // If the writer stored a serialized Arrow schema with per-field metadata via + // AddMetadata, prefer that over the plain ORC type tree. + if (reader_->HasMetadataValue(ArrowUtils::kArrowSchemaMetadataKey)) { + std::string encoded = reader_->GetMetadataValue(ArrowUtils::kArrowSchemaMetadataKey); + std::string decoded = arrow::util::base64_decode(encoded); + auto buffer = arrow::Buffer::FromString(std::move(decoded)); + arrow::io::BufferReader buf_reader(buffer); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ipc::ReadSchema(&buf_reader, nullptr)); + auto c_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, c_schema.get())); + return c_schema; + } + const auto& orc_file_type = reader_->GetOrcType(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_file_type, OrcAdapter::GetArrowType(&orc_file_type)); diff --git a/src/paimon/format/orc/orc_file_batch_reader_test.cpp b/src/paimon/format/orc/orc_file_batch_reader_test.cpp index ccb79699..b1655850 100644 --- a/src/paimon/format/orc/orc_file_batch_reader_test.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader_test.cpp @@ -25,10 +25,12 @@ #include #include +#include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/ipc/api.h" #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/defs.h" #include "paimon/format/orc/orc_adapter.h" #include "paimon/format/orc/orc_format_defs.h" @@ -45,6 +47,12 @@ namespace paimon::orc::test { +std::string SerializeSchemaToString(const std::shared_ptr& schema) { + std::shared_ptr serialized = arrow::ipc::SerializeSchema(*schema).ValueOrDie(); + return std::string(reinterpret_cast(serialized->data()), + static_cast(serialized->size())); +} + struct TestParam { uint64_t natural_read_size; bool enable_tz; @@ -1133,4 +1141,89 @@ TEST_F(OrcFileBatchReaderTest, TestListStructPartialProjection) { "type mismatch"); } +TEST_F(OrcFileBatchReaderTest, TestAddMetadataPerFieldMetadata) { + // Write a simple ORC file, call AddMetadata to inject per-field metadata + // before Finish, then read back and verify the file schema carries the metadata. + auto write_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::float64()), + }); + + auto dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto fs = dir->GetFileSystem(); + std::string file_path = dir->Str() + "/update_schema_test.orc"; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs->Create(file_path, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(auto orc_output_stream, OrcOutputStreamImpl::Create(out)); + ASSERT_OK_AND_ASSIGN(auto format_writer, + OrcFormatWriter::Create(std::move(orc_output_stream), *write_schema, + /*options=*/{}, "zstd", + /*batch_size=*/10, pool_)); + + // Write one batch of data. + auto data = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(write_schema->fields()), + R"([[1, "alice", 95.5], [2, "bob", 88.0], [3, "charlie", 72.3]])") + .ValueOrDie(); + ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*data, &c_array).ok()); + ASSERT_OK(format_writer->AddBatch(&c_array)); + ASSERT_OK(format_writer->Flush()); + + // Build an updated schema with per-field metadata on "name" and "score". + auto name_meta = std::make_shared(); + name_meta->Append("shredding.field_mapping", "0:alice,1:bob,2:charlie"); + name_meta->Append("shredding.num_columns", "3"); + auto score_meta = std::make_shared(); + score_meta->Append("custom.unit", "percent"); + + auto updated_schema = arrow::schema({ + write_schema->field(0), // id — no metadata + write_schema->field(1)->WithMetadata(name_meta), // name — shredding metadata + write_schema->field(2)->WithMetadata(score_meta), // score — custom metadata + }); + + // AddMetadata must be called before Finish. + ASSERT_OK(format_writer->AddMetadata( + {{ArrowUtils::kArrowSchemaMetadataKey, SerializeSchemaToString(updated_schema)}})); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Flush()); + ASSERT_OK(out->Close()); + + // Read back: GetFileSchema should reflect the updated per-field metadata. + auto orc_batch_reader = PrepareOrcFileBatchReader(file_path, write_schema.get(), batch_size_, + DEFAULT_NATURAL_READ_SIZE); + + ASSERT_OK_AND_ASSIGN(auto c_file_schema, orc_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + + // Field 0 "id": no metadata. + ASSERT_EQ("id", file_schema->field(0)->name()); + + // Field 1 "name": should have the shredding metadata we set. + ASSERT_EQ("name", file_schema->field(1)->name()); + auto read_name_meta = file_schema->field(1)->metadata(); + ASSERT_NE(nullptr, read_name_meta); + auto field_mapping_val = read_name_meta->Get("shredding.field_mapping").ValueOrDie(); + ASSERT_EQ("0:alice,1:bob,2:charlie", field_mapping_val); + auto num_columns_val = read_name_meta->Get("shredding.num_columns").ValueOrDie(); + ASSERT_EQ("3", num_columns_val); + + // Field 2 "score": should have the custom metadata. + ASSERT_EQ("score", file_schema->field(2)->name()); + auto read_score_meta = file_schema->field(2)->metadata(); + ASSERT_NE(nullptr, read_score_meta); + auto unit_val = read_score_meta->Get("custom.unit").ValueOrDie(); + ASSERT_EQ("percent", unit_val); + + // Also verify data integrity — read it back and compare content. + ASSERT_OK_AND_ASSIGN(auto result_array, + paimon::test::ReadResultCollector::CollectResult(orc_batch_reader.get())); + ASSERT_EQ(result_array->num_chunks(), 1); + ASSERT_TRUE(data->Equals(*result_array->chunk(0))) << result_array->ToString(); +} + } // namespace paimon::orc::test diff --git a/src/paimon/format/orc/orc_format_writer.cpp b/src/paimon/format/orc/orc_format_writer.cpp index ff6cd8a7..2be84b7d 100644 --- a/src/paimon/format/orc/orc_format_writer.cpp +++ b/src/paimon/format/orc/orc_format_writer.cpp @@ -22,12 +22,16 @@ #include #include #include +#include #include +#include +#include #include #include "arrow/api.h" #include "arrow/array/array_base.h" #include "arrow/c/bridge.h" +#include "arrow/util/base64.h" #include "fmt/format.h" #include "orc/Common.hh" #include "orc/OrcFile.hh" @@ -208,6 +212,26 @@ std::shared_ptr OrcFormatWriter::GetWriterMetrics() const { return metrics_; } +Status OrcFormatWriter::AddMetadata(const std::map& metadata) { + if (metadata.empty()) { + return Status::OK(); + } + try { + for (const auto& [key, value] : metadata) { + writer_->addUserMetadata(key, arrow::util::base64_encode(std::string_view(value))); + } + } catch (const std::exception& e) { + return Status::Invalid( + fmt::format("orc format writer AddMetadata failed for file {}, with {} error", + output_stream_->getName(), e.what())); + } catch (...) { + return Status::UnknownError( + fmt::format("orc format writer AddMetadata failed for file {}, with unknown error", + output_stream_->getName())); + } + return Status::OK(); +} + namespace { Result GetMemorySizeOption(const std::map& options, diff --git a/src/paimon/format/orc/orc_format_writer.h b/src/paimon/format/orc/orc_format_writer.h index 409e7a87..1a646424 100644 --- a/src/paimon/format/orc/orc_format_writer.h +++ b/src/paimon/format/orc/orc_format_writer.h @@ -66,6 +66,8 @@ class OrcFormatWriter : public FormatWriter { std::shared_ptr GetWriterMetrics() const override; + Status AddMetadata(const std::map& metadata) override; + private: OrcFormatWriter(const std::shared_ptr& orc_memory_pool, std::unique_ptr<::orc::OutputStream>&& output_stream, diff --git a/src/paimon/format/orc/orc_format_writer_test.cpp b/src/paimon/format/orc/orc_format_writer_test.cpp index 5026e957..a278b7e0 100644 --- a/src/paimon/format/orc/orc_format_writer_test.cpp +++ b/src/paimon/format/orc/orc_format_writer_test.cpp @@ -29,6 +29,8 @@ #include "arrow/array/builder_primitive.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "arrow/ipc/api.h" #include "gtest/gtest.h" #include "orc/Common.hh" #include "orc/MemoryPool.hh" diff --git a/src/paimon/format/orc/orc_reader_wrapper.h b/src/paimon/format/orc/orc_reader_wrapper.h index e98ad3de..f01ccd5a 100644 --- a/src/paimon/format/orc/orc_reader_wrapper.h +++ b/src/paimon/format/orc/orc_reader_wrapper.h @@ -87,6 +87,14 @@ class OrcReaderWrapper { return reader_->getType(); } + bool HasMetadataValue(const std::string& key) const { + return reader_->hasMetadataValue(key); + } + + std::string GetMetadataValue(const std::string& key) const { + return reader_->getMetadataValue(key); + } + Result>> GenReadRanges( std::vector target_column_ids, uint64_t begin_row_num, uint64_t end_row_num, bool* need_prefetch) const { diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index d324a54b..33107779 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include "arrow/api.h" #include "arrow/array/array_base.h" @@ -30,10 +31,12 @@ #include "arrow/c/bridge.h" #include "arrow/io/caching.h" #include "arrow/io/interfaces.h" +#include "arrow/ipc/api.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" @@ -58,6 +61,12 @@ class Predicate; namespace paimon::parquet::test { +std::string SerializeSchemaToString(const std::shared_ptr& schema) { + std::shared_ptr serialized = arrow::ipc::SerializeSchema(*schema).ValueOrDie(); + return std::string(reinterpret_cast(serialized->data()), + static_cast(serialized->size())); +} + class ParquetFileBatchReaderTest : public ::testing::Test, public ::testing::WithParamInterface { public: @@ -685,4 +694,88 @@ TEST_P(ParquetFileBatchReaderTest, TestTimestampType) { INSTANTIATE_TEST_SUITE_P(TestParam, ParquetFileBatchReaderTest, ::testing::Values(false, true)); +TEST_F(ParquetFileBatchReaderTest, TestAddMetadataPerFieldMetadata) { + // Write a simple parquet file, call AddMetadata to inject per-field metadata + // before Finish, then read back and verify the file schema carries the metadata. + auto write_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::float64()), + }); + + std::string file_path = PathUtil::JoinPath(dir_->Str(), "update_schema_test.parquet"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_path, /*overwrite=*/true)); + + ::parquet::WriterProperties::Builder builder; + builder.write_batch_size(10); + auto writer_properties = builder.build(); + ASSERT_OK_AND_ASSIGN(auto format_writer, + ParquetFormatWriter::Create(out, write_schema, writer_properties, + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, pool_)); + + // Write one batch of data. + auto data = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(write_schema->fields()), + R"([[1, "alice", 95.5], [2, "bob", 88.0], [3, "charlie", 72.3]])") + .ValueOrDie(); + ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*data, &c_array).ok()); + ASSERT_OK(format_writer->AddBatch(&c_array)); + ASSERT_OK(format_writer->Flush()); + + // Build an updated schema with per-field metadata on "name" and "score". + auto name_meta = std::make_shared(); + name_meta->Append("shredding.field_mapping", "0:alice,1:bob,2:charlie"); + name_meta->Append("shredding.num_columns", "3"); + auto score_meta = std::make_shared(); + score_meta->Append("custom.unit", "percent"); + + auto updated_schema = arrow::schema({ + write_schema->field(0), // id — no metadata + write_schema->field(1)->WithMetadata(name_meta), // name — shredding metadata + write_schema->field(2)->WithMetadata(score_meta), // score — custom metadata + }); + + // AddMetadata must be called before Finish. + ASSERT_OK(format_writer->AddMetadata( + {{ArrowUtils::kArrowSchemaMetadataKey, SerializeSchemaToString(updated_schema)}})); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Flush()); + ASSERT_OK(out->Close()); + + // Read back: GetFileSchema should reflect the updated per-field metadata. + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path, write_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, batch_size_); + + ASSERT_OK_AND_ASSIGN(auto c_file_schema, parquet_batch_reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + + // Field 0 "id": no metadata. + ASSERT_EQ("id", file_schema->field(0)->name()); + + // Field 1 "name": should have the shredding metadata we set. + ASSERT_EQ("name", file_schema->field(1)->name()); + auto read_name_meta = file_schema->field(1)->metadata(); + ASSERT_NE(nullptr, read_name_meta); + auto field_mapping_val = read_name_meta->Get("shredding.field_mapping").ValueOrDie(); + ASSERT_EQ("0:alice,1:bob,2:charlie", field_mapping_val); + auto num_columns_val = read_name_meta->Get("shredding.num_columns").ValueOrDie(); + ASSERT_EQ("3", num_columns_val); + + // Field 2 "score": should have the custom metadata. + ASSERT_EQ("score", file_schema->field(2)->name()); + auto read_score_meta = file_schema->field(2)->metadata(); + ASSERT_NE(nullptr, read_score_meta); + auto unit_val = read_score_meta->Get("custom.unit").ValueOrDie(); + ASSERT_EQ("percent", unit_val); + + // Also verify data integrity — read it back and compare content. + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + ASSERT_EQ(result_array->num_chunks(), 1); + ASSERT_TRUE(data->Equals(*result_array->chunk(0))) << result_array->ToString(); +} + } // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_format_writer.cpp b/src/paimon/format/parquet/parquet_format_writer.cpp index 20079486..0a8e38b4 100644 --- a/src/paimon/format/parquet/parquet_format_writer.cpp +++ b/src/paimon/format/parquet/parquet_format_writer.cpp @@ -18,12 +18,16 @@ #include "paimon/format/parquet/parquet_format_writer.h" +#include #include +#include #include #include "arrow/c/bridge.h" #include "arrow/memory_pool.h" #include "arrow/record_batch.h" +#include "arrow/util/base64.h" +#include "arrow/util/key_value_metadata.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/arrow_output_stream_adapter.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -81,6 +85,18 @@ Status ParquetFormatWriter::Finish() { return Status::OK(); } +Status ParquetFormatWriter::AddMetadata(const std::map& metadata) { + if (metadata.empty()) { + return Status::OK(); + } + auto key_value_metadata = std::make_shared(); + for (const auto& [key, value] : metadata) { + key_value_metadata->Append(key, arrow::util::base64_encode(std::string_view(value))); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(writer_->AddKeyValueMetadata(key_value_metadata)); + return Status::OK(); +} + Result ParquetFormatWriter::ReachTargetSize(bool suggested_check, int64_t target_size) const { if (suggested_check) { PAIMON_ASSIGN_OR_RAISE(const uint64_t length, GetEstimateLength()); diff --git a/src/paimon/format/parquet/parquet_format_writer.h b/src/paimon/format/parquet/parquet_format_writer.h index e5cae163..4ab58d73 100644 --- a/src/paimon/format/parquet/parquet_format_writer.h +++ b/src/paimon/format/parquet/parquet_format_writer.h @@ -19,7 +19,9 @@ #pragma once #include +#include #include +#include #include "paimon/format/format_writer.h" #include "paimon/fs/file_system.h" @@ -64,6 +66,8 @@ class ParquetFormatWriter : public FormatWriter { return metrics_; } + Status AddMetadata(const std::map& metadata) override; + private: ParquetFormatWriter(std::unique_ptr<::parquet::arrow::FileWriter> writer, const std::shared_ptr& out, diff --git a/src/paimon/format/parquet/parquet_format_writer_test.cpp b/src/paimon/format/parquet/parquet_format_writer_test.cpp index ccd08680..f70c6332 100644 --- a/src/paimon/format/parquet/parquet_format_writer_test.cpp +++ b/src/paimon/format/parquet/parquet_format_writer_test.cpp @@ -31,6 +31,7 @@ #include "arrow/array/builder_primitive.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" #include "arrow/io/file.h" #include "arrow/ipc/api.h" #include "arrow/memory_pool.h" diff --git a/src/paimon/testing/mock/mock_format_writer.cpp b/src/paimon/testing/mock/mock_format_writer.cpp index 61f7482b..f763e437 100644 --- a/src/paimon/testing/mock/mock_format_writer.cpp +++ b/src/paimon/testing/mock/mock_format_writer.cpp @@ -18,6 +18,7 @@ #include "paimon/testing/mock/mock_format_writer.h" +#include #include #include @@ -60,4 +61,8 @@ Result MockFormatWriter::ReachTargetSize(bool suggested_check, int64_t tar return false; } +Status MockFormatWriter::AddMetadata(const std::map& /*metadata*/) { + return Status::NotImplemented("AddMetadata is not supported by mock format writer."); +} + } // namespace paimon::test diff --git a/src/paimon/testing/mock/mock_format_writer.h b/src/paimon/testing/mock/mock_format_writer.h index e395e142..a97a7593 100644 --- a/src/paimon/testing/mock/mock_format_writer.h +++ b/src/paimon/testing/mock/mock_format_writer.h @@ -19,7 +19,9 @@ #pragma once #include +#include #include +#include #include "paimon/format/format_writer.h" #include "paimon/result.h" @@ -46,6 +48,7 @@ class MockFormatWriter : public FormatWriter { std::shared_ptr GetWriterMetrics() const override { return nullptr; } + Status AddMetadata(const std::map& metadata) override; private: int64_t counter_ = 0; From 4c3d877c3f261cc3cac51094c3e36d3c11242451 Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Fri, 19 Jun 2026 09:55:17 +0800 Subject: [PATCH 064/138] fix(parquet): fallback nested field reading to RowGroup reading (walkaround) --- .../page_filtered_row_group_reader_test.cpp | 270 ++++++++++++++++++ .../parquet/parquet_file_batch_reader.cpp | 12 +- 2 files changed, 281 insertions(+), 1 deletion(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index e9aff95e..5c1cb89c 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -876,5 +876,275 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding) auto partial_concat = arrow::Concatenate(result_partial->chunks()).ValueOrDie(); ASSERT_TRUE(partial_concat->Equals(expected_struct)); } +/// Helper: build a StructArray with a top-level int32 "id" column and a nested struct column +/// "info" containing two int32 fields: "x" and "y". +/// id[i] = i, info.x[i] = i * 100, info.y[i] = i * 100 + 1, for i in [0, N). +/// +/// Arrow schema: { id: int32, info: struct } +/// Parquet leaf columns: [id (index 0), info.x (index 1), info.y (index 2)] +static std::shared_ptr MakeNestedStructData(int32_t num_rows) { + arrow::Int32Builder id_builder, x_builder, y_builder; + EXPECT_TRUE(id_builder.Reserve(num_rows).ok()); + EXPECT_TRUE(x_builder.Reserve(num_rows).ok()); + EXPECT_TRUE(y_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + id_builder.UnsafeAppend(i); + x_builder.UnsafeAppend(i * 100); + y_builder.UnsafeAppend(i * 100 + 1); + } + auto id_array = id_builder.Finish().ValueOrDie(); + auto x_array = x_builder.Finish().ValueOrDie(); + auto y_array = y_builder.Finish().ValueOrDie(); + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto inner_struct = + arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie(); + + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + return arrow::StructArray::Make({id_array, inner_struct}, {field_id, field_info}).ValueOrDie(); +} + +/// Test: rowgroup-level filtering on a file with nested struct columns. +/// +/// This test exposes the bug where BuildPageFilteredSchema fails to correctly map +/// Parquet leaf column indices to Arrow fields for nested types, and +/// ReadFilteredRowGroup cannot correctly assemble nested column results. +/// +/// Schema: { id: int32, info: struct } +/// Parquet leaf columns: [id=0, info.x=1, info.y=2] +/// 100 rows, 10 per page, 2 row groups. +/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. +/// The read schema requests both "id" and "info" columns. +TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { + std::string file_name = dir_->Str() + "/nested_struct_filter.parquet"; + auto data = MakeNestedStructData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("info", arrow::struct_({field_x, field_y}))}); + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + // Should get rows 50-99 = 50 rows + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + // Build expected result: rows 50-99 from the original data + auto expected = data->Slice(50, 50); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} + +/// Test: Page-level filtering reading the nested struct column along with the predicate column. +/// +/// This verifies that when reading a subset of columns that includes a nested column +/// and the predicate column, the schema mapping and column assembly work correctly. +/// +/// Schema: { id: int32, info: struct } +/// Read schema: { id: int32, info: struct } +/// Predicate on "id": id >= 70. +TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) { + std::string file_name = dir_->Str() + "/nested_struct_only_nested.parquet"; + auto data = MakeNestedStructData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto field_id = arrow::field("id", arrow::int32()); + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + // Read "id" column only + auto read_schema = arrow::schema({field_id}); + + // Predicate is on "id" (field_index=0 in file schema) + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + // Should get rows 70-99 = 30 rows + ASSERT_TRUE(result); + ASSERT_EQ(30, result->length()); + + auto result_struct = std::dynamic_pointer_cast(result->chunk(0)); + ASSERT_TRUE(result_struct); + ASSERT_TRUE(data->field(0)->Slice(70, 30)->Equals(result_struct->field(0))); +} + +/// Helper: build a StructArray with an int32 "id" column and a list "tags" column. +/// id[i] = i, tags[i] = [i*10, i*10+1], for i in [0, N). +/// +/// Arrow schema: { id: int32, tags: list } +/// Parquet leaf columns: [id (index 0), tags.item (index 1)] +static std::shared_ptr MakeListColumnData(int32_t num_rows) { + arrow::Int32Builder id_builder; + EXPECT_TRUE(id_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + id_builder.UnsafeAppend(i); + } + auto id_array = id_builder.Finish().ValueOrDie(); + + auto value_builder = std::make_shared(); + arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder); + for (int32_t i = 0; i < num_rows; ++i) { + EXPECT_TRUE(list_builder.Append().ok()); + EXPECT_TRUE(value_builder->Append(i * 10).ok()); + EXPECT_TRUE(value_builder->Append(i * 10 + 1).ok()); + } + auto list_array = list_builder.Finish().ValueOrDie(); + + auto field_id = arrow::field("id", arrow::int32()); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + return arrow::StructArray::Make({id_array, list_array}, {field_id, field_tags}).ValueOrDie(); +} + +/// Helper: build a StructArray with an int32 "id" column and a map "props" column. +/// id[i] = i, props[i] = {"k_i": i * 100}, for i in [0, N). +/// +/// Arrow schema: { id: int32, props: map } +/// Parquet leaf columns: [id (index 0), props.key (index 1), props.value (index 2)] +static std::shared_ptr MakeMapColumnData(int32_t num_rows) { + arrow::Int32Builder id_builder; + EXPECT_TRUE(id_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + id_builder.UnsafeAppend(i); + } + auto id_array = id_builder.Finish().ValueOrDie(); + + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, value_builder); + for (int32_t i = 0; i < num_rows; ++i) { + EXPECT_TRUE(map_builder.Append().ok()); + std::string key = "k_" + std::to_string(i); + EXPECT_TRUE(key_builder->Append(key).ok()); + EXPECT_TRUE(value_builder->Append(i * 100).ok()); + } + auto map_array = map_builder.Finish().ValueOrDie(); + + auto field_id = arrow::field("id", arrow::int32()); + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + return arrow::StructArray::Make({id_array, map_array}, {field_id, field_props}).ValueOrDie(); +} + +/// Test: rowgroup-level filtering on a file with a list column. +/// +/// Schema: { id: int32, tags: list } +/// 100 rows, 10 per page, 2 row groups. +/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. +TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) { + std::string file_name = dir_->Str() + "/nested_list_filter.parquet"; + auto data = MakeListColumnData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = + arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())))}); + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + // Build expected result: rows 50-99 from the original data + auto expected = data->Slice(50, 50); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} + +/// Test: rowgroup filtering on a file with a map column. +/// +/// Schema: { id: int32, props: map } +/// 100 rows, 10 per page, 2 row groups. +/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. +TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { + std::string file_name = dir_->Str() + "/nested_map_filter.parquet"; + auto data = MakeMapColumnData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = + arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))}); + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + // Build expected result: rows 50-99 from the original data + auto expected = data->Slice(50, 50); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} + +/// Test: rowgroup-level filtering with multiple adjacent nested columns (struct + list). +/// +/// Schema: { id: int32, info: struct, tags: list } +/// This tests the boundary handling when two nested fields are adjacent in the schema. +/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected. +TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) { + std::string file_name = dir_->Str() + "/multi_nested.parquet"; + + // Build data with id, info (struct), tags (list) + arrow::Int32Builder id_builder, x_builder, y_builder; + ASSERT_TRUE(id_builder.Reserve(100).ok()); + ASSERT_TRUE(x_builder.Reserve(100).ok()); + ASSERT_TRUE(y_builder.Reserve(100).ok()); + auto value_builder = std::make_shared(); + arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder); + + for (int32_t i = 0; i < 100; ++i) { + id_builder.UnsafeAppend(i); + x_builder.UnsafeAppend(i * 100); + y_builder.UnsafeAppend(i * 100 + 1); + ASSERT_TRUE(list_builder.Append().ok()); + ASSERT_TRUE(value_builder->Append(i * 10).ok()); + } + auto id_array = id_builder.Finish().ValueOrDie(); + auto x_array = x_builder.Finish().ValueOrDie(); + auto y_array = y_builder.Finish().ValueOrDie(); + auto list_array = list_builder.Finish().ValueOrDie(); + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto inner_struct = + arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie(); + + auto field_id = arrow::field("id", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + auto data = arrow::StructArray::Make({id_array, inner_struct, list_array}, + {field_id, field_info, field_tags}) + .ValueOrDie(); + + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto read_schema = arrow::schema({field_id, field_info, field_tags}); + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + // Build expected result: rows 50-99 from the original data + auto expected = data->Slice(50, 50); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} } // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 241e65c6..f1223706 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -41,6 +41,7 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/format/parquet/parquet_field_id_converter.h" #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_timestamp_converter.h" @@ -131,6 +132,13 @@ Status ParquetFileBatchReader::SetReadSchema( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); std::unordered_map> field_index_map; + bool has_nested_field = false; + for (const auto& field : read_schema->fields()) { + if (ArrowSchemaValidator::IsNestedType(field->type())) { + has_nested_field = true; + break; + } + } int32_t i = 0; for (const auto& field : file_schema->fields()) { std::vector v; @@ -168,7 +176,9 @@ Status ParquetFileBatchReader::SetReadSchema( bool enable_page_index_filter, OptionsUtils::GetValueFromMap(options_, PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER)); - if (enable_page_index_filter) { + // walkaround: page index filter does not support nested fields for now, skip page index + // filter if there is any nested field in the schema + if (enable_page_index_filter && !has_nested_field) { // Build column name to index map for page-level filtering. // For leaf columns, indices[0] is the correct leaf column index in Parquet. // For nested types (struct/list/map), FlattenSchema produces multiple leaf indices, From f996e63fa456071ca894224d7458522733ec5559 Mon Sep 17 00:00:00 2001 From: spaces-x Date: Sun, 21 Jun 2026 09:34:07 +0800 Subject: [PATCH 065/138] feat(tantivy): add Tantivy full-text global index via Rust FFI --- .devcontainer/Dockerfile.template | 37 +- .devcontainer/centos7/Dockerfile | 241 +++ .devcontainer/centos7/run.sh | 151 ++ .devcontainer/devcontainer.json.template | 30 +- .../x86_64/devcontainer.json.template | 76 + .github/workflows/build_release.yaml | 77 - .github/workflows/clang_test.yaml | 58 - .github/workflows/gcc_test.yaml | 56 - .github/workflows/test_with_sanitizer.yaml | 56 - .gitignore | 18 +- CMakeLists.txt | 30 +- ci/scripts/build_paimon.sh | 3 + ci/scripts/setup_rust.sh | 58 + cmake_modules/BuildUtils.cmake | 10 + cmake_modules/CorrosionFetch.cmake | 87 + cmake_modules/ThirdpartyToolchain.cmake | 9 + examples/CMakeLists.txt | 2 +- include/paimon/predicate/full_text_search.h | 36 +- scripts/tantivy_smoke.sh | 83 + src/paimon/common/data/binary_row_test.cpp | 5 +- .../offset_global_index_reader_test.cpp | 37 + .../lucene/lucene_global_index_reader.cpp | 8 + .../lucene/lucene_global_index_test.cpp | 12 + .../global_index/tantivy/CMakeLists.txt | 268 +++ .../tantivy/tantivy_archive_layout.cpp | 93 + .../tantivy/tantivy_archive_layout.h | 61 + .../global_index/tantivy/tantivy_defs.cpp | 37 + .../global_index/tantivy/tantivy_defs.h | 93 + .../tantivy/tantivy_equivalence_test.cpp | 383 ++++ .../global_index/tantivy/tantivy_ffi_handle.h | 113 + .../global_index/tantivy/tantivy_ffi_log.cpp | 70 + .../global_index/tantivy/tantivy_ffi_log.h | 31 + .../global_index/tantivy/tantivy_ffi_status.h | 92 + .../global_index/tantivy/tantivy_ffi_test.cpp | 138 ++ .../tantivy/tantivy_filter_limit_test.cpp | 381 ++++ .../tantivy/tantivy_global_index.cpp | 80 + .../tantivy/tantivy_global_index.h | 55 + .../tantivy/tantivy_global_index_factory.cpp | 45 + .../tantivy/tantivy_global_index_factory.h | 48 + .../tantivy/tantivy_global_index_reader.cpp | 232 ++ .../tantivy/tantivy_global_index_reader.h | 130 ++ .../tantivy/tantivy_global_index_writer.cpp | 172 ++ .../tantivy/tantivy_global_index_writer.h | 76 + .../tantivy/tantivy_index_test.cpp | 298 +++ .../tantivy/tantivy_java_compat_test.cpp | 450 ++++ .../tantivy/tantivy_lucene_coexist_test.cpp | 275 +++ .../tantivy/tantivy_smoke_test.cpp | 51 + .../tantivy/tantivy_stream_ctx.cpp | 90 + .../global_index/tantivy/tantivy_stream_ctx.h | 74 + .../tantivy/tantivy_streaming_test.cpp | 306 +++ .../tantivy/tantivy_tokenizer_test.cpp | 127 ++ .../tantivy/tantivy_writer_test.cpp | 263 +++ src/paimon/testing/utils/CMakeLists.txt | 10 + .../english_default.archive | Bin 0 -> 6597 bytes .../test_data/java_tantivy_fixtures/README.md | 51 + .../english_simple.archive | Bin 0 -> 6044 bytes .../english_simple.golden.json | 25 + .../production_sample.archive | Bin 0 -> 5176 bytes test/test_data/tokenizer_golden/README.md | 29 + .../tokenizer_golden/golden_corpus.txt | 20 + .../tokenizer_golden/golden_synthetic.txt | 38 + .../tokenizer_golden/known_diffs.txt | 18 + third_party/tantivy_ffi/Cargo.lock | 1859 +++++++++++++++++ third_party/tantivy_ffi/Cargo.toml | 34 + third_party/tantivy_ffi/build.rs | 40 + third_party/tantivy_ffi/cbindgen.toml | 66 + third_party/tantivy_ffi/rust-toolchain.toml | 11 + third_party/tantivy_ffi/src/buffer.rs | 111 + .../tantivy_ffi/src/callback_directory.rs | 515 +++++ third_party/tantivy_ffi/src/error.rs | 137 ++ third_party/tantivy_ffi/src/handle.rs | 106 + third_party/tantivy_ffi/src/lib.rs | 79 + third_party/tantivy_ffi/src/log_bridge.rs | 103 + third_party/tantivy_ffi/src/reader.rs | 1298 ++++++++++++ third_party/tantivy_ffi/src/tokenizer.rs | 447 ++++ third_party/tantivy_ffi/src/writer.rs | 773 +++++++ 76 files changed, 11219 insertions(+), 263 deletions(-) create mode 100644 .devcontainer/centos7/Dockerfile create mode 100755 .devcontainer/centos7/run.sh create mode 100644 .devcontainer/x86_64/devcontainer.json.template delete mode 100644 .github/workflows/build_release.yaml delete mode 100644 .github/workflows/clang_test.yaml delete mode 100644 .github/workflows/gcc_test.yaml delete mode 100644 .github/workflows/test_with_sanitizer.yaml create mode 100755 ci/scripts/setup_rust.sh create mode 100644 cmake_modules/CorrosionFetch.cmake create mode 100755 scripts/tantivy_smoke.sh create mode 100644 src/paimon/global_index/tantivy/CMakeLists.txt create mode 100644 src/paimon/global_index/tantivy/tantivy_archive_layout.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_archive_layout.h create mode 100644 src/paimon/global_index/tantivy/tantivy_defs.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_defs.h create mode 100644 src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_ffi_handle.h create mode 100644 src/paimon/global_index/tantivy/tantivy_ffi_log.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_ffi_log.h create mode 100644 src/paimon/global_index/tantivy/tantivy_ffi_status.h create mode 100644 src/paimon/global_index/tantivy/tantivy_ffi_test.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_filter_limit_test.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_global_index.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_global_index.h create mode 100644 src/paimon/global_index/tantivy/tantivy_global_index_factory.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_global_index_factory.h create mode 100644 src/paimon/global_index/tantivy/tantivy_global_index_reader.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_global_index_reader.h create mode 100644 src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_global_index_writer.h create mode 100644 src/paimon/global_index/tantivy/tantivy_index_test.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_lucene_coexist_test.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_smoke_test.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_stream_ctx.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_stream_ctx.h create mode 100644 src/paimon/global_index/tantivy/tantivy_streaming_test.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_tokenizer_test.cpp create mode 100644 src/paimon/global_index/tantivy/tantivy_writer_test.cpp create mode 100644 test/test_data/cpp_tantivy_fixtures/english_default.archive create mode 100644 test/test_data/java_tantivy_fixtures/README.md create mode 100644 test/test_data/java_tantivy_fixtures/english_simple.archive create mode 100644 test/test_data/java_tantivy_fixtures/english_simple.golden.json create mode 100644 test/test_data/java_tantivy_fixtures/production_sample.archive create mode 100644 test/test_data/tokenizer_golden/README.md create mode 100644 test/test_data/tokenizer_golden/golden_corpus.txt create mode 100644 test/test_data/tokenizer_golden/golden_synthetic.txt create mode 100644 test/test_data/tokenizer_golden/known_diffs.txt create mode 100644 third_party/tantivy_ffi/Cargo.lock create mode 100644 third_party/tantivy_ffi/Cargo.toml create mode 100644 third_party/tantivy_ffi/build.rs create mode 100644 third_party/tantivy_ffi/cbindgen.toml create mode 100644 third_party/tantivy_ffi/rust-toolchain.toml create mode 100644 third_party/tantivy_ffi/src/buffer.rs create mode 100644 third_party/tantivy_ffi/src/callback_directory.rs create mode 100644 third_party/tantivy_ffi/src/error.rs create mode 100644 third_party/tantivy_ffi/src/handle.rs create mode 100644 third_party/tantivy_ffi/src/lib.rs create mode 100644 third_party/tantivy_ffi/src/log_bridge.rs create mode 100644 third_party/tantivy_ffi/src/reader.rs create mode 100644 third_party/tantivy_ffi/src/tokenizer.rs create mode 100644 third_party/tantivy_ffi/src/writer.rs diff --git a/.devcontainer/Dockerfile.template b/.devcontainer/Dockerfile.template index ebc256e6..00f1ee15 100644 --- a/.devcontainer/Dockerfile.template +++ b/.devcontainer/Dockerfile.template @@ -17,12 +17,35 @@ # Adapted from Apache Iceberg C++ # https://github.com/apache/iceberg-cpp/blob/v0.2.0/.devcontainer/Dockerfile.template - +# # This Dockerfile is used to build a development container for Paimon C++. -# It is based on the Ubuntu image and installs necessary dependencies. +# Base: Ubuntu 24.04. Rust toolchain is installed via Dev Container +# Feature `ghcr.io/devcontainers/features/rust:1` (see devcontainer.json), +# so it does NOT appear in this Dockerfile. FROM ubuntu:24.04 +# Optional apt mirror. Defaults to the upstream Ubuntu mirrors so builds work +# everywhere; pass --build-arg APT_MIRROR= (e.g. http://mirrors.aliyun.com) +# to use a regional mirror for faster downloads inside mainland China. +ARG APT_MIRROR= +RUN if [ -n "${APT_MIRROR}" ]; then \ + sed -i \ + -e "s|http://archive.ubuntu.com/ubuntu|${APT_MIRROR}/ubuntu|g" \ + -e "s|http://security.ubuntu.com/ubuntu|${APT_MIRROR}/ubuntu|g" \ + -e "s|http://ports.ubuntu.com/ubuntu-ports|${APT_MIRROR}/ubuntu-ports|g" \ + /etc/apt/sources.list.d/ubuntu.sources; \ + fi + +# Optional rustup mirror for the Dev Container Feature +# `ghcr.io/devcontainers/features/rust:1` (and any later `rustup` calls). +# Defaults to the upstream static.rust-lang.org; pass +# --build-arg RUST_DIST_SERVER=https://mirrors.ustc.edu.cn/rust-static for a +# China-friendly CDN. Set as ENV so it is inherited by every subsequent layer. +ARG RUST_DIST_SERVER=https://static.rust-lang.org +ENV RUSTUP_DIST_SERVER=${RUST_DIST_SERVER} \ + RUSTUP_UPDATE_ROOT=${RUST_DIST_SERVER}/rustup + # Install necessary packages RUN apt update && \ apt install -y \ @@ -48,6 +71,16 @@ RUN apt update && \ vim \ wget \ sudo \ + # ---- additions for tantivy-fts migration (Rust + Sanitizer + LLVM) ---- + clang \ + clang-format \ + clang-tidy \ + lld \ + llvm \ + libclang-rt-dev \ + gdb \ + lldb \ + valgrind \ && rm -rf /var/lib/apt/lists/* # Add a user for development diff --git a/.devcontainer/centos7/Dockerfile b/.devcontainer/centos7/Dockerfile new file mode 100644 index 00000000..cd14667e --- /dev/null +++ b/.devcontainer/centos7/Dockerfile @@ -0,0 +1,241 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# CentOS 7 cross-build verification image for paimon-cpp + tantivy-fts. +# +# Purpose: +# Prove the tantivy-fts stack builds on the OLDEST reasonable Linux target +# (glibc 2.17, EOL 2024-06-30). The default Ubuntu 24.04 dev container +# proves nothing about glibc compatibility; this image does. +# +# Build: +# docker build -t paimon-cpp-centos7:latest -f .devcontainer/centos7/Dockerfile . +# +# Run: +# docker run -d --name paimon-centos7 \ +# --privileged \ +# -v "$(pwd):/workspaces/paimon-cpp" \ +# paimon-cpp-centos7:latest sleep infinity +# docker exec -it paimon-centos7 bash -l +# +# Inside the container: +# scl enable devtoolset-11 rh-python38 -- bash # activate modern gcc + python +# source /opt/paimon-env.sh # PATH for rust, cmake +# cd /workspaces/paimon-cpp +# git lfs install --local && git lfs pull # critical: boost & friends are LFS +# ./scripts/tantivy_smoke.sh + +# ---------- Base ---------- +# CentOS 7 reached EOL 2024-06-30; its default mirrorlist.centos.org is down. +# Pin to vault.centos.org (Red Hat's archived location) via the `linuxserver/centos` +# vault image to avoid retired-mirror failures on `yum install`. +# +# Base image: we pull from quay.io (CentOS community's canonical registry post +# Docker Hub deprecation). Override with CENTOS7_IMAGE build arg when behind a +# firewall that can't reach quay.io (e.g. registry.aliyuncs.com/library/centos:7). +ARG CENTOS7_IMAGE=quay.io/centos/centos:centos7 +FROM ${CENTOS7_IMAGE} + +# Repoint yum at aliyun's CentOS 7 vault mirror — vault.centos.org itself +# works but is slow/blocked from many CN networks; the aliyun mirror is a +# complete rsync and reliably fast. We overwrite CentOS-Base.repo rather +# than sed-patch it so the result is deterministic regardless of what the +# upstream image ships. fastestmirror plugin is disabled because its ping +# probes against the retired mirror list add ~60s to every `yum install`. +RUN echo -e '[base]\n\ +name=CentOS-7 - Base - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/os/$basearch/\n\ +gpgcheck=0\n\ +enabled=1\n\ +\n\ +[updates]\n\ +name=CentOS-7 - Updates - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/updates/$basearch/\n\ +gpgcheck=0\n\ +enabled=1\n\ +\n\ +[extras]\n\ +name=CentOS-7 - Extras - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/extras/$basearch/\n\ +gpgcheck=0\n\ +enabled=1\n\ +\n\ +[centosplus]\n\ +name=CentOS-7 - Plus - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/centosplus/$basearch/\n\ +gpgcheck=0\n\ +enabled=0\n' > /etc/yum.repos.d/CentOS-Base.repo \ + && rm -f /etc/yum.repos.d/CentOS-CR.repo \ + /etc/yum.repos.d/CentOS-Debuginfo.repo \ + /etc/yum.repos.d/CentOS-Media.repo \ + /etc/yum.repos.d/CentOS-Sources.repo \ + /etc/yum.repos.d/CentOS-Vault.repo \ + /etc/yum.repos.d/CentOS-fasttrack.repo \ + /etc/yum.repos.d/CentOS-x86_64-kernel.repo \ + && if [ -f /etc/yum/pluginconf.d/fastestmirror.conf ]; then \ + sed -i 's/^enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf; \ + fi \ + && yum clean all \ + && yum makecache + +# ---------- Base toolchain ---------- +# EPEL provides git-lfs, ninja-build, a newer python3 than the base 3.6. +# SCL (Software Collections) provides devtoolset-11 (gcc 11) and rh-python38 +# without overriding the system gcc/python. CentOS 7's default gcc 4.8 is +# too old for C++17/20 used by lucene++ and our tantivy wrapper. +# +# Same story as CentOS-Base.repo: both epel + SCL default to mirrorlist +# endpoints that are effectively dead; overwrite with aliyun URLs that we +# know respond. +RUN yum install -y epel-release centos-release-scl \ + && echo -e '[epel]\n\ +name=Extra Packages for Enterprise Linux 7 - aliyun\n\ +baseurl=https://mirrors.aliyun.com/epel/7/$basearch\n\ +gpgcheck=0\n\ +enabled=1\n' > /etc/yum.repos.d/epel.repo \ + && rm -f /etc/yum.repos.d/epel-testing.repo /etc/yum.repos.d/epel.repo.rpmnew \ + && rm -f /etc/yum.repos.d/CentOS-SCLo-*.repo \ + /etc/yum.repos.d/CentOS-SCLo-*.repo.rpmnew \ + && echo -e '[centos-sclo-rh]\n\ +name=CentOS-7 - SCLo rh - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/sclo/$basearch/rh/\n\ +gpgcheck=0\n\ +enabled=1\n\ +\n\ +[centos-sclo-sclo]\n\ +name=CentOS-7 - SCLo sclo - aliyun vault\n\ +baseurl=https://mirrors.aliyun.com/centos-vault/7.9.2009/sclo/$basearch/sclo/\n\ +gpgcheck=0\n\ +enabled=1\n' > /etc/yum.repos.d/CentOS-SCLo-scl.repo \ + && yum clean all && yum makecache \ + && yum install -y \ + devtoolset-11-gcc \ + devtoolset-11-gcc-c++ \ + devtoolset-11-binutils \ + devtoolset-11-libasan-devel \ + devtoolset-11-libubsan-devel \ + rh-python38 \ + rh-python38-python-pip \ + git \ + git-lfs \ + ninja-build \ + make \ + patch \ + curl \ + wget \ + unzip \ + which \ + file \ + sudo \ + openssl-devel \ + zlib-devel \ + libffi-devel \ + bzip2-devel \ + xz-devel \ + perl-IPC-Cmd \ + && yum clean all + +# Enable the SCL collections for all subsequent shells (including RUN). +ENV BASH_ENV=/etc/profile.d/scl-enable.sh +SHELL ["/bin/bash", "-c"] +RUN printf '%s\n' \ + 'source scl_source enable devtoolset-11' \ + 'source scl_source enable rh-python38' \ + > /etc/profile.d/scl-enable.sh \ + && chmod +x /etc/profile.d/scl-enable.sh + +# ---------- CMake (must be >= 3.22 for Corrosion) ---------- +# CentOS 7's cmake package is 2.8.12; EPEL cmake3 is 3.17 — still too old. +# Install via pip in the rh-python38 SCL so we get a modern CMake without +# touching the system /usr/bin. Point pip at aliyun's pypi mirror: default +# pypi.org is 10-30s per request from CN, aliyun responds in <1s. +ENV PIP_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/ \ + PIP_TRUSTED_HOST=mirrors.aliyun.com +RUN source /etc/profile.d/scl-enable.sh \ + && python3 -m pip install --upgrade pip \ + && python3 -m pip install 'cmake==3.28.*' ninja + +# ---------- Rust toolchain ---------- +# Install rustup as root into /opt/rust so all users share the same toolchain. +# Use the USTC mirror to keep downloads fast in CN; the CI runner version of +# this is mirrored in ci/scripts/setup_rust.sh. +ENV RUSTUP_HOME=/opt/rust/rustup \ + CARGO_HOME=/opt/rust/cargo \ + RUSTUP_DIST_SERVER=https://mirrors.ustc.edu.cn/rust-static \ + RUSTUP_UPDATE_ROOT=https://mirrors.ustc.edu.cn/rust-static/rustup +# In-container network for Docker Desktop builds is unreliable through many +# CN mirrors (observed: curl 7.29 on CentOS 7 + rsproxy.cn HTTP/2 path ⇒ +# partial-read truncations; USTC ⇒ 5xx; rustup sh installer ⇒ 403 from +# legacy cipher). The most reliable fix is to sidestep the issue entirely: +# pre-download rustup-init on the host (where network is solid) and COPY it +# into the image. See .devcontainer/centos7/run.sh for the prefetch step. +COPY .devcontainer/centos7/rustup-init.bin /tmp/rustup-init +RUN chmod +x /tmp/rustup-init \ + && /tmp/rustup-init -y --default-toolchain stable --profile minimal --no-modify-path \ + && rm -f /tmp/rustup-init \ + && mkdir -p $CARGO_HOME \ + && echo -e '[source.crates-io]\n\ +replace-with = "rsproxy-sparse"\n\ +\n\ +[source.rsproxy]\n\ +registry = "https://rsproxy.cn/crates.io-index"\n\ +\n\ +[source.rsproxy-sparse]\n\ +registry = "sparse+https://rsproxy.cn/index/"\n\ +\n\ +[registries.rsproxy]\n\ +index = "https://rsproxy.cn/crates.io-index"\n\ +\n\ +[net]\n\ +git-fetch-with-cli = true\n' > $CARGO_HOME/config.toml \ + && $CARGO_HOME/bin/cargo install cbindgen --version 0.29.2 --locked \ + && chmod -R a+rwx /opt/rust \ + && $CARGO_HOME/bin/rustc --version \ + && $CARGO_HOME/bin/cargo --version \ + && $CARGO_HOME/bin/cbindgen --version + +# ---------- Environment file consumed by every shell ---------- +# Sets PATH for rust / cmake / cargo so `docker exec paimon-centos7 bash -l` +# and interactive sessions have the toolchain on $PATH. +RUN printf '%s\n' \ + 'export PATH=/opt/rust/cargo/bin:$PATH' \ + '# cmake + ninja live under the rh-python38 SCL; path prefix differs by arch.' \ + '# `command -v cmake` confirms which one is in use.' \ + > /opt/paimon-env.sh \ + && chmod +x /opt/paimon-env.sh \ + && printf '%s\n' 'source /opt/paimon-env.sh' >> /etc/profile.d/scl-enable.sh + +# ---------- Non-root user ---------- +# Build as `paimon` (uid 1000) so LFS objects under the mount stay owned by +# your host user, matching the main Ubuntu dev container. +RUN useradd -m -u 1000 -s /bin/bash paimon \ + && echo 'paimon ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/paimon + +USER paimon +WORKDIR /workspaces/paimon-cpp + +# Sanity check surfaces the tool versions in `docker run ... paimon-cpp-centos7 --version`. +CMD ["bash", "-lc", "\ + echo '--- CentOS 7 cross-build image sanity check ---'; \ + cat /etc/centos-release; \ + echo '--- glibc ---'; ldd --version | head -1; \ + echo '--- gcc ---'; gcc --version | head -1; \ + echo '--- cmake ---'; cmake --version | head -1; \ + echo '--- ninja ---'; ninja --version; \ + echo '--- rust ---'; rustc --version; \ + echo '--- cargo ---'; cargo --version; \ + echo '--- cbindgen ---'; cbindgen --version; \ + echo 'Ready. Mount paimon-cpp at /workspaces/paimon-cpp and run ./scripts/tantivy_smoke.sh'"] diff --git a/.devcontainer/centos7/run.sh b/.devcontainer/centos7/run.sh new file mode 100755 index 00000000..17818b56 --- /dev/null +++ b/.devcontainer/centos7/run.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# One-shot helper to build + launch + smoke-test the CentOS 7 verification +# container. Run from the paimon-cpp repo root. +# +# Usage: +# ./.devcontainer/centos7/run.sh build # build image only +# ./.devcontainer/centos7/run.sh up # start container (detached) +# ./.devcontainer/centos7/run.sh shell # exec into it +# ./.devcontainer/centos7/run.sh smoke # run scripts/tantivy_smoke.sh inside +# ./.devcontainer/centos7/run.sh down # stop + remove + +set -euo pipefail + +IMAGE=paimon-cpp-centos7:latest +CONTAINER=paimon-centos7 + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo=$(cd "${here}/../.." && pwd) + +cmd=${1:-help} + +case "${cmd}" in + build) + # Prefetch rustup-init on the host. In-container network from Docker + # Desktop builds is unreliable for CN mirrors (TLS/HTTP2 issues with + # old curl/wget on CentOS 7), but host curl works. The image copies + # this blob in. Override mirror with RUSTUP_INIT_URL=... if needed. + rustup_init="${here}/rustup-init.bin" + rustup_url="${RUSTUP_INIT_URL:-https://mirrors.ustc.edu.cn/rust-static/rustup/dist/x86_64-unknown-linux-gnu/rustup-init}" + if [ ! -s "${rustup_init}" ]; then + echo "==> Prefetching rustup-init from ${rustup_url}" + curl --proto '=https' --tlsv1.2 -sSfL --retry 5 --retry-delay 5 \ + -o "${rustup_init}" "${rustup_url}" + fi + # Override base image with CENTOS7_IMAGE=... if quay.io is unreachable. + # Common fallbacks you may need to docker-pull into local cache first: + # CENTOS7_IMAGE=quay.io/centos/centos:centos7 (default) + # CENTOS7_IMAGE=registry.aliyuncs.com/library/centos:7 + if [ -n "${CENTOS7_IMAGE:-}" ]; then + docker build -t "${IMAGE}" -f "${here}/Dockerfile" \ + --build-arg "CENTOS7_IMAGE=${CENTOS7_IMAGE}" "${repo}" + else + docker build -t "${IMAGE}" -f "${here}/Dockerfile" "${repo}" + fi + ;; + up) + docker rm -f "${CONTAINER}" 2>/dev/null || true + # Mount host SSH keys read-only (mirrors paimon-dev) so git clones of + # internal repos (e.g. aliorc_ep on gitlab.alibaba-inc.com) that go + # over SSH can authenticate with the host's key. Skip the mount if + # ~/.ssh doesn't exist so the script still works for external users. + ssh_mount=() + if [ -d "${HOME}/.ssh" ]; then + ssh_mount=(-v "${HOME}/.ssh:/home/paimon/.ssh:ro") + fi + docker run -d \ + --name "${CONTAINER}" \ + --privileged \ + -v "${repo}:/workspaces/paimon-cpp" \ + -v "paimon-centos7-cargo-registry:/opt/rust/cargo/registry" \ + -v "paimon-centos7-build:/workspaces/paimon-cpp/build-centos7" \ + "${ssh_mount[@]}" \ + "${IMAGE}" sleep infinity + # Named volumes mount as root-owned; `paimon` user (uid 1000) needs + # write access to build-centos7 and the cargo registry cache. + # Also set up the gitlab.alibaba-inc.com url rewrite so aliorc_ep + # (and any other ExternalProject pointing at internal gitlab via + # http://) picks up the mounted SSH key. + docker exec --user root "${CONTAINER}" bash -c ' + chown -R paimon:paimon /workspaces/paimon-cpp/build-centos7 \ + /opt/rust/cargo/registry + ' + docker exec "${CONTAINER}" bash -c ' + git config --global url."git@gitlab.alibaba-inc.com:".insteadOf \ + "http://gitlab.alibaba-inc.com/" + ' + echo "Container started. \`${0} shell\` to enter." + ;; + shell) + docker exec -it "${CONTAINER}" bash -l + ;; + smoke) + # Ensure container is up first; no-op if already running. + if ! docker ps --format '{{.Names}}' | grep -qx "${CONTAINER}"; then + echo "Container ${CONTAINER} not running; starting it." + "$0" up + fi + # Two env vars pass through for Rosetta 2 (Apple Silicon) compat: + # MALLOC_CHECK_=0 disables glibc 2.17 extra malloc integrity checks + # that fire false positives under Rosetta's x86_64 emulation. + # ARROW_USER_SIMD_LEVEL=SSE4_2 keeps arrow runtime-dispatched kernels + # on SSE4.2 only (Rosetta does not support AVX2/BMI2/AVX-512). + # Both are no-ops on real x86_64 CentOS 7 hardware. + # Use a distinct build dir inside the container so it does not clash + # with the Ubuntu dev container's build/ dir on the same volume. + # Propagate PAIMON_ENABLE_ALIORC so `PAIMON_ENABLE_ALIORC=OFF` env + # on the host reaches the cmake inside the container. + docker exec \ + -e "PAIMON_ENABLE_ALIORC=${PAIMON_ENABLE_ALIORC:-ON}" \ + -e "MALLOC_CHECK_=0" \ + -e "ARROW_USER_SIMD_LEVEL=SSE4_2" \ + "${CONTAINER}" bash -lc ' + set -eux + cd /workspaces/paimon-cpp + git lfs install --local + git lfs pull + cmake -S . -B build-centos7 \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DPAIMON_BUILD_TESTS=ON \ + -DPAIMON_ENABLE_FSLIB=OFF \ + -DPAIMON_ENABLE_LUMINA=OFF \ + -DPAIMON_ENABLE_LANCE=OFF \ + -DPAIMON_ENABLE_JINDO=OFF \ + -DPAIMON_ENABLE_LUCENE=ON \ + -DPAIMON_ENABLE_ORC=ON \ + -DPAIMON_ENABLE_ALIORC="${PAIMON_ENABLE_ALIORC:-ON}" \ + -DPAIMON_ENABLE_AVRO=ON + # ALIORC clones from internal gitlab. `up` mounts $HOME/.ssh and + # configures the url.insteadOf rewrite, so by default ALIORC works + # for alibaba-inc users. External users without gitlab access can + # opt out with `PAIMON_ENABLE_ALIORC=OFF ./run.sh smoke`. + cmake --build build-centos7 -j "$(nproc)" + ctest --test-dir build-centos7 \ + -R "paimon-lucene-index-test|paimon-global-index-test|paimon-tantivy-.*-test" \ + --output-on-failure + ' + ;; + down) + docker rm -f "${CONTAINER}" 2>/dev/null || true + ;; + help|*) + sed -n "2,20p" "$0" + ;; +esac diff --git a/.devcontainer/devcontainer.json.template b/.devcontainer/devcontainer.json.template index d4f7c347..bc170f50 100644 --- a/.devcontainer/devcontainer.json.template +++ b/.devcontainer/devcontainer.json.template @@ -20,6 +20,10 @@ // Adapted from Apache Iceberg C++ // https://github.com/apache/iceberg-cpp/blob/v0.2.0/.devcontainer/devcontainer.json.template +// Default Paimon C++ Dev Container. +// On Apple Silicon hosts this runs as native aarch64 Linux (fast). +// For x86_64 verification, use the variant under .devcontainer/x86_64/. + { "name": "Paimon CPP Dev Container", "build": { @@ -34,16 +38,36 @@ "seccomp=unconfined", "--privileged" ], + "features": { + "ghcr.io/devcontainers/features/rust:1": { + "version": "stable", + "profile": "default" + } + }, "mounts": [ - "source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly" + "source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly", + "source=paimon-cargo-registry,target=/home/paimon/.cargo/registry,type=volume", + "source=paimon-cargo-git,target=/home/paimon/.cargo/git,type=volume", + "source=paimon-rust-target,target=${containerWorkspaceFolder}/third_party/tantivy_ffi/target,type=volume", + "source=paimon-build,target=${containerWorkspaceFolder}/build,type=volume", + "source=paimon-ccache,target=/home/paimon/.ccache,type=volume" ], + "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/third_party/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true", "customizations": { "vscode": { "extensions": [ - "eamodio.gitlens" + "eamodio.gitlens", + "rust-lang.rust-analyzer", + "vadimcn.vscode-lldb", + "llvm-vs-code-extensions.vscode-clangd", + "ms-vscode.cmake-tools", + "twxs.cmake" ], "settings": { - "editor.formatOnSave": true + "editor.formatOnSave": true, + "rust-analyzer.linkedProjects": [ + "third_party/tantivy_ffi/Cargo.toml" + ] } } } diff --git a/.devcontainer/x86_64/devcontainer.json.template b/.devcontainer/x86_64/devcontainer.json.template new file mode 100644 index 00000000..baa40099 --- /dev/null +++ b/.devcontainer/x86_64/devcontainer.json.template @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// x86_64 variant of the Paimon CPP Dev Container. +// On Apple Silicon hosts this runs under QEMU emulation (5-10x slower). +// Use it ONLY for cross-architecture verification (Stage 11), not daily dev. +// +// Reuses the same Dockerfile as the default container; only the platform differs. +// +// Uses dedicated named volumes (suffix `-amd64`) so build/cargo cache do not +// collide with the native aarch64 container. + +{ + "name": "Paimon CPP Dev Container (x86_64 via QEMU)", + "build": { + "dockerfile": "../Dockerfile", + "options": [ + "--platform=linux/amd64" + ] + }, + "runArgs": [ + "--platform=linux/amd64", + "--ulimit=core=-1", + "--cap-add=SYS_ADMIN", + "--cap-add=SYS_PTRACE", + "--cap-add=PERFMON", + "--security-opt", + "seccomp=unconfined", + "--privileged" + ], + "features": { + "ghcr.io/devcontainers/features/rust:1": { + "version": "stable", + "profile": "default" + } + }, + "mounts": [ + "source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly", + "source=paimon-cargo-registry-amd64,target=/home/paimon/.cargo/registry,type=volume", + "source=paimon-cargo-git-amd64,target=/home/paimon/.cargo/git,type=volume", + "source=paimon-rust-target-amd64,target=${containerWorkspaceFolder}/third_party/tantivy_ffi/target,type=volume", + "source=paimon-build-amd64,target=${containerWorkspaceFolder}/build,type=volume", + "source=paimon-ccache-amd64,target=/home/paimon/.ccache,type=volume" + ], + "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/third_party/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true", + "customizations": { + "vscode": { + "extensions": [ + "eamodio.gitlens", + "rust-lang.rust-analyzer", + "vadimcn.vscode-lldb", + "llvm-vs-code-extensions.vscode-clangd", + "ms-vscode.cmake-tools" + ], + "settings": { + "editor.formatOnSave": true + } + } + } +} diff --git a/.github/workflows/build_release.yaml b/.github/workflows/build_release.yaml deleted file mode 100644 index 0bed7ced..00000000 --- a/.github/workflows/build_release.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Build Release - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - clang-release: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-clang-release - - name: Build Paimon - shell: bash - env: - CC: clang - CXX: clang++ - run: ci/scripts/build_paimon.sh $(pwd) false false Release - - name: Show ccache statistics - if: always() - run: ccache -s - gcc-release: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-gcc-release - - name: Build Paimon - shell: bash - env: - CC: gcc-14 - CXX: g++-14 - run: ci/scripts/build_paimon.sh $(pwd) false false Release - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/.github/workflows/clang_test.yaml b/.github/workflows/clang_test.yaml deleted file mode 100644 index ea30689c..00000000 --- a/.github/workflows/clang_test.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Clang Test - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - clang-test: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - fetch-depth: 0 # fetch all history for git diff in clang-tidy - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-clang-test - - name: Build Paimon - shell: bash - env: - CC: clang - CXX: clang++ - run: ci/scripts/build_paimon.sh $(pwd) false true - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/.github/workflows/gcc_test.yaml b/.github/workflows/gcc_test.yaml deleted file mode 100644 index b213f66f..00000000 --- a/.github/workflows/gcc_test.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Gcc Test - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - gcc-test: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-gcc-test - - name: Build Paimon - shell: bash - env: - CC: gcc-14 - CXX: g++-14 - run: ci/scripts/build_paimon.sh $(pwd) - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/.github/workflows/test_with_sanitizer.yaml b/.github/workflows/test_with_sanitizer.yaml deleted file mode 100644 index 4005c8a2..00000000 --- a/.github/workflows/test_with_sanitizer.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Test with sanitizer - -on: - push: - branches: - - '**' - tags: - - '**' - pull_request: - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - test-with-sanitizer: - runs-on: ubuntu-24.04 - timeout-minutes: 120 - strategy: - fail-fast: false - steps: - - name: Checkout paimon-cpp - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - name: Setup ccache - uses: ./.github/actions/setup-ccache - with: - cache-key-prefix: ccache-sanitizer - - name: Build Paimon - shell: bash - env: - CC: clang - CXX: clang++ - run: ci/scripts/build_paimon.sh $(pwd) true - - name: Show ccache statistics - if: always() - run: ccache -s diff --git a/.gitignore b/.gitignore index b71dc569..3ff833af 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,7 @@ # Build directories build -build-release -build-debug +build-*/ output release @@ -28,8 +27,20 @@ release .cache # Devcontainer configuration +# Track only *.template files (and subdirectory structure that contains them). .devcontainer/* !.devcontainer/*.template +!.devcontainer/x86_64/ +.devcontainer/x86_64/* +!.devcontainer/x86_64/*.template +# CentOS 7 cross-build image: track raw Dockerfile + helper script (not +# templated because the image is built from the repo root directly). +!.devcontainer/centos7/ +.devcontainer/centos7/* +!.devcontainer/centos7/Dockerfile +!.devcontainer/centos7/run.sh +# rustup-init.bin is a 20 MB prefetched binary — not source, don't commit. +.devcontainer/centos7/rustup-init.bin # Temporary and backup files *~ @@ -53,3 +64,6 @@ FlameGraph # Third party dependencies archives third_party/*.tar.gz + +# Rust / Cargo build artifacts +third_party/tantivy_ffi/target/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 30c2474d..8b91fefb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,11 @@ # Adapted from Apache Arrow: # https://github.com/apache/arrow/blob/apache-arrow-17.0.0/cpp/CMakeLists.txt -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.22) +# 3.22 is the minimum required by Corrosion-rs (used for the Rust<->C++ FFI +# integration, see third_party/tantivy_ffi). Ubuntu 24.04 ships CMake 3.28 and +# CentOS 8+/RHEL 9+ ship 3.20+. To build on older distros, see +# docs/dev/tantivy_fts_migration_plan.md. message(STATUS "Building using CMake version: ${CMAKE_VERSION}") # https://cmake.org/cmake/help/latest/policy/CMP0135.html @@ -58,6 +62,8 @@ option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON) option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON) option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF) option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF) +option(PAIMON_ENABLE_TANTIVY + "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF) if(PAIMON_ENABLE_ORC) add_definitions(-DPAIMON_ENABLE_ORC) endif() @@ -84,6 +90,10 @@ if(PAIMON_ENABLE_LUCENE) add_definitions(-DPAIMON_ENABLE_LUCENE) endif() +if(PAIMON_ENABLE_TANTIVY) + add_definitions(-DPAIMON_ENABLE_TANTIVY) +endif() + add_definitions(-DSNAPPY_CODEC_AVAILABLE) add_definitions(-DZSTD_CODEC_AVAILABLE) add_definitions(-DRAPIDJSON_HAS_STDSTRING) @@ -286,6 +296,21 @@ set(PAIMON_SHARED_PRIVATE_LINK_LIBS ${PAIMON_STATIC_LINK_LIBS}) add_subdirectory(third_party/roaring_bitmap EXCLUDE_FROM_ALL) add_subdirectory(third_party/xxhash EXCLUDE_FROM_ALL) +# ---- tantivy-fulltext Rust FFI via Corrosion-rs -------------------------------- +# See docs/dev/tantivy_fts_migration_plan.md Stage 1. +# +# Corrosion wraps the Cargo crate as a CMake target named `paimon_tantivy_ffi`. +# `corrosion_experimental_cbindgen` runs cbindgen from CMake and writes the +# header to a stable path; it also adds that path to the target's INTERFACE +# include dirs so C++ consumers pick it up via target_link_libraries. +if(PAIMON_ENABLE_TANTIVY) + include(CorrosionFetch) + corrosion_import_crate(MANIFEST_PATH third_party/tantivy_ffi/Cargo.toml CRATES + paimon_tantivy_ffi) + corrosion_experimental_cbindgen(TARGET paimon_tantivy_ffi HEADER_NAME + paimon_tantivy_ffi.h) +endif() + if(PAIMON_ENABLE_LUCENE) set(PAIMON_DICT_DEST "share/paimon/dict") @@ -471,6 +496,9 @@ if(PAIMON_ENABLE_LUMINA) add_subdirectory(src/paimon/global_index/lumina) endif() add_subdirectory(src/paimon/global_index/lucene) +if(PAIMON_ENABLE_TANTIVY) + add_subdirectory(src/paimon/global_index/tantivy) +endif() add_subdirectory(src/paimon/testing/mock) add_subdirectory(src/paimon/testing/utils) add_subdirectory(test/inte) diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index ecc83a75..75b00369 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -47,8 +47,10 @@ mkdir -p "${build_dir}" pushd "${build_dir}" ENABLE_LUMINA="ON" +ENABLE_TANTIVY="ON" if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then ENABLE_LUMINA="OFF" + ENABLE_TANTIVY="OFF" # tantivy-fts (Rust FFI) is not built on the gcc-8 image. fi CMAKE_ARGS=( @@ -58,6 +60,7 @@ CMAKE_ARGS=( "-DPAIMON_ENABLE_JINDO=ON" "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}" "-DPAIMON_ENABLE_LUCENE=ON" + "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}" ) if [[ "${enable_sanitizer}" == "true" ]]; then diff --git a/ci/scripts/setup_rust.sh b/ci/scripts/setup_rust.sh new file mode 100755 index 00000000..64ae9aea --- /dev/null +++ b/ci/scripts/setup_rust.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Install the Rust toolchain + cbindgen required to build the +# tantivy-fts FFI crate (third_party/tantivy_ffi) from CI. +# +# The dev container (see .devcontainer/) already has these preinstalled; +# this script is for the GitHub Actions runners. Called by +# .github/workflows/gcc_test.yaml and test_with_sanitizer.yaml before +# ci/scripts/build_paimon.sh. +# +# Idempotent: a second invocation is a no-op when the tools already exist. + +set -eux + +RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0} +# 1.88.0 is the minimum required by transitive crates (e.g. time 0.3.47). +RUST_VERSION=${RUST_VERSION:-1.88.0} +CBINDGEN_VERSION=${CBINDGEN_VERSION:-0.29.2} + +# Install rustup + default toolchain if cargo isn't on PATH yet. +if ! command -v cargo >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain "${RUST_VERSION}" --profile minimal --no-modify-path +fi + +# Export for the remainder of the CI job. +export PATH="${HOME}/.cargo/bin:${PATH}" +echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH:-/dev/null}" || true + +rustup toolchain install "${RUST_VERSION}" --profile minimal +rustup default "${RUST_VERSION}" +rustup component add rustfmt clippy + +# cbindgen is used by the crate's build.rs to emit the C header that the +# C++ side includes. Corrosion will also run cbindgen at CMake configure +# time; both paths need it available. +if ! command -v cbindgen >/dev/null 2>&1; then + cargo install cbindgen --version "${CBINDGEN_VERSION}" --locked +fi + +rustc --version +cargo --version +cbindgen --version diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index efca80b7..7377fb99 100644 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -99,6 +99,7 @@ function(add_paimon_lib LIB_NAME) endif() # Necessary to make static linking into other shared libraries work properly set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) + target_link_libraries(${LIB_NAME}_objlib PUBLIC paimon_sanitizer_flags) if(ARG_DEPENDENCIES) # In static-only builds, some dependency names are still declared as # *_shared. Map them to *_static when the shared target is unavailable. @@ -184,6 +185,10 @@ function(add_paimon_lib LIB_NAME) if(NOT APPLE) set(SHARED_LINK_OPTIONS -Wl,--exclude-libs,ALL -Wl,-Bsymbolic -Wl,--gc-sections) + # -z defs (--no-undefined) rejects the __asan_*/__ubsan_* symbols that + # sanitizer-instrumented shared libraries legitimately leave undefined + # (they are resolved at load time from the executable's sanitizer + # runtime). Only enforce it for non-sanitizer builds. if(NOT PAIMON_USE_ASAN AND NOT PAIMON_USE_UBSAN) list(APPEND SHARED_LINK_OPTIONS -Wl,-z,defs) endif() @@ -339,6 +344,11 @@ function(add_test_case REL_TEST_NAME) target_compile_options(${TEST_NAME} PRIVATE -Wno-global-constructors) endif() target_compile_options(${TEST_NAME} PRIVATE -fno-access-control) + # Test sources initialize char / vector from raw byte values like + # {1, -1, ...}; char is unsigned by default on aarch64, which triggers + # -Wnarrowing. Disable it for tests so we don't have to sprinkle + # static_cast(-1) everywhere. Production code (src/paimon/...) keeps it. + target_compile_options(${TEST_NAME} PRIVATE -Wno-narrowing) add_test(${TEST_NAME} ${BUILD_SUPPORT_DIR}/run-test.sh diff --git a/cmake_modules/CorrosionFetch.cmake b/cmake_modules/CorrosionFetch.cmake new file mode 100644 index 00000000..926919ad --- /dev/null +++ b/cmake_modules/CorrosionFetch.cmake @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Pull Corrosion-rs via FetchContent so we can import Cargo crates as CMake +# targets. Used to bring in third_party/tantivy_ffi for the tantivy-fulltext +# global index (see docs/dev/tantivy_fts_migration_plan.md). +# +# Pinned to v0.5.0 (stable release). Requires CMake >= 3.22. + +include(FetchContent) + +# Corrosion does heavy cargo/rustc work at configure+build time; pin tag for +# reproducibility and allow override via env var for offline builds. +set(PAIMON_CORROSION_TAG + "v0.5.2" + CACHE STRING "Git tag of corrosion-rs to fetch; change only when upgrading. v0.5.1+ + is required for rustup >= 1.28 whose `rustup toolchain list --verbose` + output format broke v0.5.0's FindRust.cmake regex.") + +set(PAIMON_CORROSION_REPO + "https://github.com/corrosion-rs/corrosion.git" + CACHE STRING "Override to a private mirror for offline / firewalled builds.") + +# Help Corrosion find rustc/cargo when CMake is invoked without a login shell +# or when rustup is installed to a non-default location. We try, in order: +# 1. Existing Rust_COMPILER cache variable (user override) +# 2. $CARGO_HOME/bin/rustc (when env var set) +# 3. $HOME/.cargo/bin/rustc (rustup's default install) +# 4. Fallback: let Corrosion's FindRust.cmake try its own detection +function(_paimon_find_rustup_bin _var _name) + if(DEFINED ENV{CARGO_HOME} AND EXISTS "$ENV{CARGO_HOME}/bin/${_name}") + set(${_var} + "$ENV{CARGO_HOME}/bin/${_name}" + PARENT_SCOPE) + elseif(DEFINED ENV{HOME} AND EXISTS "$ENV{HOME}/.cargo/bin/${_name}") + set(${_var} + "$ENV{HOME}/.cargo/bin/${_name}" + PARENT_SCOPE) + endif() +endfunction() + +if(NOT DEFINED Rust_COMPILER OR Rust_COMPILER STREQUAL "") + _paimon_find_rustup_bin(_rustc_path rustc) + if(_rustc_path) + set(Rust_COMPILER + "${_rustc_path}" + CACHE FILEPATH "rustc") + endif() +endif() +if(NOT DEFINED Rust_CARGO OR Rust_CARGO STREQUAL "") + _paimon_find_rustup_bin(_cargo_path cargo) + if(_cargo_path) + set(Rust_CARGO + "${_cargo_path}" + CACHE FILEPATH "cargo") + endif() +endif() +# Corrosion reads `rustup which rustc` to resolve the real toolchain binary. +# If CMake is invoked from a non-login shell, $PATH may miss ~/.cargo/bin and +# `rustup` can't be found. Prepend rustup's bin dir so child processes see it. +if(DEFINED Rust_COMPILER) + get_filename_component(_rustup_bin_dir "${Rust_COMPILER}" DIRECTORY) + if(_rustup_bin_dir AND NOT "$ENV{PATH}" MATCHES "${_rustup_bin_dir}") + set(ENV{PATH} "${_rustup_bin_dir}:$ENV{PATH}") + endif() +endif() +message(STATUS "Corrosion: Rust_COMPILER=${Rust_COMPILER}") +message(STATUS "Corrosion: Rust_CARGO=${Rust_CARGO}") + +fetchcontent_declare(Corrosion + GIT_REPOSITORY "${PAIMON_CORROSION_REPO}" + GIT_TAG "${PAIMON_CORROSION_TAG}" + GIT_SHALLOW TRUE) +fetchcontent_makeavailable(Corrosion) diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 1179f28c..e2063b99 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -789,6 +789,11 @@ macro(build_lucene) "-DBoost_INCLUDE_DIR=${BOOST_INCLUDE_DIR}" "-DBoost_LIBRARY_DIR=${BOOST_LIBRARY_DIR}" "-DBOOST_ROOT=${BOOST_INSTALL}" + # Force FindBoost module mode only; ignore system BoostConfig.cmake and + # system library paths so lucene_ep links against our vendored boost 1.66, + # not a system-installed newer version (e.g. 1.83) with ABI differences. + "-DBoost_NO_BOOST_CMAKE=ON" + "-DBoost_NO_SYSTEM_PATHS=ON" "-DBoost_CHRONO_FOUND=TRUE" "-DBoost_THREAD_FOUND=TRUE" "-DZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIR}" @@ -1964,5 +1969,9 @@ endif() if(PAIMON_ENABLE_LUCENE) build_boost() build_lucene() +endif() +# jieba (dict + headers) is needed by BOTH lucene-fts and the tantivy jieba +# tokenizer; build it whenever either backend is on, not only under lucene. +if(PAIMON_ENABLE_LUCENE OR PAIMON_ENABLE_TANTIVY) build_jieba() endif() diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 05bdee19..fbaaf1f3 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 3.16) +cmake_minimum_required(VERSION 3.22) project(example) diff --git a/include/paimon/predicate/full_text_search.h b/include/paimon/predicate/full_text_search.h index bd6ba1ea..e31cc3a6 100644 --- a/include/paimon/predicate/full_text_search.h +++ b/include/paimon/predicate/full_text_search.h @@ -48,22 +48,27 @@ struct PAIMON_EXPORT FullTextSearch { FullTextSearch(const std::string& _field_name, std::optional _limit, const std::string& _query, const SearchType& _search_type, - const std::optional& _pre_filter) + const std::optional& _pre_filter, bool _with_score = false, + std::optional _min_score = std::nullopt) : field_name(_field_name), limit(_limit), query(_query), search_type(_search_type), - pre_filter(_pre_filter) {} + pre_filter(_pre_filter), + with_score(_with_score), + min_score(_min_score) {} std::shared_ptr ReplacePreFilter( const std::optional& _pre_filter) const { - return std::make_shared(field_name, limit, query, search_type, _pre_filter); + return std::make_shared(field_name, limit, query, search_type, _pre_filter, + with_score, min_score); } /// Name of the field to search within (must be a full-text indexed field). std::string field_name; - /// Maximum number of documents to return. If set, limit ordered by top scores. Otherwise, no - /// score return. + /// Maximum number of documents to return. Purely a truncation switch, + /// orthogonal to `with_score`: set `with_score = true` to get relevance + /// scores; a non-empty `limit` does not by itself imply scoring. std::optional limit; /// The query string to search for. The interpretation depends on search_type: /// @@ -87,5 +92,26 @@ struct PAIMON_EXPORT FullTextSearch { /// Only rows whose global row ID is present in `pre_filter` will be included during search. /// If not set, all rows will be included. std::optional pre_filter; + /// Whether to compute and return relevance scores (e.g. BM25). The 4-path matrix: + /// - `with_score=false, limit=nullopt` → BitmapGlobalIndexResult (all rows, no score) + /// - `with_score=false, limit=N` → BitmapGlobalIndexResult (any N matches, unscored) + /// - `with_score=true, limit=nullopt` → BitmapScoredGlobalIndexResult (all rows + all scores) + /// - `with_score=true, limit=N` → BitmapScoredGlobalIndexResult (top-N by score + + /// scores) + /// + /// For plain `LIMIT N` without ORDER BY (the common case when an online + /// engine, e.g. StarRocks, pushes down a predicate) set `with_score=false, + /// limit=N` — the unscored fast path. For top-N by relevance use + /// `with_score=true, limit=N` and drop the scores in the caller if unneeded. + /// + /// Default is `false` to avoid score computation overhead for callers that don't need it. + bool with_score = false; + /// Minimum relevance-score threshold (exclusive); results with score ≤ this value are + /// excluded. The score is whatever the backend's similarity produces (e.g. BM25 for + /// tantivy, classic TF-IDF for lucene), so a threshold is not directly comparable across + /// backends. Only meaningful when scoring is active (`with_score = true` or `limit` set); + /// applied before truncation so low-score documents never occupy limit slots. + /// Default is nullopt (no threshold filtering). + std::optional min_score; }; } // namespace paimon diff --git a/scripts/tantivy_smoke.sh b/scripts/tantivy_smoke.sh new file mode 100755 index 00000000..e4598418 --- /dev/null +++ b/scripts/tantivy_smoke.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Smoke-test script for the tantivy-fts migration. +# +# Purpose: one-shot regression of the lucene-fts + tantivy-fts tests inside the +# Dev Container. +# Rationale: the command line gets long and error-prone, so wrap it in a script +# maintained per stage. +# +# Usage: +# ./scripts/tantivy_smoke.sh # default: release, no sanitizer +# ./scripts/tantivy_smoke.sh --asan # ASAN build +# ./scripts/tantivy_smoke.sh --tsan # TSAN build +# ./scripts/tantivy_smoke.sh --configure # cmake configure only +# ./scripts/tantivy_smoke.sh --build # cmake build only (skip configure) +# ./scripts/tantivy_smoke.sh --tests-only # ctest only (assumes already built) +# +# Maintenance notes: +# - From Stage 1 on, update TEST_REGEX below whenever a new ctest target is added +# - Stage 11 adds the full --with-asan / --with-tsan path + +set -e + +CMAKE_BUILD_TYPE="Release" +USE_ASAN="OFF" +USE_TSAN="OFF" +BUILD_DIR_SUFFIX="" +DO_CONFIGURE=1 +DO_BUILD=1 +DO_TEST=1 + +# ctest regex: during per-stage acceptance, run only this subset rather than the +# full ctest (~531s, too slow). Contents = the lucene-fts baseline + the +# tantivy-fts targets added in the current and previous stages. Append a target +# here as each stage completes. Only Stage 11 should run the full ctest. +TEST_REGEX='paimon-lucene-index-test|paimon-global-index-test|paimon-tantivy-smoke-test|paimon-tantivy-ffi-test|paimon-tantivy-tokenizer-test|paimon-tantivy-writer-test|paimon-tantivy-reader-test|paimon-tantivy-filter-limit-test|paimon-tantivy-index-test|paimon-tantivy-lucene-coexist-test|paimon-tantivy-equivalence-test|paimon-tantivy-streaming-test|paimon-tantivy-java-compat-test' + +while [ $# -gt 0 ]; do + case "$1" in + --asan) USE_ASAN="ON"; CMAKE_BUILD_TYPE="Debug"; BUILD_DIR_SUFFIX="-asan" ;; + --tsan) USE_TSAN="ON"; CMAKE_BUILD_TYPE="Debug"; BUILD_DIR_SUFFIX="-tsan" ;; + --configure) DO_BUILD=0; DO_TEST=0 ;; + --build) DO_CONFIGURE=0; DO_TEST=0 ;; + --tests-only) DO_CONFIGURE=0; DO_BUILD=0 ;; + -h|--help) sed -n '2,20p' "$0"; exit 0 ;; + *) echo "Unknown option: $1"; exit 2 ;; + esac + shift +done + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BUILD_DIR="${REPO_ROOT}/build${BUILD_DIR_SUFFIX}" + +cd "${REPO_ROOT}" + +if [ "${DO_CONFIGURE}" = "1" ]; then + echo "==> cmake configure (${BUILD_DIR})" + cmake -S . -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ + -DPAIMON_BUILD_TESTS=ON \ + -DPAIMON_USE_ASAN="${USE_ASAN}" \ + -DPAIMON_USE_TSAN="${USE_TSAN}" \ + -DPAIMON_ENABLE_FSLIB=OFF \ + -DPAIMON_ENABLE_LUMINA=OFF \ + -DPAIMON_ENABLE_LANCE=OFF \ + -DPAIMON_ENABLE_JINDO=OFF \ + -DPAIMON_ENABLE_LUCENE=ON \ + -DPAIMON_ENABLE_ORC=ON \ + -DPAIMON_ENABLE_ALIORC=ON \ + -DPAIMON_ENABLE_AVRO=ON \ + -G Ninja +fi + +if [ "${DO_BUILD}" = "1" ]; then + echo "==> cmake build" + cmake --build "${BUILD_DIR}" -j +fi + +if [ "${DO_TEST}" = "1" ]; then + echo "==> ctest (${TEST_REGEX})" + ctest --test-dir "${BUILD_DIR}" -R "${TEST_REGEX}" --output-on-failure +fi + +echo "==> tantivy_smoke.sh DONE" diff --git a/src/paimon/common/data/binary_row_test.cpp b/src/paimon/common/data/binary_row_test.cpp index 681c4e54..d851881a 100644 --- a/src/paimon/common/data/binary_row_test.cpp +++ b/src/paimon/common/data/binary_row_test.cpp @@ -341,8 +341,9 @@ TEST_F(BinaryRowTest, TestBinary) { auto pool = GetDefaultPool(); BinaryRow row(2); BinaryRowWriter writer(&row, 0, pool.get()); - char chars1[3] = {1, -1, 5}; - char chars2[8] = {1, -1, 5, 5, 1, 5, 1, 5}; + // explicit cast to avoid -Wnarrowing on platforms where char is unsigned (e.g. aarch64) + char chars1[3] = {1, static_cast(-1), 5}; + char chars2[8] = {1, static_cast(-1), 5, 5, 1, 5, 1, 5}; std::string str1(chars1, 3); std::string str2(chars2, 8); Bytes bytes1(str1, pool.get()); diff --git a/src/paimon/common/global_index/offset_global_index_reader_test.cpp b/src/paimon/common/global_index/offset_global_index_reader_test.cpp index e090045a..0815f3f1 100644 --- a/src/paimon/common/global_index/offset_global_index_reader_test.cpp +++ b/src/paimon/common/global_index/offset_global_index_reader_test.cpp @@ -25,6 +25,7 @@ #include "gtest/gtest.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/predicate/full_text_search.h" #include "paimon/predicate/literal.h" #include "paimon/testing/utils/testharness.h" #include "paimon/utils/roaring_bitmap64.h" @@ -115,9 +116,14 @@ class FakeGlobalIndexReader : public GlobalIndexReader { Result> VisitFullTextSearch( const std::shared_ptr& full_text_search) override { + captured_fts = full_text_search; return MakeResult(default_result_); } + // Captures the (possibly pre_filter-rewritten) FullTextSearch the offset + // reader forwarded, so tests can assert field propagation. + std::shared_ptr captured_fts; + bool IsThreadSafe() const override { return true; } @@ -334,6 +340,37 @@ TEST_F(OffsetGlobalIndexReaderTest, TestVisitFullTextSearchWithOffset) { CheckResult(result, {10, 13, 15}); } +TEST_F(OffsetGlobalIndexReaderTest, TestVisitFullTextSearchPreservesScoreFlags) { + // Regression (review finding #2): rewriting the pre_filter global->local ids + // in the offset reader must NOT drop with_score / min_score. Before the fix, + // FullTextSearch::ReplacePreFilter rebuilt via the 5-arg ctor and silently + // reset both back to their defaults, turning a scored / min_score query + // unscored as soon as it crossed any offset shard. + auto fake_reader = std::make_shared(); + fake_reader->SetDefaultResult({0, 3, 5}); + auto offset_reader = std::make_shared(fake_reader, 10); + + // pre_filter must be set so the offset reader takes the rewrite path. + auto fts = std::make_shared( + "f0", /*limit=*/7, "q", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/RoaringBitmap64::From({10l, 13l, 15l})); + fts->with_score = true; + fts->min_score = 1.5f; + + ASSERT_OK_AND_ASSIGN(auto result, offset_reader->VisitFullTextSearch(fts)); + CheckResult(result, {10, 13, 15}); + + ASSERT_TRUE(fake_reader->captured_fts); + ASSERT_TRUE(fake_reader->captured_fts->with_score) + << "with_score must survive the pre_filter rewrite"; + ASSERT_TRUE(fake_reader->captured_fts->min_score.has_value()) + << "min_score must survive the pre_filter rewrite"; + ASSERT_FLOAT_EQ(fake_reader->captured_fts->min_score.value(), 1.5f); + // limit and the offset-rewritten local pre_filter should still be present. + ASSERT_EQ(fake_reader->captured_fts->limit, std::optional(7)); + ASSERT_TRUE(fake_reader->captured_fts->pre_filter.has_value()); +} + TEST_F(OffsetGlobalIndexReaderTest, TestVisitVectorSearchWithOffset) { auto fake_reader = std::make_shared(); fake_reader->SetVectorSearchResult({0, 2, 5}, {0.9f, 0.7f, 0.3f}); diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index 603f9021..02edd4d3 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -210,6 +210,14 @@ std::shared_ptr LuceneGlobalIndexReader::SearchWithNoLimit( Result> LuceneGlobalIndexReader::VisitFullTextSearch( const std::shared_ptr& full_text_search) { + if (full_text_search && full_text_search->min_score.has_value()) { + // The lucene backend does not support min_score pushdown. Fail loudly + // instead of silently ignoring the threshold and returning unfiltered + // results, which would be a correctness bug for the caller. + return Status::NotImplemented( + "lucene full-text search does not support min_score; " + "min_score pushdown is only available on the tantivy backend"); + } try { Lucene::QueryPtr query; switch (full_text_search->search_type) { diff --git a/src/paimon/global_index/lucene/lucene_global_index_test.cpp b/src/paimon/global_index/lucene/lucene_global_index_test.cpp index a65d630b..cb4bffcf 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_test.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_test.cpp @@ -306,6 +306,18 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) { /*pre_filter=*/RoaringBitmap64::From({1l, 2l, 3l, 100l})))); CheckResult(result, {2l}); } + // min_score pushdown is not supported by the lucene backend: it must fail + // loudly rather than silently ignore the threshold. + { + auto fts = std::make_shared("f0", + /*limit=*/10, "document", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->min_score = 1.5f; + auto result = lucene_reader->VisitFullTextSearch(fts); + ASSERT_FALSE(result.ok()); + ASSERT_TRUE(result.status().IsNotImplemented()) << result.status().ToString(); + } } TEST_P(LuceneGlobalIndexTest, TestSimpleChinese) { diff --git a/src/paimon/global_index/tantivy/CMakeLists.txt b/src/paimon/global_index/tantivy/CMakeLists.txt new file mode 100644 index 00000000..3873e250 --- /dev/null +++ b/src/paimon/global_index/tantivy/CMakeLists.txt @@ -0,0 +1,268 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# tantivy-fulltext global index (Rust FFI). See docs/dev/tantivy_fts_migration_plan.md. +# Stage 4 grows the support lib with the C++ writer wrapper + writer test. + +if(NOT PAIMON_ENABLE_TANTIVY) + return() +endif() + +set(PAIMON_TANTIVY_SUPPORT_SRCS + tantivy_defs.cpp + tantivy_ffi_log.cpp + tantivy_archive_layout.cpp + tantivy_stream_ctx.cpp + tantivy_global_index_writer.cpp + tantivy_global_index_reader.cpp + tantivy_global_index.cpp + tantivy_global_index_factory.cpp) + +add_paimon_lib(paimon_tantivy_support + SOURCES + ${PAIMON_TANTIVY_SUPPORT_SRCS} + DEPENDENCIES + paimon_shared + paimon_tantivy_ffi + STATIC_LINK_LIBS + paimon_tantivy_ffi + arrow + glog + fmt + SHARED_LINK_LIBS + paimon_shared + SHARED_LINK_FLAGS + ${PAIMON_VERSION_SCRIPT_FLAGS}) +# Corrosion's paimon_tantivy_ffi target carries INTERFACE_INCLUDE_DIRECTORIES +# (cbindgen-generated header path). The objlib in add_paimon_lib doesn't link +# against deps,so its compile step misses include dirs.Wire them explicitly. +target_link_libraries(paimon_tantivy_support_objlib PUBLIC paimon_tantivy_ffi) + +# In test builds, bake the jieba dict dir into GetJiebaDictionaryDirFromEnv so +# tests don't have to setenv() process-wide state. Mirrors the lucene module. +if(PAIMON_BUILD_TESTS) + target_compile_definitions(paimon_tantivy_support_objlib + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") +endif() + +if(PAIMON_BUILD_TESTS) + add_paimon_test(tantivy_smoke_test + SOURCES + tantivy_smoke_test.cpp + STATIC_LINK_LIBS + paimon_tantivy_ffi + ${GTEST_LINK_TOOLCHAIN}) + + add_paimon_test(tantivy_ffi_test + SOURCES + tantivy_ffi_test.cpp + STATIC_LINK_LIBS + paimon_shared + "-Wl,--whole-archive" + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + + # Golden-sample tokenizer diff (cppjieba vs jieba-rs). Links against the + # lucene index module to reuse JiebaTokenizer::CutWithMode + Normalize, so it + # can only be built when lucene-fts is enabled (the C++ JiebaTokenizer lives + # in the lucene module). Guarded so the default LUCENE=OFF / TANTIVY=ON build + # doesn't try to link the non-existent paimon_lucene_index_static. + # Note: we mirror the lucene-fts test's link line (see lucene/CMakeLists.txt) + # rather than using the `jieba` imported target, whose INTERFACE_INCLUDE + # concatenates two paths in one string (upstream quirk). + if(PAIMON_ENABLE_LUCENE) + add_paimon_test(tantivy_tokenizer_test + SOURCES + tantivy_tokenizer_test.cpp + EXTRA_INCLUDES + ${LUCENE_INCLUDE_DIR} + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_lucene_index_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-tokenizer-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}" + PAIMON_TANTIVY_GOLDEN_DIR="${CMAKE_SOURCE_DIR}/test/test_data/tokenizer_golden" + ) + target_include_directories(paimon-tantivy-tokenizer-test SYSTEM + PRIVATE ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}) + endif() + + # Stage 4 — Writer test. Builds an Arrow batch, runs the writer through + # GlobalIndexFileManager + LocalFileSystem, then validates the packed + # on-disk format. Reader round-trip lives in Stage 6. + add_paimon_test(tantivy_writer_test + SOURCES + tantivy_writer_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-writer-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # Stage 6 — Reader + 5 query types end-to-end. + add_paimon_test(tantivy_filter_limit_test + SOURCES + tantivy_filter_limit_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-filter-limit-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # Java → C++ cross-read test. Fixture produced by paimon-java's + # `TantivyIndexFixtureGen` (see docs/dev/tantivy_java_cross_read_plan.md) + # and checked in under test/test_data/java_tantivy_fixtures/. + add_paimon_test(tantivy_java_compat_test + SOURCES + tantivy_java_compat_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-java-compat-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}" + PAIMON_TANTIVY_JAVA_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/test/test_data/java_tantivy_fixtures" + PAIMON_TANTIVY_CPP_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/test/test_data/cpp_tantivy_fixtures" + ) + + # K4 — V3 streaming reader + W1 streaming writer integration coverage: + # ParseArchiveHeader fuzz, concurrent query on shared reader, concurrent + # reader create+drop lifecycle, streaming benchmark log. + add_paimon_test(tantivy_streaming_test + SOURCES + tantivy_streaming_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-streaming-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # Stage 8 — TantivyGlobalIndex + factory + end-to-end integration test. + # `--whole-archive` is required so the static REGISTER_PAIMON_FACTORY + # symbols are not stripped out of the test binary. + add_paimon_test(tantivy_index_test + SOURCES + tantivy_index_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-index-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # Stage 9 — Cross-implementation coexistence. Links against BOTH the + # lucene and tantivy support static libs to verify they resolve their + # `REGISTER_PAIMON_FACTORY` registrations side by side and don't + # collide on shared symbols. Only built when lucene-fts is enabled. + if(PAIMON_ENABLE_LUCENE) + add_paimon_test(tantivy_lucene_coexist_test + SOURCES + tantivy_lucene_coexist_test.cpp + EXTRA_INCLUDES + ${LUCENE_INCLUDE_DIR} + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_lucene_index_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-lucene-coexist-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + + # Stage 10 — Equivalence + benchmark. Same link line as the coexist + # test (needs both impls); benchmark output goes to stderr. + add_paimon_test(tantivy_equivalence_test + SOURCES + tantivy_equivalence_test.cpp + EXTRA_INCLUDES + ${LUCENE_INCLUDE_DIR} + STATIC_LINK_LIBS + paimon_shared + test_utils_static + "-Wl,--whole-archive" + paimon_local_file_system_static + paimon_lucene_index_static + paimon_tantivy_support_static + "-Wl,--no-whole-archive" + paimon_tantivy_ffi + arrow + glog + fmt + ${GTEST_LINK_TOOLCHAIN}) + target_compile_definitions(paimon-tantivy-equivalence-test + PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") + endif() +endif() diff --git a/src/paimon/global_index/tantivy/tantivy_archive_layout.cpp b/src/paimon/global_index/tantivy/tantivy_archive_layout.cpp new file mode 100644 index 00000000..21a009b1 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_archive_layout.cpp @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_archive_layout.h" + +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/math.h" +#include "paimon/fs/file_system.h" +#include "paimon/io/data_input_stream.h" + +namespace paimon::tantivy { + +namespace { + +/// Wrap the (non-owning) raw InputStream* in a shared_ptr-like handle so +/// DataInputStream — which takes `shared_ptr` — can be used +/// without transferring ownership. We use a no-op deleter to avoid double-free. +struct NoopDeleter { + void operator()(InputStream*) const {} +}; + +} // namespace + +Result ArchiveLayout::Parse(InputStream* in) { + if (in == nullptr) { + return Status::Invalid("ArchiveLayout::Parse: null input stream"); + } + + // DataInputStream defaults to BE — matches paimon-java archive format. + std::shared_ptr wrapped(in, NoopDeleter{}); + DataInputStream dis(wrapped); + + PAIMON_RETURN_NOT_OK(dis.Seek(0)); + + PAIMON_ASSIGN_OR_RAISE(int32_t file_count, dis.ReadValue()); + if (!InRange(file_count)) { + return Status::Invalid(fmt::format("ArchiveLayout::Parse: bad file_count {}", file_count)); + } + + ArchiveLayout layout; + layout.count = static_cast(file_count); + layout.names.reserve(layout.count); + layout.offsets.reserve(layout.count); + layout.lengths.reserve(layout.count); + + for (int32_t i = 0; i < file_count; ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t name_len, dis.ReadValue()); + if (name_len <= 0 || name_len > 1 << 20) { + return Status::Invalid( + fmt::format("ArchiveLayout::Parse: bad name_len {} at entry {}", name_len, i)); + } + std::string name(static_cast(name_len), '\0'); + PAIMON_RETURN_NOT_OK(dis.Read(name.data(), static_cast(name_len))); + + PAIMON_ASSIGN_OR_RAISE(int64_t data_len, dis.ReadValue()); + if (!InRange(data_len)) { + return Status::Invalid( + fmt::format("ArchiveLayout::Parse: bad data_len {} for '{}'", data_len, name)); + } + + PAIMON_ASSIGN_OR_RAISE(int64_t data_offset, dis.GetPos()); + + layout.names.push_back(std::move(name)); + layout.offsets.push_back(static_cast(data_offset)); + layout.lengths.push_back(static_cast(data_len)); + + // Skip past the payload without reading it. + PAIMON_RETURN_NOT_OK(dis.Seek(data_offset + data_len)); + } + + return layout; +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_archive_layout.h b/src/paimon/global_index/tantivy/tantivy_archive_layout.h new file mode 100644 index 00000000..73cb3f7f --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_archive_layout.h @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { +class InputStream; +} // namespace paimon + +namespace paimon::tantivy { + +/// Parsed layout of a packed tantivy archive. Arrays are parallel; `count` is +/// their common length. +/// +/// Archive byte format (matches paimon-java `TantivyFullTextGlobalIndexReader. +/// parseArchiveHeader`; big-endian, no version header): +/// `[BE i32 file_count | (BE i32 name_len, name_utf8, BE i64 data_len, data)*]` +/// +/// `offsets[i]` is the archive-absolute byte offset of file `i`'s payload +/// (points past the per-entry header). `lengths[i]` is the payload size. +struct ArchiveLayout { + std::vector names; + std::vector offsets; + std::vector lengths; + std::size_t count = 0; + + /// Read the archive header from `in` (seeking past payloads) and return the + /// layout. Does NOT read file payloads — only header bytes (a few KB). + /// + /// `in` must support `Seek` (all production `paimon::InputStream` subclasses + /// do; we call `Seek(cur + data_len)` to skip over each file's payload). + /// + /// On return, `in`'s internal position is at the end of the archive; callers + /// typically don't care (the stream is subsequently read via pread callbacks). + static Result Parse(InputStream* in); +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_defs.cpp b/src/paimon/global_index/tantivy/tantivy_defs.cpp new file mode 100644 index 00000000..142d727e --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_defs.cpp @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_defs.h" + +#include + +namespace paimon::tantivy { + +std::optional GetJiebaDictionaryDirFromEnv() { + const char* env_dir = std::getenv(kJiebaDictDirEnv); + if (env_dir != nullptr && *env_dir != '\0') { + return std::string(env_dir); + } +#ifdef JIEBA_TEST_DICT_DIR + return std::string(JIEBA_TEST_DICT_DIR); +#endif + return std::nullopt; +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_defs.h b/src/paimon/global_index/tantivy/tantivy_defs.h new file mode 100644 index 00000000..f9e1ec13 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_defs.h @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace paimon::tantivy { + +/// Identifier used by GlobalIndexFileWriter::NewFileName to prefix on-disk +/// filenames. Tantivy and lucene file prefixes intentionally differ so a +/// reader can dispatch the right implementation by filename pattern. +static inline const char kIdentifier[] = "tantivy-fulltext"; + +/// Schema field names — fixed to match paimon-java. Callers +/// MUST NOT rename these even though `TantivyGlobalIndexWriter::Create` accepts +/// a `field_name` argument (that argument is used only to extract the correct +/// arrow column; the tantivy schema field name is always `"text"`). +static inline const char kTantivyTextFieldName[] = "text"; +static inline const char kTantivyRowIdFieldName[] = "row_id"; + +/// Option-key prefix consumed by TantivyGlobalIndex. Matches the lucene-fts +/// convention so users can configure both implementations with a uniform +/// "." key style. +static inline const char kOptionKeyPrefix[] = "tantivy-fulltext."; + +/// Buffer size for streaming raw packed bytes from FFI to OutputStream +/// (Writer) and from InputStream into Rust (Reader). +static inline const int32_t kDefaultReadBufferSize = 1024 * 1024; +/// Read buffer size knob for the reader. +static inline const char kTantivyReadBufferSize[] = "read.buffer-size"; + +/// If true, omit term frequencies/positions when indexing (smaller index, but +/// no PhraseQuery support). Default false, mirroring lucene-fts. +static inline const char kTantivyWriteOmitTermFreqAndPositions[] = + "write.omit-term-freq-and-position"; + +/// Env var carrying jieba dictionary directory; consumed by both writer and +/// reader. Same name as lucene-fts: a single env var configures both backends. +static inline const char kJiebaDictDirEnv[] = "PAIMON_JIEBA_DICT_DIR"; + +/// Default tokenize mode if not specified in options. +static inline const char kDefaultJiebaTokenizeMode[] = "mix"; +/// Tokenize mode option key. Values: "mp", "mix", "full", "query". +/// "hmm" is rejected with Unsupported (jieba-rs does not expose standalone HMM). +static inline const char kJiebaTokenizeMode[] = "jieba.tokenize-mode"; + +/// Writer-side tokenizer selector. Values: +/// "default" (default) — tantivy built-in SimpleTokenizer; +/// "paimon_jieba" — jieba-rs CJK tokenizer; opt-in for Chinese workloads +/// "whitespace" / "raw" / "en_stem" — other tantivy built-ins +/// The reader side is schema-driven and auto-dispatches to whatever tokenizer +/// name is baked into the archive, so the default here also determines what +/// paimon-java sees when it cross-reads the archive. +static inline const char kTantivyWriteTokenizer[] = "tantivy.write.tokenizer"; +/// Default tokenizer for writer: tantivy built-in "default" (SimpleTokenizer), +/// chosen so paimon-cpp ↔ paimon-java cross-read works out of the box. +/// Chinese workloads must opt into "paimon_jieba" via kTantivyWriteTokenizer. +static inline const char kDefaultTantivyWriteTokenizer[] = "default"; + +/// Reads the jieba dictionary directory from kJiebaDictDirEnv. Returns the +/// directory when the env var is set and non-empty, otherwise std::nullopt. +/// Shared by the writer and reader so the env-lookup lives in one place; each +/// caller applies its own policy for the missing case (the writer treats it as +/// an error because a jieba index needs a dictionary, while the reader tolerates +/// it because paimon-java archives use the built-in tokenizer and need none). +/// +/// In test builds, falls back to the JIEBA_TEST_DICT_DIR compile-time macro (set +/// on the support objlib) so tests don't have to mutate process-wide env state. +/// Defined in tantivy_defs.cpp (single TU) and mirrors LuceneUtils:: +/// GetJiebaDictionaryDir. +std::optional GetJiebaDictionaryDirFromEnv(); + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp new file mode 100644 index 00000000..33f1e596 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +struct ReaderPair { + std::shared_ptr lucene; + std::shared_ptr tantivy; + std::unique_ptr lucene_root; + std::unique_ptr tantivy_root; +}; + +class TantivyEquivalenceTest : public ::testing::Test { + public: + std::unique_ptr<::ArrowSchema> CreateArrowSchema( + const std::shared_ptr& data_type) const { + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + return c_schema; + } + + GlobalIndexIOMeta WriteOne(const std::string& factory_id, + const std::shared_ptr& data_type, + const std::map& options, + const std::shared_ptr& array, + const std::string& root) { + EXPECT_OK_AND_ASSIGN(auto indexer, GlobalIndexerFactory::Get(factory_id, options)); + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + EXPECT_OK_AND_ASSIGN( + auto writer, + indexer->CreateWriter("f0", CreateArrowSchema(data_type).get(), file_writer, pool_)); + ::ArrowArray c_array; + EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + EXPECT_OK(writer->AddBatch(&c_array, std::move(relative_row_ids))); + EXPECT_OK_AND_ASSIGN(auto metas, writer->Finish()); + return metas[0]; + } + + std::shared_ptr OpenOne(const std::string& factory_id, + const std::shared_ptr& data_type, + const std::map& options, + const GlobalIndexIOMeta& meta, + const std::string& root) { + EXPECT_OK_AND_ASSIGN(auto indexer, GlobalIndexerFactory::Get(factory_id, options)); + auto path_factory = std::make_shared(root); + auto file_reader = std::make_shared(fs_, path_factory); + EXPECT_OK_AND_ASSIGN(auto reader, indexer->CreateReader(CreateArrowSchema(data_type).get(), + file_reader, {meta}, pool_)); + return reader; + } + + /// Build BOTH lucene + tantivy indexes for the same corpus + options. + /// Returns an opened-reader pair plus owning UniqueTestDirectory handles. + ReaderPair WriteAndOpenBoth(const std::shared_ptr& data_type, + const std::shared_ptr& array, + std::map lucene_opts, + const std::map& tantivy_opts) { + auto lroot = paimon::test::UniqueTestDirectory::Create(); + auto troot = paimon::test::UniqueTestDirectory::Create(); + EXPECT_TRUE(lroot && troot); + // lucene requires a tmp directory option; reuse lroot if caller didn't set one. + lucene_opts.emplace("lucene-fts.write.tmp.directory", lroot->Str()); + auto lmeta = WriteOne("lucene-fts", data_type, lucene_opts, array, lroot->Str()); + auto tmeta = WriteOne("tantivy-fulltext", data_type, tantivy_opts, array, troot->Str()); + ReaderPair p; + p.lucene = OpenOne("lucene-fts", data_type, lucene_opts, lmeta, lroot->Str()); + p.tantivy = OpenOne("tantivy-fulltext", data_type, tantivy_opts, tmeta, troot->Str()); + p.lucene_root = std::move(lroot); + p.tantivy_root = std::move(troot); + return p; + } + + static std::set Ids(const std::shared_ptr& result) { + Result br = Status::Invalid("unrecognized result type"); + if (auto scored = std::dynamic_pointer_cast(result)) { + br = scored->GetBitmap(); + } else if (auto plain = std::dynamic_pointer_cast(result)) { + br = plain->GetBitmap(); + } + EXPECT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, std::move(br)); + std::set out; + if (bitmap) { + for (auto it = bitmap->Begin(); it != bitmap->End(); ++it) { + out.insert(static_cast(*it)); + } + } + return out; + } + + /// Run a single FullTextSearch through both readers, return (lucene, tantivy) + /// doc id sets. + std::pair, std::set> RunPair( + const ReaderPair& p, const std::string& q, FullTextSearch::SearchType t, + std::optional limit = std::nullopt, + std::optional filter = std::nullopt) { + auto lr = p.lucene->VisitFullTextSearch( + std::make_shared("f0", limit, q, t, filter)); + auto tr = p.tantivy->VisitFullTextSearch( + std::make_shared("f0", limit, q, t, filter)); + EXPECT_OK_AND_ASSIGN(auto lresult, std::move(lr)); + EXPECT_OK_AND_ASSIGN(auto tresult, std::move(tr)); + return {Ids(lresult), Ids(tresult)}; + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); +}; + +} // namespace + +TEST_F(TantivyEquivalenceTest, EnglishBagOfWordsBattery) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["alpha beta gamma delta"], + ["alpha alpha alpha beta"], + ["beta gamma delta epsilon"], + ["zeta eta theta iota"], + ["alpha gamma epsilon iota"], + ["lone outlier word here"], + ["alpha beta gamma alpha beta"], + ["delta epsilon zeta eta theta"], + ["nothing matches this row"], + ["alpha"] + ])") + .ValueOrDie(); + auto pair = WriteAndOpenBoth(data_type, array, {}, {}); + + struct Case { + std::string query; + FullTextSearch::SearchType type; + }; + std::vector cases = { + {"alpha", FullTextSearch::SearchType::MATCH_ALL}, + {"alpha", FullTextSearch::SearchType::MATCH_ANY}, + {"alpha beta", FullTextSearch::SearchType::MATCH_ALL}, + {"alpha beta", FullTextSearch::SearchType::MATCH_ANY}, + {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ALL}, + {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ANY}, + {"epsilon iota", FullTextSearch::SearchType::MATCH_ALL}, + {"alpha beta gamma", FullTextSearch::SearchType::PHRASE}, + {"beta gamma delta", FullTextSearch::SearchType::PHRASE}, + {"delta epsilon", FullTextSearch::SearchType::PHRASE}, + }; + for (const auto& c : cases) { + auto [l, t] = RunPair(pair, c.query, c.type); + ASSERT_EQ(l, t) << "diverge: query=" << c.query << " type=" << static_cast(c.type); + } +} + +TEST_F(TantivyEquivalenceTest, ChineseQueryModeBattery) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ +["智能助手 AI 模块 开发"], +["智能助手 在 Python 开发 中"], +["AI 助手 开发 框架"], +["智能 模块 技术 实现"], +["发展方向 是 智能 助手"] + ])") + .ValueOrDie(); + std::map lopts = {{"lucene-fts.jieba.tokenize-mode", "query"}}; + std::map topts = { + {"tantivy-fulltext.tantivy.write.tokenizer", "paimon_jieba"}, + {"tantivy-fulltext.jieba.tokenize-mode", "query"}, + }; + auto pair = WriteAndOpenBoth(data_type, array, lopts, topts); + + struct Case { + std::string query; + FullTextSearch::SearchType type; + }; + // Note: jieba is shared (same dictionary), so tokenization should agree + // for plain Chinese text. Differences (if any) come from the lowercase / + // stopword normalization step — tested with neutral CJK terms below. + std::vector cases = { + {"智能", FullTextSearch::SearchType::MATCH_ALL}, + {"智能 助手", FullTextSearch::SearchType::MATCH_ALL}, + {"模块", FullTextSearch::SearchType::MATCH_ANY}, + {"发展方向", FullTextSearch::SearchType::PHRASE}, + }; + for (const auto& c : cases) { + auto [l, t] = RunPair(pair, c.query, c.type); + ASSERT_EQ(l, t) << "diverge: query=" << c.query << " type=" << static_cast(c.type); + } +} + +TEST_F(TantivyEquivalenceTest, PreFilterIntersectionEquivalent) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["alpha beta"], + ["alpha gamma"], + ["alpha delta"], + ["beta gamma"], + ["beta delta"] + ])") + .ValueOrDie(); + auto pair = WriteAndOpenBoth(data_type, array, {}, {}); + + auto pf = RoaringBitmap64::From({0l, 2l, 4l}); + { + auto [l, t] = + RunPair(pair, "alpha", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, pf); + ASSERT_EQ(l, t); + ASSERT_EQ(l, (std::set{0, 2})); + } + { + auto [l, t] = + RunPair(pair, "beta gamma", FullTextSearch::SearchType::MATCH_ANY, std::nullopt, pf); + ASSERT_EQ(l, t); + } + { + auto empty = RoaringBitmap64(); + auto [l, t] = + RunPair(pair, "alpha", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, empty); + ASSERT_EQ(l, t); + ASSERT_TRUE(l.empty()); + } +} + +TEST_F(TantivyEquivalenceTest, BenchmarkBuildAndQuery) { + // Build a synthetic 200-doc corpus and time write + 100 random queries. + // This is a reportable baseline, NOT a perf gate — assertions only check + // semantic correctness (each query returns >= 0 docs without erroring). + constexpr int32_t kDocCount = 200; + constexpr int32_t kQueryCount = 100; + std::vector vocab = {"alpha", "beta", "gamma", "delta", "epsilon", + "zeta", "eta", "theta", "iota", "kappa", + "lambda", "mu", "nu", "xi", "omicron"}; + std::mt19937 rng(0xC0DE); + std::uniform_int_distribution word_pick(0, vocab.size() - 1); + std::uniform_int_distribution word_count(3, 12); + + // Build the corpus as a JSON Arrow array. + std::string json = "["; + for (int32_t i = 0; i < kDocCount; ++i) { + json += "[\""; + int32_t n = word_count(rng); + for (int32_t w = 0; w < n; ++w) { + if (w > 0) { + json += ' '; + } + json += vocab[word_pick(rng)]; + } + json += "\"]"; + if (i + 1 < kDocCount) { + json += ","; + } + } + json += "]"; + + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, json).ValueOrDie(); + + auto time_ms = [](auto&& fn) { + auto t0 = std::chrono::steady_clock::now(); + fn(); + auto t1 = std::chrono::steady_clock::now(); + return std::chrono::duration_cast(t1 - t0).count(); + }; + + // -------- Lucene: write + open + queries -------- + auto lroot = paimon::test::UniqueTestDirectory::Create(); + std::map lopt = {{"lucene-fts.write.tmp.directory", lroot->Str()}}; + GlobalIndexIOMeta lmeta{"", 0, nullptr}; + auto lwrite_ms = + time_ms([&] { lmeta = WriteOne("lucene-fts", data_type, lopt, array, lroot->Str()); }); + auto lreader = OpenOne("lucene-fts", data_type, lopt, lmeta, lroot->Str()); + + auto lquery_ms = time_ms([&] { + for (int32_t i = 0; i < kQueryCount; ++i) { + const std::string& w = vocab[word_pick(rng)]; + ASSERT_OK_AND_ASSIGN( + auto r, + lreader->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, w, FullTextSearch::SearchType::MATCH_ALL, std::nullopt))); + } + }); + + // -------- Tantivy: write + open + queries -------- + auto troot = paimon::test::UniqueTestDirectory::Create(); + GlobalIndexIOMeta tmeta{"", 0, nullptr}; + auto twrite_ms = + time_ms([&] { tmeta = WriteOne("tantivy-fulltext", data_type, {}, array, troot->Str()); }); + auto treader = OpenOne("tantivy-fulltext", data_type, {}, tmeta, troot->Str()); + + auto tquery_ms = time_ms([&] { + for (int32_t i = 0; i < kQueryCount; ++i) { + const std::string& w = vocab[word_pick(rng)]; + ASSERT_OK_AND_ASSIGN( + auto r, + treader->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, w, FullTextSearch::SearchType::MATCH_ALL, std::nullopt))); + } + }); + + std::cerr << fmt::format( + "[STAGE10-BENCH docs={} queries={}] lucene_write={}ms lucene_query={}ms" + " tantivy_write={}ms tantivy_query={}ms file_size_lucene={} file_size_tantivy={}\n", + kDocCount, kQueryCount, lwrite_ms, lquery_ms, twrite_ms, tquery_ms, lmeta.file_size, + tmeta.file_size); + SUCCEED() << "benchmark prints to stderr"; +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_handle.h b/src/paimon/global_index/tantivy/tantivy_ffi_handle.h new file mode 100644 index 00000000..3c8ecac2 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_handle.h @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy { + +/// Deleter template; specialize per handle type with the matching free function. +/// Usage: +/// template <> struct FfiDeleter { +/// void operator()(paimon_tantivy_writer_t* p) const noexcept { +/// paimon_tantivy_writer_free(p); +/// } +/// }; +/// using WriterPtr = FfiUniquePtr; +template +struct FfiDeleter { + // Default unsupported so missing specializations fail at compile time + void operator()(Handle*) const noexcept { + static_assert(sizeof(Handle) == 0, "FfiDeleter must be specialized for this handle type"); + } +}; + +/// Generic RAII owning pointer for an FFI handle. +template +using FfiUniquePtr = std::unique_ptr>; + +/// Tokenizer handle. +template <> +struct FfiDeleter { + void operator()(PaimonJiebaTokenizer* p) const noexcept { + paimon_tantivy_tokenizer_free(p); + } +}; +using JiebaTokenizerPtr = FfiUniquePtr; + +/// Writer handle. +template <> +struct FfiDeleter { + void operator()(PaimonTantivyWriter* p) const noexcept { + paimon_tantivy_writer_free(p); + } +}; +using WriterPtr = FfiUniquePtr; + +/// Reader handle. +template <> +struct FfiDeleter { + void operator()(PaimonTantivyReader* p) const noexcept { + paimon_tantivy_reader_free(p); + } +}; +using ReaderPtr = FfiUniquePtr; + +/// Specialization: buffer_t is special - not an opaque handle but a value +/// struct owned on the stack. The contained `data` pointer is the Rust-owned +/// allocation; we call `paimon_tantivy_buffer_free` on the struct pointer. +/// Use BufferGuard to ensure free-on-scope-exit even on early return. +class BufferGuard { + public: + BufferGuard() noexcept { + buf_.data = nullptr; + buf_.len = 0; + buf_.capacity = 0; + } + BufferGuard(const BufferGuard&) = delete; + BufferGuard& operator=(const BufferGuard&) = delete; + BufferGuard(BufferGuard&&) = delete; + BufferGuard& operator=(BufferGuard&&) = delete; + + ~BufferGuard() noexcept { + paimon_tantivy_buffer_free(&buf_); + } + + PaimonTantivyBuffer* out() noexcept { + return &buf_; + } + + const uint8_t* data() const noexcept { + return buf_.data; + } + std::size_t size() const noexcept { + return buf_.len; + } + + private: + PaimonTantivyBuffer buf_{}; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_log.cpp b/src/paimon/global_index/tantivy/tantivy_ffi_log.cpp new file mode 100644 index 00000000..151cc1e9 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_log.cpp @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_ffi_log.h" + +#include +#include + +#include "glog/logging.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy { + +// Not in an anonymous namespace: `extern "C"` (external linkage) and an +// anonymous namespace (internal linkage) are contradictory. Only referenced +// via function pointer below, so symbol visibility is not a concern. +/// Level mapping matches Rust side (0=trace..4=error). +extern "C" void PaimonTantivyLogAdapter(int32_t level, const char* msg, std::size_t len) { + // msg is NOT null-terminated; slice with len. + std::string s(msg, len); + switch (level) { + case 4: + LOG(ERROR) << "[tantivy] " << s; + break; + case 3: + LOG(WARNING) << "[tantivy] " << s; + break; + case 2: + LOG(INFO) << "[tantivy] " << s; + break; + case 1: + VLOG(1) << "[tantivy] " << s; + break; + case 0: + VLOG(2) << "[tantivy] " << s; + break; + default: + LOG(INFO) << "[tantivy:lvl=" << level << "] " << s; + break; + } +} + +void InstallTantivyLogBridge() { + paimon_tantivy_set_log_callback(&PaimonTantivyLogAdapter); +} + +void UninstallTantivyLogBridge() { + paimon_tantivy_clear_log_callback(); +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_log.h b/src/paimon/global_index/tantivy/tantivy_ffi_log.h new file mode 100644 index 00000000..7fad3299 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_log.h @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +namespace paimon::tantivy { + +/// Install the Rust -> C++ log callback. Idempotent; only the last caller's +/// callback is active. Threading: C callback runs on tantivy worker threads; +/// our adapter must be thread-safe (it routes to glog which is). +void InstallTantivyLogBridge(); + +/// Uninstall (revert to Rust stderr). Mostly useful for tests. +void UninstallTantivyLogBridge(); + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_status.h b/src/paimon/global_index/tantivy/tantivy_ffi_status.h new file mode 100644 index 00000000..74c0fd3e --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_status.h @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include "fmt/format.h" +#include "paimon/status.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy { + +/// Translate an FFI status code to a paimon::Status. OK returns Status::OK(). +/// On error, the returned Status carries the thread-local last_error() text +/// prefixed with the status code name for easier grep. +/// +/// Note: cbindgen emits `PaimonTantivyStatus` in the **global** namespace as +/// a C-style enum, so we accept it via its global type here. C++ ADL still +/// lets call sites write the unqualified enumerator names. +inline Status FfiStatusToStatus(::PaimonTantivyStatus code) { + if (code == PAIMON_TANTIVY_STATUS_OK) { + return Status::OK(); + } + const char* err = paimon_tantivy_last_error(); + const char* name = [code]() -> const char* { + switch (code) { + case PAIMON_TANTIVY_STATUS_INVALID_ARGUMENT: + return "InvalidArgument"; + case PAIMON_TANTIVY_STATUS_NOT_FOUND: + return "NotFound"; + case PAIMON_TANTIVY_STATUS_IO_ERROR: + return "IoError"; + case PAIMON_TANTIVY_STATUS_UNSUPPORTED: + return "Unsupported"; + case PAIMON_TANTIVY_STATUS_TOKENIZER_ERROR: + return "TokenizerError"; + case PAIMON_TANTIVY_STATUS_QUERY_PARSE_ERROR: + return "QueryParseError"; + case PAIMON_TANTIVY_STATUS_INDEX_FORMAT_ERROR: + return "IndexFormatError"; + case PAIMON_TANTIVY_STATUS_INTERNAL_ERROR: + return "InternalError"; + default: + return "UnknownFfiStatus"; + } + }(); + std::string msg = fmt::format("tantivy-ffi[{}({})]: {}", name, static_cast(code), + err ? err : "(null)"); + switch (code) { + case PAIMON_TANTIVY_STATUS_NOT_FOUND: + return Status::NotExist(msg); + case PAIMON_TANTIVY_STATUS_IO_ERROR: + return Status::IOError(msg); + case PAIMON_TANTIVY_STATUS_UNSUPPORTED: + return Status::NotImplemented(msg); + case PAIMON_TANTIVY_STATUS_INVALID_ARGUMENT: + case PAIMON_TANTIVY_STATUS_TOKENIZER_ERROR: + case PAIMON_TANTIVY_STATUS_QUERY_PARSE_ERROR: + case PAIMON_TANTIVY_STATUS_INDEX_FORMAT_ERROR: + return Status::Invalid(msg); + default: + return Status::UnknownError(msg); + } +} + +/// Like PAIMON_RETURN_NOT_OK but for FFI calls returning PaimonTantivyStatus. +#define PAIMON_TANTIVY_RETURN_NOT_OK(expr) \ + do { \ + ::PaimonTantivyStatus _paimon_tantivy_status_ = (expr); \ + if (_paimon_tantivy_status_ != PAIMON_TANTIVY_STATUS_OK) { \ + return ::paimon::tantivy::FfiStatusToStatus(_paimon_tantivy_status_); \ + } \ + } while (0) + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_ffi_test.cpp b/src/paimon/global_index/tantivy/tantivy_ffi_test.cpp new file mode 100644 index 00000000..a10f3231 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_ffi_test.cpp @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/global_index/tantivy/tantivy_ffi_handle.h" +#include "paimon/global_index/tantivy/tantivy_ffi_log.h" +#include "paimon/global_index/tantivy/tantivy_ffi_status.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy::test { + +// ------------------------- last_error contract ------------------------- + +TEST(TantivyFfiError, LastErrorIsNeverNull) { + // Before anything, last_error should be a valid non-null pointer to "" + const char* ptr = paimon_tantivy_last_error(); + ASSERT_NE(ptr, nullptr); + // Content is thread-local; for freshly-spawned thread it must be empty + std::atomic child_ok{false}; + std::thread t([&]() { + const char* p = paimon_tantivy_last_error(); + child_ok.store(p != nullptr && p[0] == '\0'); + }); + t.join(); + ASSERT_TRUE(child_ok.load()); +} + +// ------------------------- status translation ------------------------- + +TEST(TantivyFfiStatus, OkTranslates) { + Status s = FfiStatusToStatus(PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_OK); + ASSERT_TRUE(s.ok()) << s.ToString(); +} + +TEST(TantivyFfiStatus, ErrorCodeNamesShowUp) { + // Translate a few codes and ensure the name appears in the string form. + struct Case { + PaimonTantivyStatus code; + const char* expected_substr; + }; + const Case cases[] = { + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_INVALID_ARGUMENT, "InvalidArgument"}, + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_NOT_FOUND, "NotFound"}, + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_IO_ERROR, "IoError"}, + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_UNSUPPORTED, "Unsupported"}, + {PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_TOKENIZER_ERROR, "TokenizerError"}, + }; + for (const auto& c : cases) { + Status s = FfiStatusToStatus(c.code); + ASSERT_FALSE(s.ok()); + ASSERT_NE(s.ToString().find(c.expected_substr), std::string::npos) + << "got: " << s.ToString(); + } +} + +// ------------------------- buffer lifetime ------------------------- + +TEST(TantivyFfiBuffer, EmptyBufferGuard) { + BufferGuard g; + ASSERT_EQ(g.size(), 0u); + ASSERT_EQ(g.data(), nullptr); + // Destructor must accept empty buffer +} + +// ------------------------- handle stress ------------------------- + +// Sanity stress: create/destroy a dummy "handle" via into_handle/free_handle. +// Since the Rust side doesn't yet export writer/reader, we stress via a +// temporary wrapping of the buffer API: alloc buffers repeatedly, ensure no +// crash (LSAN / ASAN would catch leaks). +TEST(TantivyFfiBuffer, StressAllocFree) { + for (int32_t i = 0; i < 1000; ++i) { + BufferGuard g; + // We don't have a way to populate the buffer from C++ here; + // this just exercises empty construction + destruction path. + (void)g; + } +} + +// ------------------------- log bridge ------------------------- + +namespace { +std::atomic g_log_count{0}; +extern "C" void CountingLogCb(int32_t /*level*/, const char* /*msg*/, std::size_t /*len*/) { + g_log_count.fetch_add(1, std::memory_order_relaxed); +} +} // namespace + +TEST(TantivyFfiLog, SetCallbackIsIdempotent) { + g_log_count.store(0); + paimon_tantivy_set_log_callback(&CountingLogCb); + paimon_tantivy_set_log_callback(&CountingLogCb); + paimon_tantivy_clear_log_callback(); + // Should not crash even though called multiple times (idempotent install) + SUCCEED(); +} + +TEST(TantivyFfiLog, InstallBridgeThenUninstall) { + // Bridge to glog; must not crash. + InstallTantivyLogBridge(); + UninstallTantivyLogBridge(); + SUCCEED(); +} + +// ------------------------- version still works ------------------------- + +TEST(TantivyFfi, VersionReachable) { + const char* v = paimon_tantivy_version(); + ASSERT_NE(v, nullptr); + ASSERT_GT(std::strlen(v), 0u); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_filter_limit_test.cpp b/src/paimon/global_index/tantivy/tantivy_filter_limit_test.cpp new file mode 100644 index 00000000..5de18683 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_filter_limit_test.cpp @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +class TantivyFilterLimitTest : public ::testing::Test { + public: + std::pair, GlobalIndexIOMeta> WriteAndOpen( + const std::shared_ptr& array, + const std::map& options) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + EXPECT_TRUE(root_dir); + std::string root = root_dir->Str(); + kept_dirs_.push_back(std::move(root_dir)); + auto path_factory = std::make_shared(root); + auto fm = std::make_shared(fs_, path_factory); + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + EXPECT_OK_AND_ASSIGN(auto writer_res, TantivyGlobalIndexWriter::Create( + "f0", data_type, fm, options, GetDefaultPool())); + auto writer = writer_res; + ::ArrowArray c_array; + EXPECT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + EXPECT_TRUE(writer->AddBatch(&c_array, std::move(relative_row_ids)).ok()); + EXPECT_OK_AND_ASSIGN(auto metas_res, writer->Finish()); + return {fm, metas_res[0]}; + } + + static std::vector BitmapToVec(const RoaringBitmap64& b) { + std::vector ids; + for (auto it = b.Begin(); it != b.End(); ++it) { + ids.push_back(static_cast(*it)); + } + std::sort(ids.begin(), ids.end()); + return ids; + } + + std::shared_ptr DataType() const { + return arrow::struct_({arrow::field("f0", arrow::utf8())}); + } + + protected: + std::shared_ptr fs_ = std::make_shared(); + std::vector> kept_dirs_; +}; + +} // namespace + +TEST_F(TantivyFilterLimitTest, LimitProducesScoredResultTopN) { + // Three docs with very different term frequencies for "doc"; limit=2 must + // pick the top 2 by score (doc 1 highest, then doc 2). + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], + ["doc doc doc doc doc"], + ["doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/2, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = true; // v0.2: explicit score opt-in + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored) << "expected BitmapScoredGlobalIndexResult"; + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + auto ids = BitmapToVec(*bitmap); + ASSERT_EQ(ids, (std::vector{1, 2})); + ASSERT_EQ(scored->GetScores().size(), 2u); + // Per-doc scores must be > 0 and present in iteration (doc-id) order. + for (auto s : scored->GetScores()) { + ASSERT_GT(s, 0.0f); + } +} + +TEST_F(TantivyFilterLimitTest, NoLimitReturnsBitmapResult) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["other"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN( + auto res, reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, "doc", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt))); + // No limit ⇒ NOT a BitmapScoredGlobalIndexResult; just BitmapGlobalIndexResult. + ASSERT_FALSE(std::dynamic_pointer_cast(res)); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 1})); +} + +TEST_F(TantivyFilterLimitTest, PreFilterIntersectsWithoutLimit) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["alpha"], ["alpha"], ["alpha"], ["beta"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + ASSERT_OK_AND_ASSIGN( + auto res, reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, "alpha", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/RoaringBitmap64::From({0l, 2l, 100l})))); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 2})); +} + +TEST_F(TantivyFilterLimitTest, PreFilterAppliedBeforeLimit) { + // doc 0 has highest score for "doc" but is excluded by pre_filter; the + // result must contain doc 1 only, even with limit=10. + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc doc doc doc doc"], + ["doc doc"], + ["doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/10, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/RoaringBitmap64::From({1l})); + fts->with_score = true; // v0.2: explicit score opt-in + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{1})); + ASSERT_EQ(scored->GetScores().size(), 1u); +} + +TEST_F(TantivyFilterLimitTest, EmptyPreFilterReturnsEmpty) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["alpha"], ["beta"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + RoaringBitmap64 empty; // explicitly empty + ASSERT_OK_AND_ASSIGN( + auto res, reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, "alpha", FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/empty))); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + ASSERT_TRUE(bitmap->IsEmpty()); +} + +TEST_F(TantivyFilterLimitTest, LimitGreaterThanMatchesReturnsAll) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["other"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/100, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = true; // v0.2: explicit score opt-in + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 1})); + ASSERT_EQ(scored->GetScores().size(), 2u); +} + +// =========================================================================== +// v0.2: with_score × limit 4-path matrix guards +// =========================================================================== +// Decouple with_score from limit. The four combinations must each map to the +// correct concrete result type and content. + +// Path A: with_score=false, limit=None → BitmapGlobalIndexResult, all rows, no score. +TEST_F(TantivyFilterLimitTest, WithScoreFalseLimitNoneAllRowsNoScore) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["doc doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/std::nullopt, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = false; + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + // Must NOT be scored. + ASSERT_FALSE(std::dynamic_pointer_cast(res)); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 1, 2})); +} + +// Path B: with_score=false, limit=N → BitmapGlobalIndexResult, any N matches, +// no scoring (no BM25 sort). Used by `WHERE MATCH ... LIMIT N` without ORDER BY. +TEST_F(TantivyFilterLimitTest, WithScoreFalseLimitNAnyNNoScore) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], + ["doc doc doc doc doc"], + ["doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/2, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = false; + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + // Must NOT be scored. + ASSERT_FALSE(std::dynamic_pointer_cast(res)); + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, plain->GetBitmap()); + // Only cardinality matters — selection order is arbitrary and depends on + // tantivy's posting iteration; the two returned row_ids must each be one + // of the three input docs. + ASSERT_EQ(bitmap->Cardinality(), 2u); + auto vec = BitmapToVec(*bitmap); + for (auto id : vec) { + ASSERT_TRUE(id == 0 || id == 1 || id == 2); + } +} + +// Path C (new in v0.2): with_score=true, limit=None → BitmapScoredGlobalIndexResult, +// all rows + all scores, ordered by row_id asc. +TEST_F(TantivyFilterLimitTest, WithScoreTrueLimitNoneAllRowsWithScore) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["doc doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/std::nullopt, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = true; + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored) << "with_score=true must produce BitmapScoredGlobalIndexResult"; + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + ASSERT_EQ(BitmapToVec(*bitmap), (std::vector{0, 1, 2})); + // All 3 docs have scores; sizes must match. + ASSERT_EQ(scored->GetScores().size(), 3u); + for (auto s : scored->GetScores()) { + ASSERT_GT(s, 0.0f); + } +} + +// Path D: with_score=true, limit=N → BitmapScoredGlobalIndexResult, top-N with scores. +// Equivalent to the v0.1 happy-path (LimitProducesScoredResultTopN), kept here +// as an explicit anchor of the 4-path matrix. +TEST_F(TantivyFilterLimitTest, WithScoreTrueLimitNTopNWithScore) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], + ["doc doc doc doc doc"], + ["doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + auto fts = std::make_shared("f0", /*limit=*/2, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + fts->with_score = true; + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + auto scored = std::dynamic_pointer_cast(res); + ASSERT_TRUE(scored); + ASSERT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, scored->GetBitmap()); + ASSERT_EQ(bitmap->Cardinality(), 2u); + ASSERT_TRUE(bitmap->Contains(1)); // highest TF must be included + ASSERT_EQ(scored->GetScores().size(), 2u); +} + +// Migration guard: when caller omits `with_score`, the default is `false` — +// even with limit set, the result is a BitmapGlobalIndexResult (NOT scored). +// This catches v0.1 callers that relied on `limit >= 0` to implicitly get scores. +TEST_F(TantivyFilterLimitTest, WithScoreDefaultIsFalse) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(DataType(), R"([ + ["doc"], ["doc doc"], ["doc doc doc"] + ])") + .ValueOrDie(); + auto [fm, meta] = WriteAndOpen(array, {}); + ASSERT_OK_AND_ASSIGN(auto reader, + TantivyGlobalIndexReader::Create("f0", meta, fm, {}, GetDefaultPool())); + // Note: NOT setting fts->with_score; relying on the default value. + auto fts = std::make_shared("f0", /*limit=*/2, "doc", + FullTextSearch::SearchType::MATCH_ALL, + /*pre_filter=*/std::nullopt); + ASSERT_OK_AND_ASSIGN(auto res, reader->VisitFullTextSearch(fts)); + // v0.2 contract: with_score defaults to false, so even with limit set the + // result is BitmapGlobalIndexResult (NOT BitmapScoredGlobalIndexResult). + ASSERT_FALSE(std::dynamic_pointer_cast(res)) + << "v0.2: limit alone must NOT imply scoring; with_score=true is required"; + auto plain = std::dynamic_pointer_cast(res); + ASSERT_TRUE(plain); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.cpp b/src/paimon/global_index/tantivy/tantivy_global_index.cpp new file mode 100644 index 00000000..77980448 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index.cpp @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_global_index.h" + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" + +namespace paimon::tantivy { + +#define CHECK_NOT_NULL(pointer, error_msg) \ + do { \ + if (!(pointer)) { \ + return Status::Invalid(error_msg); \ + } \ + } while (0) + +TantivyGlobalIndex::TantivyGlobalIndex(const std::map& options) + : options_(OptionsUtils::FetchOptionsWithPrefix(kOptionKeyPrefix, options)) {} + +Result> TantivyGlobalIndex::CreateWriter( + const std::string& field_name, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_type, + arrow::ImportType(arrow_schema)); + auto struct_type = std::dynamic_pointer_cast(arrow_type); + CHECK_NOT_NULL(struct_type, + "arrow schema must be struct type when create TantivyGlobalIndexWriter"); + auto index_field = struct_type->GetFieldByName(field_name); + CHECK_NOT_NULL( + index_field, + fmt::format("field {} not exist in arrow schema when create TantivyGlobalIndexWriter", + field_name)); + if (index_field->type()->id() != arrow::Type::type::STRING) { + return Status::Invalid("field type must be string"); + } + return TantivyGlobalIndexWriter::Create(field_name, arrow_type, file_writer, options_, pool); +} + +Result> TantivyGlobalIndex::CreateReader( + ::ArrowSchema* c_arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, const std::shared_ptr& pool) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, + arrow::ImportSchema(c_arrow_schema)); + if (files.size() != 1) { + return Status::Invalid(fmt::format( + "tantivy index only has one index file per shard, now num: {}", files.size())); + } + if (arrow_schema->num_fields() != 1) { + return Status::Invalid("TantivyGlobalIndex now only support one field"); + } + auto index_field = arrow_schema->field(0); + if (index_field->type()->id() != arrow::Type::type::STRING) { + return Status::Invalid("field type must be string"); + } + return TantivyGlobalIndexReader::Create(index_field->name(), files[0], file_reader, options_, + pool); +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.h b/src/paimon/global_index/tantivy/tantivy_global_index.h new file mode 100644 index 00000000..747a2156 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" + +namespace paimon::tantivy { + +/// `GlobalIndexer` implementation backed by tantivy-fulltext. Counterpart to +/// `LuceneGlobalIndex`; the two coexist and are NOT cross-readable. Selection +/// between them happens at the factory layer via the `index_type` identifier. +class TantivyGlobalIndex : public GlobalIndexer { + public: + explicit TantivyGlobalIndex(const std::map& options); + + Result> CreateWriter( + const std::string& field_name, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const override; + + Result> CreateReader( + ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, + const std::shared_ptr& pool) const override; + + private: + /// Options after the `tantivy-fulltext.` prefix has been stripped. + std::map options_; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_factory.cpp b/src/paimon/global_index/tantivy/tantivy_global_index_factory.cpp new file mode 100644 index 00000000..654fa54e --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_factory.cpp @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_global_index_factory.h" + +#include +#include +#include +#include + +#include "paimon/factories/factory.h" +#include "paimon/global_index/tantivy/tantivy_global_index.h" + +namespace paimon::tantivy { + +/// Identifier convention: lucene-fts uses "lucene-fts-global"; we use +/// "tantivy-fulltext-global" so `GlobalIndexerFactory::Get("tantivy-fulltext", ...)` +/// (which appends "-global") routes to us. Keeps both backends discoverable +/// via the same lookup path. +const char TantivyGlobalIndexFactory::IDENTIFIER[] = "tantivy-fulltext-global"; + +Result> TantivyGlobalIndexFactory::Create( + const std::map& options) const { + return std::make_unique(options); +} + +REGISTER_PAIMON_FACTORY(TantivyGlobalIndexFactory); + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_factory.h b/src/paimon/global_index/tantivy/tantivy_global_index_factory.h new file mode 100644 index 00000000..e9f9863d --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_factory.h @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" + +namespace paimon::tantivy { + +/// Factory for creating tantivy-fulltext global indexers. Registered into +/// `FactoryCreator` via `REGISTER_PAIMON_FACTORY` so it is selectable +/// alongside `lucene-fts-global` by passing `index_type = "tantivy-fulltext"` +/// (the suffix `-global` is appended automatically by +/// `GlobalIndexerFactory::Get`). +class TantivyGlobalIndexFactory : public GlobalIndexerFactory { + public: + static const char IDENTIFIER[]; + + const char* Identifier() const override { + return IDENTIFIER; + } + + Result> Create( + const std::map& options) const override; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_reader.cpp b/src/paimon/global_index/tantivy/tantivy_global_index_reader.cpp new file mode 100644 index 00000000..fd357cb4 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_reader.cpp @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/tantivy/tantivy_archive_layout.h" +#include "paimon/global_index/tantivy/tantivy_ffi_log.h" +#include "paimon/global_index/tantivy/tantivy_ffi_status.h" +#include "paimon/global_index/tantivy/tantivy_stream_ctx.h" + +namespace paimon::tantivy { + +namespace { + +// One-shot install of the Rust log bridge so log::warn! in Rust surfaces via glog. +void EnsureTantivyLogBridge() { + static std::once_flag flag; + std::call_once(flag, [] { InstallTantivyLogBridge(); }); +} + +} // namespace + +Result> TantivyGlobalIndexReader::Create( + const std::string& field_name, const GlobalIndexIOMeta& io_meta, + const std::shared_ptr& file_reader, + const std::map& options, const std::shared_ptr& pool) { + (void)field_name; // Rust-side knows the field via the schema embedded in meta.json + EnsureTantivyLogBridge(); + + std::map write_options; + if (io_meta.metadata) { + PAIMON_RETURN_NOT_OK(RapidJsonUtil::FromJsonString( + std::string(io_meta.metadata->data(), io_meta.metadata->size()), &write_options)); + } + + PAIMON_ASSIGN_OR_RAISE( + std::string tokenize_mode, + OptionsUtils::GetValueFromMap(options, kJiebaTokenizeMode, std::string(""))); + if (tokenize_mode.empty()) { + // Reader-side option not set; look at the (possibly empty) write_options blob. + // When write_options is empty (paimon-java-written archive), the value below is + // a placeholder that satisfies FFI validation but is discarded at runtime — + // see the comment block above. Do NOT treat the placeholder as a real default + // for jieba indices; jieba archives written by paimon-cpp always stamp their + // chosen mode into metadata, so the placeholder branch never applies to them. + PAIMON_ASSIGN_OR_RAISE( + tokenize_mode, OptionsUtils::GetValueFromMap(write_options, kJiebaTokenizeMode, + std::string(kDefaultJiebaTokenizeMode))); + } + PAIMON_ASSIGN_OR_RAISE( + bool omit_term_freq_and_positions, + OptionsUtils::GetValueFromMap(write_options, kTantivyWriteOmitTermFreqAndPositions, false)); + + // Tolerate a missing jieba dict dir on the read path: paimon-java archives + // use the built-in "default" tokenizer and need no dictionary, so the Rust + // reader ignores dict_dir for them. Archives that do use jieba still get an + // actionable error from the Rust side when the dictionary path is empty. + std::string dict_dir = GetJiebaDictionaryDirFromEnv().value_or(std::string()); + + // Streaming read path: + // 1) open stream + // 2) ArchiveLayout::Parse — reads only header bytes, seeks past payloads + // 3) wrap stream in StreamCtx (owned by Rust via release callback) + // 4) build PaimonStreamCallbacks → paimon_tantivy_reader_new_streaming + // Archive payloads are read lazily through read_at callbacks as tantivy + // accesses posting lists, meta.json, etc. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stream, + file_reader->GetInputStream(io_meta.file_path)); + PAIMON_ASSIGN_OR_RAISE(ArchiveLayout layout, ArchiveLayout::Parse(stream.get())); + + // Transfer stream ownership to a heap-allocated StreamCtx. Ownership passes + // to Rust as soon as paimon_tantivy_reader_new_streaming is called: Rust + // `paimon_cpp_stream_release(ctx)`s it on any failure and on reader drop, so + // C++ must NOT release it after that call. + auto* stream_ctx = new StreamCtx{std::move(stream), {}}; + PaimonStreamCallbacks callbacks{ + static_cast(stream_ctx), + paimon_cpp_stream_read_at, + paimon_cpp_stream_release, + }; + + // Build C-string array pointing into layout.names (stable during this call). + std::vector name_ptrs; + name_ptrs.reserve(layout.count); + for (const auto& n : layout.names) { + name_ptrs.push_back(n.c_str()); + } + + PaimonTantivyReader* raw = nullptr; + ::PaimonTantivyStatus st = paimon_tantivy_reader_new_streaming( + name_ptrs.data(), layout.offsets.data(), layout.lengths.data(), layout.count, callbacks, + tokenize_mode.c_str(), + /*with_position=*/!omit_term_freq_and_positions, dict_dir.c_str(), &raw); + if (st != PAIMON_TANTIVY_STATUS_OK) { + // Rust already released stream_ctx on the failure path; do not release + // it again here (that would double-free the StreamCtx). + PAIMON_TANTIVY_RETURN_NOT_OK(st); + } + return std::shared_ptr( + new TantivyGlobalIndexReader(ReaderPtr(raw), pool)); +} + +Result> TantivyGlobalIndexReader::VisitFullTextSearch( + const std::shared_ptr& full_text_search) { + if (!full_text_search) { + return Status::Invalid("VisitFullTextSearch: null FullTextSearch pointer"); + } + + // Serialize pre_filter (if any) to croaring portable bytes for FFI. + // NB: Serialize() returns a pooled_unique_ptr with MemoryPool::AllocatorDelete; + // converting via raw.release() + shared_ptr(raw_ptr) would substitute + // std::default_delete, causing an alloc/dealloc mismatch (malloc vs operator + // delete). Move directly into shared_ptr so the pooled deleter is preserved + // in the control block. + PAIMON_UNIQUE_PTR pre_filter_bytes_owned; + const char* pre_filter_ptr = nullptr; + std::size_t pre_filter_len = 0; + if (full_text_search->pre_filter.has_value()) { + pre_filter_bytes_owned = full_text_search->pre_filter.value().Serialize(pool_.get()); + pre_filter_ptr = pre_filter_bytes_owned->data(); + pre_filter_len = pre_filter_bytes_owned->size(); + } + + int32_t limit_arg = full_text_search->limit.has_value() + ? static_cast(full_text_search->limit.value()) + : -1; + + float min_score_arg = + full_text_search->min_score.has_value() ? full_text_search->min_score.value() : 0.0f; + + BufferGuard out; + PaimonTantivyStatus st = paimon_tantivy_reader_search( + reader_.get(), static_cast(full_text_search->search_type), + full_text_search->query.data(), full_text_search->query.size(), + full_text_search->with_score, limit_arg, pre_filter_ptr, pre_filter_len, min_score_arg, + out.out()); + PAIMON_TANTIVY_RETURN_NOT_OK(st); + + // Decode `[u8 has_scores | u64 count | u64 row_ids[] | optional f32 scores[]]`. + // row_id is the explicit u64 column read from the fast field. + if (out.size() < 9) { + return Status::Invalid( + fmt::format("tantivy reader output too small ({} bytes)", out.size())); + } + const uint8_t* p = out.data(); + bool has_scores = (p[0] != 0); + // v0.2 consistency check: the wire-level has_scores byte must match the caller's + // with_score flag. A mismatch would indicate FFI / wire-protocol drift. + if (has_scores != full_text_search->with_score) { + return Status::Invalid(fmt::format( + "tantivy wire protocol mismatch: caller with_score={} but buffer has_scores={}", + full_text_search->with_score, has_scores)); + } + uint64_t count; + std::memcpy(&count, p + 1, sizeof(uint64_t)); + std::size_t expected = 1 + 8 + count * 8 + (has_scores ? count * 4 : 0); + if (out.size() != expected) { + return Status::Invalid(fmt::format( + "tantivy reader output size mismatch: has_scores={} count={} expected {} bytes, got {}", + has_scores, count, expected, out.size())); + } + + const uint8_t* row_id_p = p + 9; + if (!has_scores) { + // Copy the contiguous u64 block (unaligned at offset 9) into an aligned + // buffer, then bulk-insert via AddMany instead of a per-row Add loop. + std::vector row_ids(count); + if (count > 0) { + std::memcpy(row_ids.data(), row_id_p, count * sizeof(uint64_t)); + } + RoaringBitmap64 bitmap; + bitmap.AddMany(row_ids.size(), row_ids.data()); + return std::make_shared( + [b = std::move(bitmap)]() -> Result { return b; }); + } + // has_scores=true: produce BitmapScoredGlobalIndexResult. Rust may send rows + // in either row_id-asc order (path C: with_score=true, limit=None) or score-desc + // order (path D: with_score=true, limit=Some). The bitmap iteration order is + // row_id-asc (RoaringBitmap set semantics), so we always re-sort by row_id here + // to keep `scores[i]` aligned with the i-th row_id from the bitmap iterator — + // matching the contract documented in BitmapScoredGlobalIndexResult. + const uint8_t* score_p = row_id_p + count * 8; + std::vector> id_score_pairs; + id_score_pairs.reserve(count); + for (uint64_t i = 0; i < count; i++) { + uint64_t row_id; + std::memcpy(&row_id, row_id_p + i * 8, sizeof(uint64_t)); + float score; + std::memcpy(&score, score_p + i * 4, sizeof(float)); + id_score_pairs.emplace_back(static_cast(row_id), score); + } + // Sort by row_id ascending so scores align with bitmap iteration order. + std::sort(id_score_pairs.begin(), id_score_pairs.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + RoaringBitmap64 bitmap; + std::vector scores; + scores.reserve(id_score_pairs.size()); + for (const auto& [id, sc] : id_score_pairs) { + bitmap.Add(id); + scores.push_back(sc); + } + return std::make_shared(std::move(bitmap), std::move(scores)); +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_reader.h b/src/paimon/global_index/tantivy/tantivy_global_index_reader.h new file mode 100644 index 00000000..fcfdc455 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_reader.h @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/io/global_index_file_reader.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_ffi_handle.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/full_text_search.h" + +namespace paimon::tantivy { + +/// Tantivy-backed implementation of `GlobalIndexReader`. +/// +/// Mirrors LuceneGlobalIndexReader's surface but delegates query construction +/// + execution into Rust over FFI. Supports the 5 FullTextSearch SearchTypes +/// (MATCH_ALL, MATCH_ANY, PHRASE, PREFIX, WILDCARD), optionally with limit and +/// pre_filter. +/// +/// All non-FullTextSearch visit methods return nullptr (matches +/// LuceneGlobalIndexReader): the FTS index has no contribution for non-FTS +/// predicates, framework treats nullptr as "no filter constraint". +class TantivyGlobalIndexReader : public GlobalIndexReader { + public: + static Result> Create( + const std::string& field_name, const GlobalIndexIOMeta& io_meta, + const std::shared_ptr& file_reader, + const std::map& options, const std::shared_ptr& pool); + + // === FunctionVisitor surface — non-FTS predicates fall back to full range. === + + Result> VisitIsNotNull() override { + return CreateAllResult(); + } + Result> VisitIsNull() override { + return CreateAllResult(); + } + Result> VisitEqual(const Literal&) override { + return CreateAllResult(); + } + Result> VisitNotEqual(const Literal&) override { + return CreateAllResult(); + } + Result> VisitLessThan(const Literal&) override { + return CreateAllResult(); + } + Result> VisitLessOrEqual(const Literal&) override { + return CreateAllResult(); + } + Result> VisitGreaterThan(const Literal&) override { + return CreateAllResult(); + } + Result> VisitGreaterOrEqual(const Literal&) override { + return CreateAllResult(); + } + Result> VisitIn(const std::vector&) override { + return CreateAllResult(); + } + Result> VisitNotIn(const std::vector&) override { + return CreateAllResult(); + } + Result> VisitStartsWith(const Literal&) override { + return CreateAllResult(); + } + Result> VisitEndsWith(const Literal&) override { + return CreateAllResult(); + } + Result> VisitContains(const Literal&) override { + return CreateAllResult(); + } + Result> VisitLike(const Literal&) override { + return CreateAllResult(); + } + + Result> VisitVectorSearch( + const std::shared_ptr&) override { + return Status::Invalid( + "TantivyGlobalIndexReader is not supposed to handle vector search query"); + } + + Result> VisitFullTextSearch( + const std::shared_ptr& full_text_search) override; + + bool IsThreadSafe() const override { + return false; + } + + std::string GetIndexType() const override { + return kIdentifier; + } + + private: + TantivyGlobalIndexReader(ReaderPtr reader, std::shared_ptr pool) + : reader_(std::move(reader)), pool_(std::move(pool)) {} + + std::shared_ptr CreateAllResult() const { + return nullptr; + } + + /// Owning handle to the Rust-side reader. + ReaderPtr reader_; + /// MemoryPool used for serializing pre-filter bitmaps to bytes for FFI. + std::shared_ptr pool_; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp b/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp new file mode 100644 index 00000000..ea697fa1 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" + +#include +#include +#include + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/global_index/global_index_utils.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/global_index/tantivy/tantivy_ffi_status.h" +#include "paimon/global_index/tantivy/tantivy_stream_ctx.h" + +namespace paimon::tantivy { + +#define CHECK_NOT_NULL(pointer, error_msg) \ + do { \ + if (!(pointer)) { \ + return Status::Invalid(error_msg); \ + } \ + } while (0) + +Result> TantivyGlobalIndexWriter::Create( + const std::string& field_name, const std::shared_ptr& arrow_type, + const std::shared_ptr& file_writer, + const std::map& options, const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE( + bool omit_term_freq_and_positions, + OptionsUtils::GetValueFromMap(options, kTantivyWriteOmitTermFreqAndPositions, false)); + PAIMON_ASSIGN_OR_RAISE(std::string tokenize_mode, + OptionsUtils::GetValueFromMap(options, kJiebaTokenizeMode, + std::string(kDefaultJiebaTokenizeMode))); + PAIMON_ASSIGN_OR_RAISE(std::string tokenizer, OptionsUtils::GetValueFromMap( + options, kTantivyWriteTokenizer, + std::string(kDefaultTantivyWriteTokenizer))); + // Jieba dict is only needed when actually using jieba. For tantivy built-in + // tokenizers (e.g. "default") we don't force the caller to ship the jieba + // dict dir — pass an empty string and Rust skips jieba construction. + std::string dict_dir; + if (tokenizer == "paimon_jieba") { + std::optional env_dir = GetJiebaDictionaryDirFromEnv(); + if (!env_dir.has_value()) { + return Status::Invalid(fmt::format( + "jieba dictionary dir not found, please set {} env var", kJiebaDictDirEnv)); + } + dict_dir = std::move(*env_dir); + } + + PaimonTantivyWriter* raw = nullptr; + PaimonTantivyStatus st = paimon_tantivy_writer_new( + field_name.c_str(), tokenize_mode.c_str(), + /*with_position=*/!omit_term_freq_and_positions, dict_dir.c_str(), tokenizer.c_str(), &raw); + PAIMON_TANTIVY_RETURN_NOT_OK(st); + WriterPtr writer(raw); + return std::shared_ptr(new TantivyGlobalIndexWriter( + field_name, arrow_type, std::move(writer), file_writer, options, pool)); +} + +TantivyGlobalIndexWriter::TantivyGlobalIndexWriter( + const std::string& field_name, const std::shared_ptr& arrow_type, + WriterPtr writer, const std::shared_ptr& file_writer, + const std::map& options, const std::shared_ptr& pool) + : pool_(pool), + field_name_(field_name), + arrow_type_(arrow_type), + writer_(std::move(writer)), + file_writer_(file_writer), + options_(options) {} + +Status TantivyGlobalIndexWriter::AddBatch(::ArrowArray* arrow_array, + std::vector&& relative_row_ids) { + // First-element check mirrors lucene; trust caller to feed sequential ids + // within a batch (same contract LuceneGlobalIndexWriter relies on). + PAIMON_RETURN_NOT_OK( + GlobalIndexUtils::CheckRelativeRowIds(arrow_array, relative_row_ids, row_id_)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + arrow::ImportArray(arrow_array, arrow_type_)); + auto struct_array = std::dynamic_pointer_cast(array); + CHECK_NOT_NULL(struct_array, + "invalid input array in TantivyGlobalIndexWriter, must be struct array"); + auto field_array = struct_array->GetFieldByName(field_name_); + CHECK_NOT_NULL( + field_array, + fmt::format("invalid input array in TantivyGlobalIndexWriter, field {} not in input array", + field_name_)); + auto string_array = std::dynamic_pointer_cast(field_array); + CHECK_NOT_NULL(string_array, + fmt::format("invalid input array in TantivyGlobalIndexWriter, field array {} " + "is not a string array", + field_name_)); + + for (int64_t i = 0; i < string_array->length(); i++) { + const char* text_ptr = nullptr; + size_t text_len = 0; + if (!string_array->IsNull(i)) { + std::string_view view = string_array->Value(i); + text_ptr = view.data(); + text_len = view.size(); + } + // Pass the caller-tracked row_id as an explicit u64 field. + PaimonTantivyStatus st = paimon_tantivy_writer_add( + writer_.get(), static_cast(row_id_), text_ptr, text_len); + PAIMON_TANTIVY_RETURN_NOT_OK(st); + row_id_++; + } + return Status::OK(); +} + +Result> TantivyGlobalIndexWriter::Finish() { + // Streaming finish: open the output file, pipe archive bytes from Rust + // through `paimon_cpp_writer_push` directly into the OutputStream. Peak + // RAM (Rust side) = one fixed streaming buffer, independent of archive size. + PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, file_writer_->NewFileName(kIdentifier)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, + file_writer_->NewOutputStream(index_file_name)); + + WriteCtx ctx{out.get(), Status::OK()}; + PaimonWriteCallbacks cb{ + static_cast(&ctx), + paimon_cpp_writer_push, + }; + + int64_t rust_row_count = 0; + ::PaimonTantivyStatus st = + paimon_tantivy_writer_finish_streaming(writer_.get(), cb, &rust_row_count); + if (st != PAIMON_TANTIVY_STATUS_OK) { + // Prefer the detailed C++-side Status stashed by the write callback + // (if the failure originated there); fall back to FFI-derived status. + if (!ctx.last_error.ok()) { + return ctx.last_error; + } + PAIMON_TANTIVY_RETURN_NOT_OK(st); + } + if (rust_row_count != row_id_) { + return Status::Invalid( + fmt::format("tantivy writer row count {} mismatch paimon inner row count {}", + rust_row_count, row_id_)); + } + + PAIMON_RETURN_NOT_OK(out->Flush()); + PAIMON_RETURN_NOT_OK(out->Close()); + + PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name)); + std::string options_json; + PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); + auto meta_bytes = std::make_shared(options_json, pool_.get()); + GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, + /*metadata=*/meta_bytes); + return std::vector({meta}); +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_writer.h b/src/paimon/global_index/tantivy/tantivy_global_index_writer.h new file mode 100644 index 00000000..0d3247ab --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_global_index_writer.h @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/io/global_index_file_writer.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_ffi_handle.h" + +namespace paimon::tantivy { + +/// Tantivy-backed implementation of GlobalIndexWriter. +/// +/// Mirrors LuceneGlobalIndexWriter's lifecycle: +/// Create() → AddBatch()* → Finish() +/// Each shard produces exactly one .index file via the GlobalIndexFileWriter, +/// containing the full packed tantivy on-disk index in a single contiguous blob. +/// +/// Indexes written by this class are NOT cross-readable with lucene-fts. The +/// C++ side of this writer is intentionally thin: index construction, segment +/// merging, and packing all happen in Rust behind the FFI boundary. +class TantivyGlobalIndexWriter : public GlobalIndexWriter { + public: + static Result> Create( + const std::string& field_name, const std::shared_ptr& arrow_type, + const std::shared_ptr& file_writer, + const std::map& options, const std::shared_ptr& pool); + + ~TantivyGlobalIndexWriter() override = default; + + Status AddBatch(::ArrowArray* arrow_array, std::vector&& relative_row_ids) override; + + Result> Finish() override; + + private: + TantivyGlobalIndexWriter(const std::string& field_name, + const std::shared_ptr& arrow_type, WriterPtr writer, + const std::shared_ptr& file_writer, + const std::map& options, + const std::shared_ptr& pool); + + std::shared_ptr pool_; + std::string field_name_; + std::shared_ptr arrow_type_; + /// Owning handle to the Rust-side writer. + WriterPtr writer_; + std::shared_ptr file_writer_; + std::map options_; + /// Last document index processed (matches caller-passed relative_row_ids). + int64_t row_id_ = 0; +}; + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_index_test.cpp b/src/paimon/global_index/tantivy/tantivy_index_test.cpp new file mode 100644 index 00000000..199e347b --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_index_test.cpp @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index.h" +#include "paimon/global_index/tantivy/tantivy_global_index_factory.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +class TantivyGlobalIndexIntegrationTest : public ::testing::Test { + public: + std::unique_ptr<::ArrowSchema> CreateArrowSchema( + const std::shared_ptr& data_type) const { + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + return c_schema; + } + + Result WriteGlobalIndex(const std::string& root, + const std::shared_ptr& data_type, + const std::map& options, + const std::shared_ptr& array, + int64_t /*unused_expected_range_end*/) const { + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr w, + global_index->CreateWriter("f0", CreateArrowSchema(data_type).get(), + file_writer, pool_)); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + PAIMON_RETURN_NOT_OK(w->AddBatch(&c_array, std::move(relative_row_ids))); + PAIMON_ASSIGN_OR_RAISE(auto metas, w->Finish()); + EXPECT_EQ(metas.size(), 1u); + auto file_name = PathUtil::GetName(metas[0].file_path); + EXPECT_TRUE(StringUtils::StartsWith(file_name, "tantivy-fulltext-global-index-")) + << file_name; + EXPECT_TRUE(StringUtils::EndsWith(file_name, ".index")); + EXPECT_TRUE(metas[0].metadata); + return metas[0]; + } + + Result> CreateReader( + const std::string& root, const std::shared_ptr& data_type, + const std::map& options, const GlobalIndexIOMeta& meta) const { + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(root); + auto file_reader = std::make_shared(fs_, path_factory); + return global_index->CreateReader(CreateArrowSchema(data_type).get(), file_reader, {meta}, + pool_); + } + + void CheckResult(const std::shared_ptr& result, + const std::vector& expected_ids) const { + const RoaringBitmap64* bitmap = nullptr; + if (auto scored = std::dynamic_pointer_cast(result)) { + ASSERT_OK_AND_ASSIGN(bitmap, scored->GetBitmap()); + ASSERT_EQ(scored->GetScores().size(), expected_ids.size()); + } else if (auto plain = std::dynamic_pointer_cast(result)) { + ASSERT_OK_AND_ASSIGN(bitmap, plain->GetBitmap()); + } + ASSERT_TRUE(bitmap); + ASSERT_EQ(*bitmap, RoaringBitmap64::From(expected_ids)) + << "result=" << bitmap->ToString() + << ", expected=" << RoaringBitmap64::From(expected_ids).ToString(); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); + std::shared_ptr data_type_ = + arrow::struct_({arrow::field("f0", arrow::utf8())}); +}; + +} // namespace + +TEST_F(TantivyGlobalIndexIntegrationTest, EnglishCorpus) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options = { + {"tantivy-fulltext.write.omit-term-freq-and-position", "false"}, + }; + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["This is an test document."], + ["This is an new document document document."], + ["Document document document document test."], + ["unordered user-defined doc id"] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto meta, WriteGlobalIndex(root, data_type_, options, array, 3)); + ASSERT_EQ(std::string(meta.metadata->data(), meta.metadata->size()), + R"({"write.omit-term-freq-and-position":"false"})"); + + ASSERT_OK_AND_ASSIGN(auto reader, CreateReader(root, data_type_, options, meta)); + auto t_reader = std::dynamic_pointer_cast(reader); + ASSERT_TRUE(t_reader); + ASSERT_EQ(t_reader->GetIndexType(), std::string(kIdentifier)); + + auto run = [&](const std::string& q, FullTextSearch::SearchType t, + std::optional limit = std::nullopt, + std::optional filter = std::nullopt) { + // Use scored path so `limit` returns top-N by BM25, matching test + // expectations (otherwise unscored Path B returns any-N, non-deterministic). + auto fts = std::make_shared("f0", limit, q, t, filter); + fts->with_score = true; + EXPECT_OK_AND_ASSIGN(auto res, t_reader->VisitFullTextSearch(fts)); + return res; + }; + + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL, 10), {2, 1, 0}); + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ANY, 1), {2}); + CheckResult(run("test document", FullTextSearch::SearchType::MATCH_ALL, 10), {2, 0}); + CheckResult(run("test new", FullTextSearch::SearchType::MATCH_ANY, 10), {1, 0, 2}); + CheckResult(run("test document", FullTextSearch::SearchType::PHRASE, 10), {0}); + CheckResult(run("unordered", FullTextSearch::SearchType::MATCH_ALL, 10), {3}); + CheckResult(run("unorder", FullTextSearch::SearchType::PREFIX, 10), {3}); + CheckResult(run("*order*", FullTextSearch::SearchType::WILDCARD, 10), {3}); + CheckResult(run("*or*er*", FullTextSearch::SearchType::WILDCARD, 10), {3}); + + // pre_filter + CheckResult( + run("document", FullTextSearch::SearchType::MATCH_ALL, 10, RoaringBitmap64::From({0l, 1l})), + {0, 1}); + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL, 10, + RoaringBitmap64::From({2l, 100l})), + {2}); + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL, 10, + RoaringBitmap64::From({20l, 100l})), + {}); + + // No limit + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL), {0, 1, 2}); + CheckResult(run("document", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, + RoaringBitmap64::From({2l})), + {2}); + CheckResult(run("document test", FullTextSearch::SearchType::MATCH_ALL, std::nullopt, + RoaringBitmap64::From({1l, 2l, 3l, 100l})), + {2}); + + // Unscored path: no with_score ⇒ BitmapGlobalIndexResult (not scored), same + // matches across all 5 SearchTypes. (Previously covered by reader_test.) + auto run_unscored = [&](const std::string& q, FullTextSearch::SearchType t) { + EXPECT_OK_AND_ASSIGN(auto res, + t_reader->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, q, t, std::nullopt))); + EXPECT_FALSE(std::dynamic_pointer_cast(res)) + << "unscored query must not return a scored result"; + return res; + }; + CheckResult(run_unscored("document", FullTextSearch::SearchType::MATCH_ALL), {0, 1, 2}); + CheckResult(run_unscored("test new", FullTextSearch::SearchType::MATCH_ANY), {0, 1, 2}); + CheckResult(run_unscored("test document", FullTextSearch::SearchType::PHRASE), {0}); + CheckResult(run_unscored("unorder", FullTextSearch::SearchType::PREFIX), {3}); + CheckResult(run_unscored("*order*", FullTextSearch::SearchType::WILDCARD), {3}); +} + +TEST_F(TantivyGlobalIndexIntegrationTest, ChineseCorpus) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options = { + {"tantivy-fulltext.write.omit-term-freq-and-position", "false"}, + {"tantivy-fulltext.tantivy.write.tokenizer", "paimon_jieba"}, + {"tantivy-fulltext.jieba.tokenize-mode", "query"}, + }; + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ +["QianWen 是一个基于 AI 的智能助手，类似于 Siri 和 Alexa。我们正在用 Python 开发 QianWen 的 Natural Language Understanding 模块，该模块支持多轮对话和意图识别功能，是新一代智能助手的核心技术之一。"], +["最近开源了一个新项目叫ｑｉａｎｗｅｎ（全角字符），功能类似之前的 Qianwen，是一个面向 AI 应用的智能助手。它不仅支持 Machine Learning 和 NLP 技术，还提供了可扩展的开发框架，便于开发者构建自己的智能助手系统。"], +["我们在测试 qianwen-core v1.2 和 ai-engine-alpha 中的 bug，重点优化了 qianwen 的响应速度和稳定性。本次更新增强了核心模块的功能，提升了智能助手的开发效率，并修复了与 NLP 模块相关的多个问题。"], +["AI 助手开发中常用的技术包括 Speech Recognition、Natural Language Processing 和 Recommendation System。我们使用 TensorFlow 和 PyTorch 构建模型，开发了多个智能助手原型，支持语音交互和上下文理解功能，是当前热门的人工智能发展应用方向。"], +["新一代的 AI 助手代号为「千问」，内部命名为 QianwenX-2024，计划在 next quarter 发布。QianwenX 将集成更强的 multimodel 能力，支持图像和文本联合处理，进一步提升智能助手的理解能力和交互体验，是未来智能助手的重要发展方向。"] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto meta, WriteGlobalIndex(root, data_type_, options, array, 4)); + ASSERT_EQ( + std::string(meta.metadata->data(), meta.metadata->size()), + R"({"jieba.tokenize-mode":"query","tantivy.write.tokenizer":"paimon_jieba","write.omit-term-freq-and-position":"false"})"); + + ASSERT_OK_AND_ASSIGN(auto reader, CreateReader(root, data_type_, options, meta)); + auto t_reader = std::dynamic_pointer_cast(reader); + ASSERT_TRUE(t_reader); + + auto run = [&](const std::string& q, FullTextSearch::SearchType t, + std::optional limit = std::nullopt, + std::optional filter = std::nullopt) { + // Use scored path so `limit` returns top-N by BM25, matching test + // expectations (otherwise unscored Path B returns any-N, non-deterministic). + auto fts = std::make_shared("f0", limit, q, t, filter); + fts->with_score = true; + EXPECT_OK_AND_ASSIGN(auto res, t_reader->VisitFullTextSearch(fts)); + return res; + }; + + CheckResult(run("模块", FullTextSearch::SearchType::MATCH_ALL, 10), {0, 2}); + CheckResult(run("模块", FullTextSearch::SearchType::MATCH_ANY, 1), {0}); + CheckResult(run("模块技术", FullTextSearch::SearchType::MATCH_ALL, 10), {0}); + CheckResult(run("模块技术", FullTextSearch::SearchType::MATCH_ANY, 10), {0, 1, 2, 3}); + CheckResult(run("发展方向", FullTextSearch::SearchType::PHRASE, 10), {4}); + CheckResult(run("模块技术", FullTextSearch::SearchType::MATCH_ANY, 10, + RoaringBitmap64::From({1l, 3l, 4l})), + {1, 3}); + CheckResult(run("模块技术", FullTextSearch::SearchType::MATCH_ANY), {0, 1, 2, 3}); + + // Unscored path on jieba-tokenized Chinese. (Previously covered by reader_test.) + auto run_unscored = [&](const std::string& q, FullTextSearch::SearchType t) { + EXPECT_OK_AND_ASSIGN(auto res, + t_reader->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, q, t, std::nullopt))); + return res; + }; + CheckResult(run_unscored("模块", FullTextSearch::SearchType::MATCH_ALL), {0, 2}); + CheckResult(run_unscored("模块技术", FullTextSearch::SearchType::MATCH_ANY), {0, 1, 2, 3}); + CheckResult(run_unscored("发展方向", FullTextSearch::SearchType::PHRASE), {4}); +} + +TEST_F(TantivyGlobalIndexIntegrationTest, FactoryLookupReturnsTantivyIndexer) { + std::map options = { + {"tantivy-fulltext.jieba.tokenize-mode", "query"}, + }; + // Identifier passed to GlobalIndexerFactory::Get is the prefix; "-global" + // is appended automatically. So "tantivy-fulltext" must route to our factory. + ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, + GlobalIndexerFactory::Get("tantivy-fulltext", options)); + ASSERT_TRUE(indexer); + auto* casted = dynamic_cast(indexer.get()); + ASSERT_TRUE(casted) << "factory did not return a TantivyGlobalIndex"; +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp b/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp new file mode 100644 index 00000000..bd9339e3 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp @@ -0,0 +1,450 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/tantivy/tantivy_archive_layout.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif +#ifndef PAIMON_TANTIVY_JAVA_FIXTURE_DIR +#error "PAIMON_TANTIVY_JAVA_FIXTURE_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FixturePathFactory : public IndexPathFactory { + public: + explicit FixturePathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +class JavaCompatTest : public ::testing::Test { + public: + /// Build a TantivyGlobalIndexReader on top of the Java-produced fixture. + /// `fixture_name` is relative to `PAIMON_TANTIVY_JAVA_FIXTURE_DIR`. + std::shared_ptr OpenFixture(const std::string& fixture_name) { + std::string fixture_dir = PAIMON_TANTIVY_JAVA_FIXTURE_DIR; + std::string archive_path = PathUtil::JoinPath(fixture_dir, fixture_name); + + EXPECT_OK_AND_ASSIGN(auto file_status, fs_->GetFileStatus(archive_path)); + int64_t file_size = file_status->GetLen(); + EXPECT_GT(file_size, 4) << "fixture archive must exist and be > 4 bytes"; + + // Empty metadata (options not needed for cross-read — we use defaults) + std::string metadata_json = "{}"; + auto meta_bytes = std::make_shared(metadata_json, pool_.get()); + + GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes); + + std::map options; + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(fixture_dir); + auto file_reader = std::make_shared(fs_, path_factory); + + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + + EXPECT_OK_AND_ASSIGN(auto reader_res, global_index->CreateReader( + c_schema.get(), file_reader, {io_meta}, pool_)); + return reader_res; + } + + std::shared_ptr BuildFts(FullTextSearch::SearchType type, + const std::string& query) { + return std::make_shared( + /*_field_name=*/"f0", + /*_limit=*/std::optional{}, + /*_query=*/query, + /*_search_type=*/type, + /*_pre_filter=*/std::optional{}); + } + + /// Run the search and return the sorted row_ids from the result bitmap. + std::vector RunSearchRowIds(const std::shared_ptr& reader, + FullTextSearch::SearchType type, + const std::string& query) { + auto fts = BuildFts(type, query); + // This helper returns a value, so gtest ASSERT_* (which `return;`) cannot + // be used here; use the EXPECT_OK family. + EXPECT_OK_AND_ASSIGN(std::shared_ptr r, + reader->VisitFullTextSearch(fts)); + + const RoaringBitmap64* bitmap = nullptr; + if (auto plain = std::dynamic_pointer_cast(r)) { + EXPECT_OK_AND_ASSIGN(bitmap, plain->GetBitmap()); + } else if (auto scored = std::dynamic_pointer_cast(r)) { + EXPECT_OK_AND_ASSIGN(bitmap, scored->GetBitmap()); + } + EXPECT_TRUE(bitmap != nullptr); + if (bitmap == nullptr) { + return {}; + } + + std::vector out; + for (auto it = bitmap->Begin(); it != bitmap->End(); ++it) { + out.push_back(static_cast(*it)); + } + std::sort(out.begin(), out.end()); + return out; + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); +}; + +} // namespace + +// ============================================================================ +// 1. Archive basics: opening the Java-produced fixture succeeds +// ============================================================================ + +TEST_F(JavaCompatTest, OpenJavaArchiveSucceeds) { + auto reader = OpenFixture("english_simple.archive"); + ASSERT_TRUE(reader != nullptr); +} + +// ============================================================================ +// 2. MATCH_ALL — single and multi-term +// ============================================================================ + +TEST_F(JavaCompatTest, MatchAllApple) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, "apple"); + // Docs containing "apple": 0 ("apple banana cherry"), 1 ("apple durian"), + // 4 ("apple cherry fig"), 7 ("apple") + ASSERT_EQ(ids, (std::vector{0, 1, 4, 7})); +} + +TEST_F(JavaCompatTest, MatchAllAppleBananaIntersection) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, "apple banana"); + // Only doc 0 contains both "apple" and "banana" + ASSERT_EQ(ids, (std::vector{0})); +} + +// ============================================================================ +// 3. MATCH_ANY — union +// ============================================================================ + +TEST_F(JavaCompatTest, MatchAnyDurianElderberryUnion) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ANY, "durian elderberry"); + // durian: 1, 6 elderberry: 5, 8 union: {1, 5, 6, 8} + ASSERT_EQ(ids, (std::vector{1, 5, 6, 8})); +} + +// ============================================================================ +// 4. PHRASE — consecutive term order matters +// ============================================================================ + +TEST_F(JavaCompatTest, PhraseAppleBanana) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::PHRASE, "apple banana"); + // Only doc 0 has "apple banana" as consecutive phrase + ASSERT_EQ(ids, (std::vector{0})); +} + +TEST_F(JavaCompatTest, PhraseBananaCherry) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::PHRASE, "banana cherry"); + // "banana cherry" consecutive in doc 0 ("apple banana cherry") and doc 2 ("banana cherry") + ASSERT_EQ(ids, (std::vector{0, 2})); +} + +// ============================================================================ +// 5. PREFIX — byte-level (not tokenized) via RegexQuery +// ============================================================================ + +TEST_F(JavaCompatTest, PrefixAp) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::PREFIX, "ap"); + // Tokens starting with "ap": "apple" → docs 0, 1, 4, 7 + ASSERT_EQ(ids, (std::vector{0, 1, 4, 7})); +} + +// ============================================================================ +// 6. WILDCARD — glob-style via regex +// ============================================================================ + +TEST_F(JavaCompatTest, WildcardErr) { + auto reader = OpenFixture("english_simple.archive"); + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::WILDCARD, "*err*"); + // Tokens matching *err*: "cherry" (0,2,4,6,9), "elderberry" (5,8) + ASSERT_EQ(ids, (std::vector{0, 2, 4, 5, 6, 8, 9})); +} + +// ============================================================================ +// 7. row_id invariant — must return the *caller-supplied* row_ids (not doc_ids) +// ============================================================================ + +TEST_F(JavaCompatTest, AllDocsReachableByRowId) { + auto reader = OpenFixture("english_simple.archive"); + // Union of all terms matches all 10 docs. + auto ids = RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ANY, + "apple banana cherry durian fig grape elderberry"); + ASSERT_EQ(ids, (std::vector{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + // This confirms Java wrote row_ids 0..9 via `addDocument(rowId, text)` and + // paimon-cpp reader extracted them via fast_fields().u64("row_id") — + // the schema invariant survives round-trip across implementations. +} + +// ============================================================================ +// 8. Probe: real paimon-java production archive (handed over by Java team). +// Data was claimed to be (id INT, content STRING) with 5 rows but ids +// rewritten multiple times; dump layout + per-term hits so caller can +// reverse-engineer what's actually inside. +// ============================================================================ + +TEST_F(JavaCompatTest, ProductionSampleProbe) { + const std::string fixture_name = "production_sample.archive"; + + // Open a reader over the Java-written production sample archive. + auto reader = OpenFixture(fixture_name); + ASSERT_TRUE(reader != nullptr); + + // Keywords expected from the production text samples; tokenizer is "default" + // (lowercased, word-granular). + const std::vector probes = { + "apache", "paimon", "is", "a", "lake", "format", "supports", + "full", "text", "search", "in", "vector", "similarity", "using", + "lumina", "streaming", "and", "batch", "processing", "engine", + }; + + // The archive must be readable — at least one probe term hits. + bool any_hit = false; + for (const auto& term : probes) { + if (!RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, term).empty()) { + any_hit = true; + break; + } + } + ASSERT_TRUE(any_hit) << "no probe term hit; archive may be empty or schema mismatched"; +} + +// ============================================================================ +// 9. Reverse direction: paimon-cpp writes with tokenizer="default" → fixture +// consumed by paimon-java test. This test emits the archive into +// test/test_data/cpp_tantivy_fixtures/english_default.archive and +// round-trips it through the cpp reader first (schema-driven tokenizer +// dispatch picks "default" automatically). +// ============================================================================ + +namespace { + +/// GlobalIndexFileWriter that emits to a single fixed filename under `root`. +/// Mirrors paimon-java's `FixedNameLocalFileWriter` from +/// `TantivyIndexFixtureGen.java`: `newFileName(prefix)` ignores the prefix and +/// always returns the caller-chosen name. Used to produce a stable fixture +/// path consumed by the paimon-java cross-read test. +class FixedNameGlobalIndexFileWriter : public GlobalIndexFileWriter { + public: + FixedNameGlobalIndexFileWriter(std::shared_ptr fs, std::string root, + std::string fixed_name) + : fs_(std::move(fs)), root_(std::move(root)), fixed_name_(std::move(fixed_name)) {} + + Result NewFileName(const std::string& /*prefix*/) const override { + return fixed_name_; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + Result> NewOutputStream( + const std::string& file_name) const override { + return fs_->Create(ToPath(file_name), /*overwrite=*/true); + } + Result GetFileSize(const std::string& file_name) const override { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, + fs_->GetFileStatus(ToPath(file_name))); + return file_status->GetLen(); + } + + private: + std::shared_ptr fs_; + std::string root_; + std::string fixed_name_; +}; + +/// Same 10-doc English corpus paimon-java uses in TantivyIndexFixtureGen +/// (pure ASCII, no punctuation inside words). SimpleTokenizer (tantivy's +/// "default") tokenizes identically on both sides for this subset, so the +/// golden row_ids match byte-for-byte between cpp-write and java-read. +constexpr const char* kEnglishDocs[] = { + "apple banana cherry", // 0 + "apple durian", // 1 + "banana cherry", // 2 + "fig grape", // 3 + "apple cherry fig", // 4 + "banana elderberry", // 5 + "cherry durian", // 6 + "apple", // 7 + "grape fig elderberry", // 8 + "cherry fig", // 9 +}; + +} // namespace + +TEST_F(JavaCompatTest, CppWriteDefaultTokenizerForJavaCrossRead) { + // 1) Produce an archive into test/test_data/cpp_tantivy_fixtures/ via the + // production TantivyGlobalIndexWriter, configured with tantivy's + // built-in "default" tokenizer (same as paimon-java's TEXT field). + const std::string out_dir = PAIMON_TANTIVY_CPP_FIXTURE_DIR; + const std::string fixture_name = "english_default.archive"; + // Ensure dir exists (CMake does NOT create it automatically). + ASSERT_OK(fs_->Mkdirs(out_dir)); + // Clean any prior fixture so each test run writes fresh bytes. + { + const std::string archive_path_cleanup = PathUtil::JoinPath(out_dir, fixture_name); + auto existing = fs_->GetFileStatus(archive_path_cleanup); + if (existing.ok()) { + ASSERT_TRUE(fs_->Delete(archive_path_cleanup, false).ok()); + } + } + + auto file_writer = std::make_shared(fs_, out_dir, fixture_name); + + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + + std::map options{ + {kTantivyWriteTokenizer, "default"}, + }; + ASSERT_OK_AND_ASSIGN(auto writer_res, TantivyGlobalIndexWriter::Create( + "f0", data_type, file_writer, options, pool_)); + auto writer = writer_res; + + // Build an arrow batch from kEnglishDocs. + std::string json = "["; + for (std::size_t i = 0; i < sizeof(kEnglishDocs) / sizeof(kEnglishDocs[0]); ++i) { + if (i > 0) { + json += ","; + } + json += "[\""; + json += kEnglishDocs[i]; + json += "\"]"; + } + json += "]"; + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, json).ValueOrDie(); + ::ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + ASSERT_TRUE(writer->AddBatch(&c_array, std::move(relative_row_ids)).ok()); + ASSERT_OK_AND_ASSIGN(auto metas_res, writer->Finish()); + ASSERT_EQ(metas_res.size(), 1u); + const auto& meta = metas_res.front(); + const std::string archive_path = meta.file_path; + + // 2) Archive header sanity: 16+ files, meta.json present, tokenizer in schema. + ASSERT_OK_AND_ASSIGN(std::shared_ptr stream, fs_->Open(archive_path)); + ASSERT_OK_AND_ASSIGN(auto layout_res, ArchiveLayout::Parse(stream.get())); + const auto& layout = layout_res; + bool has_meta_json = false; + for (std::size_t i = 0; i < layout.count; ++i) { + if (layout.names[i] == "meta.json") { + has_meta_json = true; + } + } + ASSERT_TRUE(has_meta_json); + + // 3) Round-trip through the cpp reader first — the reader must auto-register + // "default" from the schema so the search path works without passing + // any reader-side tokenizer config. + // Build a reader directly off the archive path (mirrors OpenFixture + // but rooted at the cpp fixtures dir). + ASSERT_OK_AND_ASSIGN(auto file_status, fs_->GetFileStatus(archive_path)); + int64_t file_size = file_status->GetLen(); + auto meta_bytes = std::make_shared(std::string("{}"), pool_.get()); + GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes); + auto reader_factory = + std::make_shared(std::map{}); + auto reader_path_factory = std::make_shared(out_dir); + auto reader_file_mgr = std::make_shared(fs_, reader_path_factory); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + + ASSERT_OK_AND_ASSIGN(auto reader_res, reader_factory->CreateReader( + c_schema.get(), reader_file_mgr, {io_meta}, pool_)); + auto reader = reader_res; + + // Golden expectations (identical to paimon-java's english_simple.golden.json) + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, "apple"), + (std::vector{0, 1, 4, 7})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ALL, "apple banana"), + (std::vector{0})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ANY, "durian elderberry"), + (std::vector{1, 5, 6, 8})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::PHRASE, "apple banana"), + (std::vector{0})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::PHRASE, "banana cherry"), + (std::vector{0, 2})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::PREFIX, "ap"), + (std::vector{0, 1, 4, 7})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::WILDCARD, "*err*"), + (std::vector{0, 2, 4, 5, 6, 8, 9})); + ASSERT_EQ(RunSearchRowIds(reader, FullTextSearch::SearchType::MATCH_ANY, + "apple banana cherry durian fig grape elderberry"), + (std::vector{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_lucene_coexist_test.cpp b/src/paimon/global_index/tantivy/tantivy_lucene_coexist_test.cpp new file mode 100644 index 00000000..2b6fdeb5 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_lucene_coexist_test.cpp @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_index_reader.h" +#include "paimon/global_index/global_index_writer.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/lucene/lucene_defs.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +/// Adopt one of the two factory identifiers; everything else (paths, queries, +/// arrow plumbing) is shared. +struct ImplSpec { + std::string factory_id; // "lucene-fts" or "tantivy-fulltext" + std::string file_prefix; // "lucene-fts-global-index-" or "tantivy-fulltext-global-index-" + std::string option_prefix; // "lucene-fts." or "tantivy-fulltext." +}; + +class TantivyLuceneCoexistTest : public ::testing::Test { + public: + std::unique_ptr<::ArrowSchema> CreateArrowSchema( + const std::shared_ptr& data_type) const { + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + return c_schema; + } + + Result WriteWith(const ImplSpec& impl, const std::string& root, + const std::shared_ptr& data_type, + const std::map& options, + const std::shared_ptr& array) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(impl.factory_id, options)); + if (!indexer) { + return Status::Invalid(fmt::format("factory returned null for {}", impl.factory_id)); + } + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr w, + indexer->CreateWriter("f0", CreateArrowSchema(data_type).get(), file_writer, pool_)); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + PAIMON_RETURN_NOT_OK(w->AddBatch(&c_array, std::move(relative_row_ids))); + PAIMON_ASSIGN_OR_RAISE(auto metas, w->Finish()); + EXPECT_EQ(metas.size(), 1u); + EXPECT_TRUE( + StringUtils::StartsWith(PathUtil::GetName(metas[0].file_path), impl.file_prefix)) + << metas[0].file_path << " did not start with " << impl.file_prefix; + return metas[0]; + } + + Result> OpenReader( + const ImplSpec& impl, const std::string& root, + const std::shared_ptr& data_type, + const std::map& options, const GlobalIndexIOMeta& meta) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(impl.factory_id, options)); + auto path_factory = std::make_shared(root); + auto file_reader = std::make_shared(fs_, path_factory); + return indexer->CreateReader(CreateArrowSchema(data_type).get(), file_reader, {meta}, + pool_); + } + + static std::set ExtractDocIds(const std::shared_ptr& result) { + Result br = Status::Invalid("no result"); + if (auto scored = std::dynamic_pointer_cast(result)) { + br = scored->GetBitmap(); + } else if (auto plain = std::dynamic_pointer_cast(result)) { + br = plain->GetBitmap(); + } + EXPECT_OK_AND_ASSIGN(const RoaringBitmap64* bitmap, std::move(br)); + std::set out; + if (bitmap) { + for (auto it = bitmap->Begin(); it != bitmap->End(); ++it) { + out.insert(static_cast(*it)); + } + } + return out; + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); + + inline static const ImplSpec kLucene{"lucene-fts", "lucene-fts-global-index-", "lucene-fts."}; + inline static const ImplSpec kTantivy{"tantivy-fulltext", "tantivy-fulltext-global-index-", + "tantivy-fulltext."}; +}; + +} // namespace + +TEST_F(TantivyLuceneCoexistTest, BothFactoriesResolve) { + // No options needed; just verify both factories register and dispatch. + ASSERT_OK_AND_ASSIGN(auto lucene_indexer, GlobalIndexerFactory::Get("lucene-fts", {})); + ASSERT_OK_AND_ASSIGN(auto tantivy_indexer, GlobalIndexerFactory::Get("tantivy-fulltext", {})); + ASSERT_TRUE(lucene_indexer); + ASSERT_TRUE(tantivy_indexer); + // Sanity: factories return distinct types — different vtables → different + // GetIndexType() once we open a reader (not testable here without an + // index), so just check shared_ptr identity differs. + ASSERT_NE(static_cast(lucene_indexer.get()), static_cast(tantivy_indexer.get())); +} + +TEST_F(TantivyLuceneCoexistTest, SideBySideEnglishCorpusReturnsSameDocIds) { + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["alpha beta gamma document"], + ["alpha alpha document"], + ["gamma delta epsilon"], + ["alpha beta document document"] + ])") + .ValueOrDie(); + + auto lucene_root = paimon::test::UniqueTestDirectory::Create(); + auto tantivy_root = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(lucene_root && tantivy_root); + + // Lucene requires a tmp directory option; tantivy ignores unknown keys. + std::map lucene_options = { + {"lucene-fts.write.tmp.directory", lucene_root->Str()}}; + + // Write through BOTH factories side by side in the same process. + ASSERT_OK_AND_ASSIGN(auto lucene_meta, + WriteWith(kLucene, lucene_root->Str(), data_type, lucene_options, array)); + ASSERT_OK_AND_ASSIGN(auto tantivy_meta, + WriteWith(kTantivy, tantivy_root->Str(), data_type, {}, array)); + + ASSERT_OK_AND_ASSIGN(auto lucene_reader, + OpenReader(kLucene, lucene_root->Str(), data_type, {}, lucene_meta)); + ASSERT_OK_AND_ASSIGN(auto tantivy_reader, + OpenReader(kTantivy, tantivy_root->Str(), data_type, {}, tantivy_meta)); + ASSERT_EQ(lucene_reader->GetIndexType(), std::string("lucene-fts")); + ASSERT_EQ(tantivy_reader->GetIndexType(), std::string("tantivy-fulltext")); + + auto run_pair = [&](const std::string& q, FullTextSearch::SearchType t) { + EXPECT_OK_AND_ASSIGN(auto lr, + lucene_reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, q, t, /*pre_filter=*/std::nullopt))); + EXPECT_OK_AND_ASSIGN(auto tr, + tantivy_reader->VisitFullTextSearch(std::make_shared( + "f0", /*limit=*/std::nullopt, q, t, /*pre_filter=*/std::nullopt))); + return std::make_pair(ExtractDocIds(lr), ExtractDocIds(tr)); + }; + + // For an English bag-of-words corpus the two implementations should agree + // on which docs contain which terms — Lucene and tantivy both store + // lowercased word tokens. + { + auto [l, t] = run_pair("document", FullTextSearch::SearchType::MATCH_ALL); + ASSERT_EQ(l, t) << "MATCH_ALL document — lucene vs tantivy doc id set differs"; + ASSERT_EQ(l, (std::set{0, 1, 3})); + } + { + auto [l, t] = run_pair("alpha beta", FullTextSearch::SearchType::MATCH_ALL); + ASSERT_EQ(l, t) << "MATCH_ALL 'alpha beta' — sets differ"; + ASSERT_EQ(l, (std::set{0, 3})); + } + { + auto [l, t] = run_pair("alpha epsilon", FullTextSearch::SearchType::MATCH_ANY); + ASSERT_EQ(l, t) << "MATCH_ANY 'alpha epsilon' — sets differ"; + ASSERT_EQ(l, (std::set{0, 1, 2, 3})); + } + { + auto [l, t] = run_pair("alpha beta", FullTextSearch::SearchType::PHRASE); + ASSERT_EQ(l, t) << "PHRASE 'alpha beta' — sets differ"; + ASSERT_EQ(l, (std::set{0, 3})); + } +} + +TEST_F(TantivyLuceneCoexistTest, IndependentLifecycleNoStateLeakage) { + // Build a lucene index and a tantivy index back-to-back many times in the + // same process; if either factory leaked global state across instances + // we'd see crashes or stale results. + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + + for (int32_t round = 0; round < 3; ++round) { + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + ["round payload one"], + ["round payload two"] + ])") + .ValueOrDie(); + auto lroot = paimon::test::UniqueTestDirectory::Create(); + auto troot = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(lroot && troot); + + std::map lopt = { + {"lucene-fts.write.tmp.directory", lroot->Str()}}; + ASSERT_OK_AND_ASSIGN(auto lm, WriteWith(kLucene, lroot->Str(), data_type, lopt, array)); + ASSERT_OK_AND_ASSIGN(auto tm, WriteWith(kTantivy, troot->Str(), data_type, {}, array)); + ASSERT_OK_AND_ASSIGN(auto lr, OpenReader(kLucene, lroot->Str(), data_type, {}, lm)); + ASSERT_OK_AND_ASSIGN(auto tr, OpenReader(kTantivy, troot->Str(), data_type, {}, tm)); + + ASSERT_OK_AND_ASSIGN(auto lq, lr->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, "payload", + FullTextSearch::SearchType::MATCH_ALL, std::nullopt))); + ASSERT_OK_AND_ASSIGN(auto tq, tr->VisitFullTextSearch(std::make_shared( + "f0", std::nullopt, "payload", + FullTextSearch::SearchType::MATCH_ALL, std::nullopt))); + ASSERT_EQ(ExtractDocIds(lq), (std::set{0, 1})) << "lucene round " << round; + ASSERT_EQ(ExtractDocIds(tq), (std::set{0, 1})) << "tantivy round " << round; + } +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_smoke_test.cpp b/src/paimon/global_index/tantivy/tantivy_smoke_test.cpp new file mode 100644 index 00000000..9fb17822 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_smoke_test.cpp @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "gtest/gtest.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +namespace paimon::tantivy::test { + +TEST(TantivySmoke, VersionIsReachable) { + const char* version = paimon_tantivy_version(); + ASSERT_NE(version, nullptr) << "paimon_tantivy_version returned null"; + + const std::string v(version); + ASSERT_FALSE(v.empty()); + // build.rs pins version from Cargo.toml (CARGO_PKG_VERSION), semver "x.y.z" + ASSERT_NE(v.find('.'), std::string::npos) << "expected semver, got: " << v; +} + +TEST(TantivySmoke, VersionPointerIsStable) { + // The pointer is documented as 'static — two calls should return either + // the same pointer or at least equivalent string content. + const char* v1 = paimon_tantivy_version(); + const char* v2 = paimon_tantivy_version(); + ASSERT_NE(v1, nullptr); + ASSERT_NE(v2, nullptr); + ASSERT_EQ(std::strcmp(v1, v2), 0); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_stream_ctx.cpp b/src/paimon/global_index/tantivy/tantivy_stream_ctx.cpp new file mode 100644 index 00000000..fa12b3de --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_stream_ctx.cpp @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/global_index/tantivy/tantivy_stream_ctx.h" + +#include + +#include "fmt/format.h" +#include "paimon/fs/file_system.h" + +namespace paimon::tantivy { + +extern "C" int32_t paimon_cpp_stream_read_at(void* ctx_ptr, uint64_t offset, std::size_t len, + uint8_t* out_buf) { + if (ctx_ptr == nullptr || out_buf == nullptr) { + return 1; + } + auto* ctx = static_cast(ctx_ptr); + std::lock_guard lock(ctx->pread_mu); + + std::size_t total = 0; + while (total < len) { + auto r = ctx->stream->Read(reinterpret_cast(out_buf + total), + static_cast(len - total), offset + total); + if (!r.ok()) { + return 1; + } + int32_t got = r.value(); + if (got <= 0) { + return 1; // unexpected EOF / 0-byte read + } + total += static_cast(got); + } + return 0; +} + +extern "C" void paimon_cpp_stream_release(void* ctx_ptr) { + if (ctx_ptr == nullptr) { + return; + } + auto* ctx = static_cast(ctx_ptr); + // ~shared_ptr closes the underlying stream. + delete ctx; +} + +extern "C" int32_t paimon_cpp_writer_push(void* ctx_ptr, const uint8_t* data, std::size_t len) { + if (ctx_ptr == nullptr) { + return 1; + } + auto* ctx = static_cast(ctx_ptr); + if (ctx->out == nullptr) { + ctx->last_error = Status::Invalid("writer_push: null OutputStream"); + return 1; + } + std::size_t total = 0; + while (total < len) { + auto r = ctx->out->Write(reinterpret_cast(data + total), + static_cast(len - total)); + if (!r.ok()) { + ctx->last_error = r.status(); + return 1; + } + int32_t written = r.value(); + if (written <= 0) { + ctx->last_error = Status::IOError(fmt::format( + "writer_push: short write (wrote {} of {} bytes)", written, len - total)); + return 1; + } + total += static_cast(written); + } + return 0; +} + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_stream_ctx.h b/src/paimon/global_index/tantivy/tantivy_stream_ctx.h new file mode 100644 index 00000000..a4cb02f7 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_stream_ctx.h @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/status.h" + +namespace paimon { +class InputStream; +class OutputStream; +} // namespace paimon + +namespace paimon::tantivy { + +/// C++ side wrapper around a seekable InputStream, used as the `ctx` of +/// `PaimonStreamCallbacks`. Lifetime is transferred to Rust via +/// `paimon_tantivy_reader_new_streaming`; Rust invokes `paimon_cpp_stream_release` +/// when the reader handle is freed, which `delete`s this struct. +/// +/// `pread_mu` is a defensive per-ctx lock: `InputStream::Read(buffer, size, +/// offset)` is declared pread-style (thread-safe, no position mutation), but a +/// few subclasses (notably `JindoInputStream`) have member-variable races. Rust +/// already serializes reads via its own `stream_mutex`; `pread_mu` is a +/// redundant safeguard at the C++ layer. +struct StreamCtx { + std::shared_ptr stream; + std::mutex pread_mu; +}; + +/// `ctx` of `PaimonWriteCallbacks`. Holds a raw (non-owning) pointer to +/// a paimon `OutputStream` plus a sticky error for conveying write failures +/// back to the C++ caller of `TantivyGlobalIndexWriter::Finish`. +struct WriteCtx { + OutputStream* out = nullptr; + Status last_error = Status::OK(); +}; + +/// Rust -> C++ read callback. Reads `len` bytes starting at archive-absolute +/// `offset` into `out_buf`. Returns 0 on success, 1 on IO error. Thread-safe +/// (serialized via `StreamCtx::pread_mu`; Rust also holds its own mutex). +extern "C" int32_t paimon_cpp_stream_read_at(void* ctx_ptr, uint64_t offset, std::size_t len, + uint8_t* out_buf); + +/// Rust -> C++ release callback. Called exactly once when the Rust reader is +/// dropped. Deletes the ctx (which closes the underlying stream via ~shared_ptr). +extern "C" void paimon_cpp_stream_release(void* ctx_ptr); + +/// Rust -> C++ write push callback. Writes `len` bytes from `data` to the +/// underlying OutputStream. Returns 0 on success, 1 on IO error (with the +/// detailed Status stashed in `WriteCtx::last_error` for the caller to pick up). +extern "C" int32_t paimon_cpp_writer_push(void* ctx_ptr, const uint8_t* data, std::size_t len); + +} // namespace paimon::tantivy diff --git a/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp b/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp new file mode 100644 index 00000000..40283fcb --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/tantivy/tantivy_archive_layout.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index.h" +#include "paimon/global_index/tantivy/tantivy_global_index_reader.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/predicate/full_text_search.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +/// Helper: build an archive with `n` documents, return the GlobalIndexIOMeta. +/// Holds the tmp dir alive (via `holder`) so it's cleaned up when the +/// WriteResult goes out of scope. +struct WriteResult { + std::unique_ptr holder; + std::string root_dir; + GlobalIndexIOMeta meta; +}; + +class StreamingTestFixture : public ::testing::Test { + public: + WriteResult BuildArchive(std::size_t n_docs, + const std::string& text_template = "apple banana cherry {}") { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + EXPECT_TRUE(root_dir); + std::string root = root_dir->Str(); + + // Build arrow StringArray + arrow::StringBuilder sb; + for (std::size_t i = 0; i < n_docs; ++i) { + char buf[128]; + std::snprintf(buf, sizeof(buf), text_template.c_str(), i); + EXPECT_TRUE(sb.Append(buf).ok()); + } + auto text_array = sb.Finish().ValueOrDie(); + auto struct_array = + arrow::StructArray::Make({text_array}, {arrow::field("f0", arrow::utf8())}) + .ValueOrDie(); + + std::map options; + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + EXPECT_OK_AND_ASSIGN(auto w, + global_index->CreateWriter("f0", c_schema.get(), file_writer, pool_)); + ::ArrowArray c_array; + EXPECT_TRUE(arrow::ExportArray(*struct_array, &c_array).ok()); + std::vector relative_row_ids(struct_array->length()); + for (int64_t i = 0; i < struct_array->length(); ++i) { + relative_row_ids[i] = i; + } + EXPECT_OK(w->AddBatch(&c_array, std::move(relative_row_ids))); + EXPECT_OK_AND_ASSIGN(auto metas, w->Finish()); + EXPECT_EQ(metas.size(), 1u); + + // Move root_dir into the result — it stays alive as long as the + // caller holds WriteResult; cleaned up when TEST_F scope exits. + return WriteResult{std::move(root_dir), std::move(root), metas[0]}; + } + + std::shared_ptr OpenReader(const std::string& root, + const GlobalIndexIOMeta& meta) { + std::map options; + auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())}); + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + auto global_index = std::make_shared(options); + auto path_factory = std::make_shared(root); + auto file_reader = std::make_shared(fs_, path_factory); + EXPECT_OK_AND_ASSIGN( + auto reader, global_index->CreateReader(c_schema.get(), file_reader, {meta}, pool_)); + return reader; + } + + std::shared_ptr BuildMatchAll(const std::string& query) { + return std::make_shared( + /*_field_name=*/"f0", + /*_limit=*/std::optional{}, + /*_query=*/query, + /*_search_type=*/FullTextSearch::SearchType::MATCH_ALL, + /*_pre_filter=*/std::optional{}); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); +}; + +// ========================================================================= +// 1. ParseArchiveHeader fuzz +// ========================================================================= + +TEST(ParseArchiveHeaderFuzz, TruncatedHeader) { + // Fewer than 4 bytes → DataInputStream::ReadValue fails + std::string bytes = "\x00\x00"; + ByteArrayInputStream in(bytes.data(), bytes.size()); + ASSERT_NOK(ArchiveLayout::Parse(&in)) << "expected failure on truncated header"; +} + +TEST(ParseArchiveHeaderFuzz, NegativeFileCount) { + // BE int32 -1 = 0xFFFFFFFF + char bytes[4] = {static_cast(0xFF), static_cast(0xFF), static_cast(0xFF), + static_cast(0xFF)}; + ByteArrayInputStream in(bytes, 4); + ASSERT_NOK_WITH_MSG(ArchiveLayout::Parse(&in), "bad file_count"); +} + +TEST(ParseArchiveHeaderFuzz, NameLenOutOfRange) { + // file_count=1, name_len=2GB (BE int32 0x7FFFFFFF) + char bytes[8] = {0, + 0, + 0, + 1, + static_cast(0x7F), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF)}; + ByteArrayInputStream in(bytes, 8); + ASSERT_NOK_WITH_MSG(ArchiveLayout::Parse(&in), "bad name_len"); +} + +TEST(ParseArchiveHeaderFuzz, ZeroFileCountSucceeds) { + // file_count=0 is structurally valid; caller will fail later when + // tantivy::Index::open finds no meta.json, but parse itself OK. + char bytes[4] = {0, 0, 0, 0}; + ByteArrayInputStream in(bytes, 4); + ASSERT_OK_AND_ASSIGN(auto r, ArchiveLayout::Parse(&in)); + ASSERT_EQ(r.count, 0u); +} + +TEST(ParseArchiveHeaderFuzz, PayloadLenNegative) { + // file_count=1, name_len=1, name="a", data_len=-1 (BE int64 0xFFFFFFFFFFFFFFFF) + char bytes[4 + 4 + 1 + 8] = { + // file_count=1 + 0, + 0, + 0, + 1, + // name_len=1 + 0, + 0, + 0, + 1, + // name='a' + 'a', + // data_len = -1 (BE int64 0xFFFFFFFFFFFFFFFF) + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + }; + ByteArrayInputStream in(bytes, sizeof(bytes)); + ASSERT_NOK_WITH_MSG(ArchiveLayout::Parse(&in), "bad data_len"); +} + +// ========================================================================= +// 2. Concurrent query on same reader +// ========================================================================= + +TEST_F(StreamingTestFixture, ConcurrentQueryOnSameReader) { + // 50 docs containing "apple" in every one (all should match) + auto wr = BuildArchive(50, "apple banana {}"); + auto reader = OpenReader(wr.root_dir, wr.meta); + + auto fts = BuildMatchAll("apple"); + + // 4 threads × 20 queries each, all must return 50 rowIds + constexpr int32_t kThreads = 4; + constexpr int32_t kIters = 20; + std::vector threads; + std::atomic failures{0}; + for (int32_t t = 0; t < kThreads; ++t) { + threads.emplace_back([&] { + for (int32_t i = 0; i < kIters; ++i) { + auto result = reader->VisitFullTextSearch(fts); + if (!result.ok() || !result.value()) { + failures++; + continue; + } + std::shared_ptr r = result.value(); + auto plain = std::dynamic_pointer_cast(r); + if (!plain) { + failures++; + continue; + } + auto bres = plain->GetBitmap(); + if (!bres.ok() || bres.value() == nullptr || bres.value()->Cardinality() != 50) { + failures++; + } + } + }); + } + for (auto& th : threads) { + th.join(); + } + ASSERT_EQ(failures.load(), 0) << "concurrent queries produced inconsistent results"; +} + +// ========================================================================= +// 3. Concurrent reader open + close +// ========================================================================= + +TEST_F(StreamingTestFixture, ConcurrentCreateAndDropReaders) { + // One archive, many readers opening/closing it concurrently. + // Validates exactly-once release (no UAF under ASAN) and open/close race safety. + auto wr = BuildArchive(20); + + constexpr int32_t kThreads = 10; + std::vector threads; + std::atomic failures{0}; + for (int32_t t = 0; t < kThreads; ++t) { + threads.emplace_back([&, t] { + for (int32_t i = 0; i < 5; ++i) { + auto reader = OpenReader(wr.root_dir, wr.meta); + if (!reader) { + failures++; + continue; + } + auto fts = BuildMatchAll("apple"); + auto r = reader->VisitFullTextSearch(fts); + if (!r.ok()) { + failures++; + } + // reader drops here → Rust Arc::drop → paimon_cpp_stream_release + } + (void)t; + }); + } + for (auto& th : threads) { + th.join(); + } + ASSERT_EQ(failures.load(), 0); +} + +} // namespace +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_tokenizer_test.cpp b/src/paimon/global_index/tantivy/tantivy_tokenizer_test.cpp new file mode 100644 index 00000000..d82be514 --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_tokenizer_test.cpp @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/global_index/tantivy/tantivy_ffi_handle.h" +#include "paimon/global_index/tantivy/tantivy_ffi_status.h" + +extern "C" { +#include "paimon_tantivy_ffi.h" // NOLINT(build/include_subdir) +} + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time for this test" +#endif + +namespace paimon::tantivy::test { +namespace { + +/// Parse the FFI `tokenize` output (tab-separated: from\tto\tpos\ttext\n) and +/// return only the token text sequence. +std::vector ExtractTokenTexts(const PaimonTantivyBuffer& buf) { + std::vector out; + if (buf.len == 0) { + return out; + } + std::string s(reinterpret_cast(buf.data), buf.len); + std::istringstream in(s); + std::string row; + while (std::getline(in, row)) { + // extract text field = after 3rd '\t' + size_t p1 = row.find('\t'); + if (p1 == std::string::npos) { + continue; + } + size_t p2 = row.find('\t', p1 + 1); + if (p2 == std::string::npos) { + continue; + } + size_t p3 = row.find('\t', p2 + 1); + if (p3 == std::string::npos) { + continue; + } + out.emplace_back(row.substr(p3 + 1)); + } + return out; +} + +std::vector TokenizeWithTantivy(PaimonJiebaTokenizer* tok, const std::string& text) { + BufferGuard buf; + PaimonTantivyStatus st = + paimon_tantivy_tokenizer_tokenize(tok, text.data(), text.size(), buf.out()); + EXPECT_EQ(st, PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_OK) + << "FFI tokenize failed: " << paimon_tantivy_last_error(); + return ExtractTokenTexts(*buf.out()); +} + +} // namespace + +TEST(TantivyTokenizer, HmmModeReturnsUnsupported) { + std::string dict_dir = JIEBA_TEST_DICT_DIR; + PaimonJiebaTokenizer* handle = nullptr; + PaimonTantivyStatus st = + paimon_tantivy_tokenizer_new("hmm", /*with_position=*/true, dict_dir.c_str(), &handle); + ASSERT_EQ(st, PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_UNSUPPORTED); + ASSERT_EQ(handle, nullptr); + std::string err = paimon_tantivy_last_error(); + ASSERT_NE(err.find("hmm"), std::string::npos); +} + +// ---------------- positive jieba-rs behavior assertions ---------------- +// +// We do NOT require byte-level parity with cppjieba (the two backends coexist +// and each reads only its own index). Instead assert jieba-rs produces the +// expected token sequence for a curated set of inputs. + +struct JiebaRsCase { + std::string mode; + std::string input; + std::vector expected; +}; + +class JiebaRsBehavior : public ::testing::TestWithParam {}; + +TEST_P(JiebaRsBehavior, ProducesExpectedTokens) { + const auto& c = GetParam(); + std::string dict_dir = JIEBA_TEST_DICT_DIR; + PaimonJiebaTokenizer* handle = nullptr; + PaimonTantivyStatus st = paimon_tantivy_tokenizer_new(c.mode.c_str(), /*with_position=*/true, + dict_dir.c_str(), &handle); + ASSERT_EQ(st, PaimonTantivyStatus::PAIMON_TANTIVY_STATUS_OK) << paimon_tantivy_last_error(); + auto got = TokenizeWithTantivy(handle, c.input); + ASSERT_EQ(got, c.expected) << "mode=" << c.mode << " input=" << c.input; + paimon_tantivy_tokenizer_free(handle); +} + +INSTANTIATE_TEST_SUITE_P( + BasicCases, JiebaRsBehavior, + ::testing::Values(JiebaRsCase{"mix", "Hello World", {"hello", "world"}}, + JiebaRsCase{"mix", "HELLO", {"hello"}}, + JiebaRsCase{"mix", "中国人民", {"中国", "人民"}}, + // the two single-char stop words in the input are in + // stop_words.utf8, so Normalize drops them from the output + JiebaRsCase{"mix", "他来到了网易杭研大厦", {"来到", "网易", "杭研", "大厦"}}, + JiebaRsCase{"full", "中国", {"中", "中国", "国"}}, + JiebaRsCase{"query", "中国人民", {"中国", "人民"}})); + +} // namespace paimon::tantivy::test diff --git a/src/paimon/global_index/tantivy/tantivy_writer_test.cpp b/src/paimon/global_index/tantivy/tantivy_writer_test.cpp new file mode 100644 index 00000000..1b51fc7f --- /dev/null +++ b/src/paimon/global_index/tantivy/tantivy_writer_test.cpp @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include "arrow/array.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/api.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/global_index/tantivy/tantivy_defs.h" +#include "paimon/global_index/tantivy/tantivy_global_index_writer.h" +#include "paimon/testing/utils/testharness.h" + +#ifndef JIEBA_TEST_DICT_DIR +#error "JIEBA_TEST_DICT_DIR must be set at compile time" +#endif + +namespace paimon::tantivy::test { + +namespace { + +class FakeIndexPathFactory : public IndexPathFactory { + public: + explicit FakeIndexPathFactory(const std::string& root) : root_(root) {} + std::string NewPath() const override { + assert(false); + return ""; + } + std::string ToPath(const std::shared_ptr&) const override { + assert(false); + return ""; + } + std::string ToPath(const std::string& file_name) const override { + return PathUtil::JoinPath(root_, file_name); + } + bool IsExternalPath() const override { + return false; + } + + private: + std::string root_; +}; + +/// Read the entire file at `path` into a byte buffer. +std::vector ReadFile(const std::string& path) { + std::ifstream in(path, std::ios::binary); + EXPECT_TRUE(in.good()) << "open " << path; + in.seekg(0, std::ios::end); + auto sz = static_cast(in.tellg()); + in.seekg(0, std::ios::beg); + std::vector buf(sz); + in.read(reinterpret_cast(buf.data()), sz); + return buf; +} + +/// Read a big-endian integer from a raw pointer. +template +T ReadBE(const uint8_t* p) { + T v = 0; + for (std::size_t i = 0; i < sizeof(T); ++i) { + v = static_cast((v << 8) | static_cast(p[i])); + } + return v; +} + +struct PackedEntry { + std::string name; + int64_t length = 0; + std::size_t offset = 0; // offset in the buffer where bytes start +}; + +/// Parse the packing header into a list of entries; verifies that the offsets +/// and lengths cover the full buffer with no leftover bytes. +/// Format (Java-compatible, big-endian, no version header): +/// [i32 BE file_count | (i32 BE name_len | name | i64 BE file_len | bytes)*] +std::vector ParsePacked(const std::vector& bytes) { + std::vector entries; + EXPECT_GE(bytes.size(), 4u); + auto file_count = ReadBE(bytes.data()); + EXPECT_GT(file_count, 0); + std::size_t off = 4; + for (int32_t i = 0; i < file_count; ++i) { + EXPECT_LE(off + 4, bytes.size()); + auto nlen = ReadBE(bytes.data() + off); + off += 4; + EXPECT_GT(nlen, 0); + EXPECT_LE(off + static_cast(nlen), bytes.size()); + std::string name(reinterpret_cast(bytes.data() + off), + static_cast(nlen)); + off += nlen; + EXPECT_LE(off + 8, bytes.size()); + auto flen = ReadBE(bytes.data() + off); + off += 8; + EXPECT_GE(flen, 0); + EXPECT_LE(off + static_cast(flen), bytes.size()); + entries.push_back({name, flen, off}); + off += static_cast(flen); + } + EXPECT_EQ(off, bytes.size()) << "trailing bytes after pack"; + return entries; +} + +class TantivyGlobalIndexWriterTest : public ::testing::Test { + public: + std::unique_ptr<::ArrowSchema> CreateArrowSchema( + const std::shared_ptr& data_type) const { + auto c_schema = std::make_unique<::ArrowSchema>(); + EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); + return c_schema; + } + + Result> WriteIndex( + const std::string& root, const std::shared_ptr& data_type, + const std::map& options, + const std::shared_ptr& array) { + auto path_factory = std::make_shared(root); + auto file_writer = std::make_shared(fs_, path_factory); + PAIMON_ASSIGN_OR_RAISE(auto writer, TantivyGlobalIndexWriter::Create( + "f0", data_type, file_writer, options, pool_)); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + std::vector relative_row_ids(array->length()); + for (int64_t i = 0; i < array->length(); ++i) { + relative_row_ids[i] = i; + } + PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array, std::move(relative_row_ids))); + return writer->Finish(); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr fs_ = std::make_shared(); + std::shared_ptr data_type_ = + arrow::struct_({arrow::field("f0", arrow::utf8())}); +}; + +} // namespace + +TEST_F(TantivyGlobalIndexWriterTest, EnglishCorpusProducesValidPackedIndex) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options = { + {kTantivyWriteOmitTermFreqAndPositions, "false"}, + }; + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["This is an test document."], + ["This is an new document document document."], + ["Document document document document test."], + ["unordered user-defined doc id"] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto metas, WriteIndex(root, data_type_, options, array)); + ASSERT_EQ(metas.size(), 1u); + const auto& meta = metas[0]; + + auto file_name = PathUtil::GetName(meta.file_path); + ASSERT_TRUE(StringUtils::StartsWith(file_name, "tantivy-fulltext-global-index-")) + << "file_name=" << file_name; + ASSERT_TRUE(StringUtils::EndsWith(file_name, ".index")); + ASSERT_TRUE(meta.metadata); + ASSERT_EQ(std::string(meta.metadata->data(), meta.metadata->size()), + R"({"write.omit-term-freq-and-position":"false"})"); + ASSERT_GT(meta.file_size, 8); + + auto bytes = ReadFile(meta.file_path); + ASSERT_EQ(static_cast(bytes.size()), meta.file_size); + auto entries = ParsePacked(bytes); + ASSERT_FALSE(entries.empty()); + bool has_meta_json = false; + for (const auto& e : entries) { + if (e.name == "meta.json") { + has_meta_json = true; + } + } + ASSERT_TRUE(has_meta_json) << "expected meta.json in packed entries"; +} + +TEST_F(TantivyGlobalIndexWriterTest, ChineseCorpusProducesValidPackedIndex) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options = { + {kTantivyWriteOmitTermFreqAndPositions, "false"}, + {kTantivyWriteTokenizer, "paimon_jieba"}, + {kJiebaTokenizeMode, "query"}, + }; + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["千问是一个智能助手"], + ["新一代AI助手发布"] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto metas, WriteIndex(root, data_type_, options, array)); + ASSERT_EQ(metas.size(), 1u); + const auto& meta = metas[0]; + auto bytes = ReadFile(meta.file_path); + ASSERT_EQ(static_cast(bytes.size()), meta.file_size); + auto entries = ParsePacked(bytes); + ASSERT_FALSE(entries.empty()); +} + +TEST_F(TantivyGlobalIndexWriterTest, NullStringRowsBecomeEmptyDocuments) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + std::string root = root_dir->Str(); + + std::map options; + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ + ["nonempty"], + [null], + ["another"] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto metas, WriteIndex(root, data_type_, options, array)); + ASSERT_EQ(metas.size(), 1u); +} + +TEST_F(TantivyGlobalIndexWriterTest, RejectsHmmTokenizeMode) { + auto root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(root_dir); + auto path_factory = std::make_shared(root_dir->Str()); + auto file_writer = std::make_shared(fs_, path_factory); + // hmm rejection only fires when the jieba tokenizer is actually constructed, + // so this test must explicitly opt into jieba (default tokenizer skips + // jieba construction entirely). + std::map options = { + {kTantivyWriteTokenizer, "paimon_jieba"}, + {kJiebaTokenizeMode, "hmm"}, + }; + auto res = TantivyGlobalIndexWriter::Create("f0", data_type_, file_writer, options, pool_); + ASSERT_FALSE(res.ok()); + ASSERT_TRUE(res.status().IsNotImplemented()) << res.status().ToString(); +} + +} // namespace paimon::tantivy::test diff --git a/src/paimon/testing/utils/CMakeLists.txt b/src/paimon/testing/utils/CMakeLists.txt index b12af3c8..25f99d3f 100644 --- a/src/paimon/testing/utils/CMakeLists.txt +++ b/src/paimon/testing/utils/CMakeLists.txt @@ -28,6 +28,16 @@ if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS) STATIC_LINK_LIBS paimon_static ${GTEST_LINK_TOOLCHAIN}) + + # testharness.cpp includes , but the objlib compile step does + # not link the gtest targets, so it has no ordering dependency on the + # googletest ExternalProject. Without this it can race ahead and compile + # before googletest_ep extracts its headers (observed as a flaky + # "gtest/gtest.h: No such file" in Release builds). Force the headers to + # exist before compiling test_utils. + if(TARGET googletest_ep) + add_dependencies(test_utils_objlib googletest_ep) + endif() endif() if(PAIMON_BUILD_TESTS) diff --git a/test/test_data/cpp_tantivy_fixtures/english_default.archive b/test/test_data/cpp_tantivy_fixtures/english_default.archive new file mode 100644 index 0000000000000000000000000000000000000000..d195af7ec631833bd29d7eb7e8fd41d03a4cd1ed GIT binary patch literal 6597 zcmd5>+iP4!7@ytEy|-P_Y9(MiBoRVV&dj+7>w}{7MWMASDw3YvJ)3Tty>$1ajV;l$ zNiRwDK~!ih_~wgZK?;Jt)nZIb!Git~`Xc!3i+7-Jw7QMxpQms*O{h7^f=F+56XcUxbP16i!*p8tuy$Yi{z2S5d$<~#<`65hL z;qc}%tpMQ#{qWO4eALnm)zPR)WqhNF_{zmaj%-DEGeCD@z_kck%R!i43sK{;bP%K1j_(#tR`cWE-@daa3wAS?s{AYA4MBfoYSp{6rU)3^F{JkhUBzt->~l;rAuNE~lr zF4arYX>uGAkZE0L0sbqPd=w%EOv|BzzSgM5RF&GcDesWv^XYqH^QIbU85^dvBp^5C zKViU=yya8;Dav^^X6V*Bs7%>XKdj$A^YXnrcS!ca-SxZcB>U;c#;qUI&~IR0A@Alg zVhbPS!w49n{O@4MQ(!uk$sE10XPP%f5E%Jp`vcQKhc_Elb<=Q=9b-#mJK46#0Re1bKfjF0R2L@ca}wU%9z1)OONfr ziDUz}u;{6DI+ISPGU+ansckuyYUpMR=4c|!Hkp2o4#WWbxpMx(#Y?#;OO{t!tpMXP zrdqA@trh;n95905+Yh?2!8Bd9G~F<4Ti#&FhP5g8B2X>{umHIi&<4wp277|g+@Kq< znL`Z+4u-Mx3QWQ;it)tdkU99*kD+J1g&0%mKEmL+m*g_LNzY!A?b$_ohe@s%a>FFs z7v=h++(48Y+>ONjdu}gu9ELan5jpAwh{qu2A>x{coWA-CdSMnK229JLgPg82%b^VK zD0w}{5~M=W6cj7UMgR6l+g*|BAxB<% zUfa0!(>;>Ca(gtpw!Xp9r}~C+k3BIyTk?u{ zA@uk28IJI$Kr705act%uQ3y}GI5K^`-)Hj!*AKtA|FhS|^3U;V0h*tHheQ1u;6~}- zN`v+^nWH)ei> zhSfD8tMU9}SdLUQNNZ5go8lVeHDq@X*dO9*cKQttjMQ}-Ub17Wk!)m}cHc*JjRCG| z(AdlP`(RWyh)EMRKinGfP8v^KAc>Gu6;AFTgz2z4Q{Ok%c{O~6qT|RW`=<%^H z)=%#s!-u;f!{JrShPP+NVi(F_#q-^RbIn>ca$tJn+|bAf|D@dXe7{_sZ7S0vSPJFE zS|L0$Glg2E-td~y7wQpZ{;aM{ZfTOAuN6)<%V$04PEF0$p?(P)DNS#-;#GYd_sy_V zGOOp?Z~G|pQ}{{VLnHjgf+x^G z>yo&;utUk6Z^kO&{|Vc>gbo)&g%*GxBTJ%zH@zot``)6z9d*2PXV|df0748Jn$z$K zwMMuk!1iUdcXRl#K4w+iVzAk-o%E`(58xjcxF98WVcu7`A$&{VDA01#)Ezt%7ROxx Uxo9zK^``Gu>M#e@hL(o@0v5QI4FCWD literal 0 HcmV?d00001 diff --git a/test/test_data/java_tantivy_fixtures/README.md b/test/test_data/java_tantivy_fixtures/README.md new file mode 100644 index 00000000..fa7fd4e1 --- /dev/null +++ b/test/test_data/java_tantivy_fixtures/README.md @@ -0,0 +1,51 @@ +# Java -> C++ tantivy cross-read fixtures + +> Generated on **2026-04-23** for the J6 `paimon-tantivy-java-compat-test`. + +## Contents + +| File | Purpose | +|---|---| +| `english_simple.archive` | A BE archive produced by paimon-java's `TantivyIndexWriter + packIndex` path; 10 plain-English documents, row_ids 0..9 | +| `english_simple.golden.json` | Human-readable golden file: expected row_ids for each query type | + +## Pinned versions + +| Component | Version | +|---|---| +| tantivy crate | **0.22.1** | +| paimon-tantivy-jni | latest git sha at generation time (commit lives in the paimon repo) | +| schema | B1: `row_id` u64 stored+indexed+fast + `text` TEXT | +| archive byte format | Java-compatible, big-endian, no version header | + +Upgrading any component (especially the **tantivy version**) can make the segment +files binary-incompatible — regenerate the fixtures: + +```bash +# 1. Build the Java native lib (if the Rust side changed) +cd /path/to/paimon/paimon-tantivy/paimon-tantivy-jni/rust && cargo build --release +cp target/release/libtantivy_jni.dylib \ + ../src/main/resources/native/darwin-aarch64/ + +# 2. mvn install + run the fixture generator +cd /path/to/paimon +mvn install -pl paimon-tantivy/paimon-tantivy-index -am -DskipTests -Denforcer.skip=true +mvn -pl paimon-tantivy/paimon-tantivy-index test \ + -Dtest=TantivyIndexFixtureGen -DfailIfNoTests=false \ + -Denforcer.skip=true \ + -DfixtureOutDir=/path/to/paimon-cpp/test/test_data/java_tantivy_fixtures +``` + +## Verification + +``` +xxd english_simple.archive | head -1 +# 00000000: 00 00 00 16 ... <- BE int32 file_count = 22 (Java does not +# force-merge, so multiple segments) +``` + +## Related docs + +- `docs/dev/tantivy_java_cross_read_plan.md` — overall J6 plan +- `docs/dev/test_execute.md` — J6 execution log +- `docs/dev/tantivy_java_compat_plan.md` — overall paimon-cpp <-> paimon-java alignment plan diff --git a/test/test_data/java_tantivy_fixtures/english_simple.archive b/test/test_data/java_tantivy_fixtures/english_simple.archive new file mode 100644 index 0000000000000000000000000000000000000000..c0849957858b871d0172f4fa152d1030107fbb6f GIT binary patch literal 6044 zcmd5=&1)M+6dy^p{L#dfLYkHmsJqy136|N}?=?L&w9rEf=_fPt+x;`P6#y6{tpcl3O)DK_jYHbT{~fwBGHZ*X6DV;`}n;# z+D8aEMhKZVBvmpEYS(4SP!v%!ExT?@k|x(2TdSG&Qr+tMun^0SgoNSEo3kgK8#x#r zMp~ZC1fF|orBNeD>y?*Sj}r?pri?ZTEzwKxafG@hT9gFuL><|&Gm$3aZzYB z+Srq%MWJK)_BmmhE((oy&DmI~yI#xkS9;sD%|*fXY++f~s4gp-p=^EvhnwTkZS#_9 z$aSZtS7ljID77qAG;6Y^S(dFSAZAj)yMzQ`5GI71es1A#^9Y2KWmyGLOjX(Zv@hLU zrgV+kMx+Nqwvj6=0ono!${K`QLDav6(*fFah$hiQMW?2;`4Jq;46;xYWopSv-L5%c z5Lq?EIu#9@s8hq<3QtBhkZQ5+vDz@o&Rc9T&STxr6De&ye)y1-b{_3M+9joL_xA4pLP}rkKG^$hs&r@X{;!Wo>Gp%U z(yiS+#(l0hU4G@Yg;j_ecGW6ROt2eHv6geo>5NRcTj99KiO`NgEfmL#Q`O(!eDD2Z z-+eY;J;|my)655tMxiH|f!QHKSCUkPs#Mm^WawDPv~6^Q_5v9{PRipjI!;Q(cr+1@Cgai6A(GFNzse}uai}Mt8c^Sb`Z3fhRJ4BN)Hk1? zURI%E!G5v~Q8TKFN{}TpO5Z`l9vEo8b-GPZn;Z~WO1<;{&_m^d4wFn-`~qgXHFBoyo?%MeW{Q&|#4Y8WX= z=QE%1L@+p)^p}XuQvN>+2Fdphqkvs4gKFb4&srV>IGvGGYEI5!A7j7~ldVL1U>#)$ zkR^0~MKq0+Z)HNP<8~RB>jC^x47M1PaR3R)^qOl&BCptPtK&J{I3G7FG_NSa;=UzSOmcJ^R~*<* zHQ7*LeiIi7U1zoBw0%7HbhIfxy*~p!7OJRtqg7#5vji1^(quL|@mqNkK_&$A`>o9|#c` zqAaU}mh!!IhbsV9_goS;)U9SWHHofuU`vE5epF;90LxSD#rgJ~4Tz&`sld5TQ=iQ? zz#5p~xQoQ`9NYCG+<~JuP~P70(R@m(w8S8@@1A$s0CvblBkbzddeavecMSJ<3IYzZ dmUz!IG!v22Du7XJOt;hZtyTv%Q+axG`X6gdS(*R< literal 0 HcmV?d00001 diff --git a/test/test_data/java_tantivy_fixtures/english_simple.golden.json b/test/test_data/java_tantivy_fixtures/english_simple.golden.json new file mode 100644 index 00000000..9776b720 --- /dev/null +++ b/test/test_data/java_tantivy_fixtures/english_simple.golden.json @@ -0,0 +1,25 @@ +{ + "description": "10 English docs; row_ids 0..9; generated by TantivyIndexFixtureGen via TantivyFullTextGlobalIndexWriter production path; consumed by paimon-cpp V3 reader cross-read test (J6).", + "docs": [ + {"row_id": 0, "text": "apple banana cherry"}, + {"row_id": 1, "text": "apple durian"}, + {"row_id": 2, "text": "banana cherry"}, + {"row_id": 3, "text": "fig grape"}, + {"row_id": 4, "text": "apple cherry fig"}, + {"row_id": 5, "text": "banana elderberry"}, + {"row_id": 6, "text": "cherry durian"}, + {"row_id": 7, "text": "apple"}, + {"row_id": 8, "text": "grape fig elderberry"}, + {"row_id": 9, "text": "cherry fig"} + ], + "queries": [ + {"type": "match_all", "query": "apple", "expected_row_ids": [0, 1, 4, 7]}, + {"type": "match_all", "query": "apple banana", "expected_row_ids": [0]}, + {"type": "match_any", "query": "durian elderberry", "expected_row_ids": [1, 5, 6, 8]}, + {"type": "phrase", "query": "apple banana", "expected_row_ids": [0]}, + {"type": "phrase", "query": "banana cherry", "expected_row_ids": [0, 2]}, + {"type": "prefix", "query": "ap", "expected_row_ids": [0, 1, 4, 7]}, + {"type": "wildcard", "query": "*err*", "expected_row_ids": [0, 2, 4, 5, 6, 8, 9]}, + {"type": "match_any", "query": "apple banana cherry durian fig grape elderberry", "expected_row_ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} + ] +} diff --git a/test/test_data/java_tantivy_fixtures/production_sample.archive b/test/test_data/java_tantivy_fixtures/production_sample.archive new file mode 100644 index 0000000000000000000000000000000000000000..0f82971897acd4fa02acc0141bfdc2b2f659613a GIT binary patch literal 5176 zcmcH-TWlLeaL=`!cXDo%rbv+*u1%?`II_>ri>iK9EvSGhA=(P5YQxn&r*8bhKKDt= zofEh8{h&~Ugn){Qhtv;{-~)b0l(cCpXhkLX<(m)GFMdFUncH3aoHW=-$)3lzGdnvw zvpYL8HX%eLgbW0gppx)MWM3>E3@R~IQNmg{7)^u|iG&=D#D@}UK?g=`w+Jz>nI(q+ z{SG8JNSl$*9vMl-3DPR9hxIsd1>k;0bHE$vjyH~MJs};|@`YqJBMqOB((0jXUK;id zO6g<SUCSr0Z5Q+Fgnx=&O(V=AgC{<0k+)F#eFiR7Y zUA}VloonyDH_nGA%jL3(Z2~%zR?jdeO4egx^s72r?HQH1<|KRg zjH;{(PnEk=9W^IlVgBKh@m*bok5V=}PgPEpn^`kYpDL%VXW$kXA6e&z4!nft=U4pL zB0jn1{OUSb<^0GsycyUGvz2zBXPWb&WLP|Bm&FJ<6oDrcuwSB=hYHyY)oR7? ziPrvpb{`9xt|v321!=e+sX&isV+B2%*G6L5bS|$I==JWGQpbbR;5?QCcpnSN;~Gr& zhXc_djE^IeRM1A#T1H2?d#y=SdUXV14vb0set7YMc=3jK@rroucZ4kqbJOb45zr4N z$^ohiDy*e6UBi_!#Z<~#%+jHsS+rnHex_cGnO+Q3thxXvOwW07mO05E!(*<3+a(>a(d zM_R}6Uf0jKkf>O?5iawXK(8LnoIQfaL4L45oNIuwJ;&ahZlC?MKGFmS>4~yIj4cTB z=M(YLgGWR>H}i0AmWUVc-GT2@Q+Hgig*xOc2~ehIc3N^t2O)gBYEF;U1f1(=bY(q7CZ-uwmccmM~Sx5a1PG zC>*XSH8$~3x@f6Uxlc?l!|ist-0;}H09&2{$?p$`gR(!ODE7msUm!tjVEX=JOkl%= z3Tf4ttUM)b#w0cas)=$L4#Rp3Hp9+9+3>{pr(EHhES2a7#IaQjY`7j2J@}Z6*`hP& zr3K{@7bK9%qY-~36t(B_j-j-gQAf47EuRa2?v?7gG{z)pu##>U^%2B@dd>TFw7%}P zSMS!i0WJ>>Zjj~T%jwyXXjGAdP^Q{FdqZQJL_>ja{szR?*J)ISJmesbP9i$mhznx# zigw}z%n4W@@wfo%BchvPO%!XUSPR8kDb_}@c8YaStdnA06cZ`tq1Z}_t)f^r#a2_S zmtt$IJVDNq$2d&z+a5!DTsyFTT6giJ2aX73|SaFI` z@x^yv-}-C%tIdzT85ediZe&JSSY%x#V(F)Qv&10?O}UI`E0$VQv#Z2&bHCn)0&Di+ z9W2D4#&Q+29?Q@|#^VTxpAoTqf9@d+G!<2>#jsE#;`RG;w{K71n*n-T9_TNNr=f29 zaBA+(?1Q<7ed6WWX%I;3INUd(rR$lZp42?fG4ac3BA%X^x;G6Q+|kz31UpNFt zog<^i-YOq1LXDS7CbR>pr>&bAHpkw4$2gsliM{aSzjSfT4d85UZgp>;`gmjSz@t|- zuhXuT_Qw2>|}E=70aG YhE8eu($fB^76=D?p^y>^_$s#k3m#qhj{pDw literal 0 HcmV?d00001 diff --git a/test/test_data/tokenizer_golden/README.md b/test/test_data/tokenizer_golden/README.md new file mode 100644 index 00000000..b6ba3d20 --- /dev/null +++ b/test/test_data/tokenizer_golden/README.md @@ -0,0 +1,29 @@ +# Tokenizer golden samples + +Used by `paimon-tantivy-tokenizer-test` to compare cppjieba vs jieba-rs +tokenization output. + +## Files + +- `golden_synthetic.txt` — hand-written edge cases (mixed Chinese/English, + digits, punctuation, emoji, whitespace, very long words, ...) +- `golden_corpus.txt` — short excerpts from public corpora (general knowledge, + no copyright concerns) + +## Usage + +The test code (see `src/paimon/global_index/tantivy/tantivy_tokenizer_test.cpp`): +1. reads the files line by line +2. tokenizes each line with cppjieba `JiebaTokenizer::CutWithMode` + `Normalize` + to get token sequence A +3. tokenizes each line with the jieba-rs FFI `paimon_tantivy_tokenizer_tokenize` + to get token sequence B +4. compares A and B: the line passes if they are identical, otherwise it is + recorded in the diff report +5. (historical) the original acceptance bar was a diff rate <= 1%; the test is + now advisory only and logs diffs without failing + +## Extending + +To add business query logs later, drop a new `golden_business.txt` in this +directory; the test scans `golden_*.txt` automatically. diff --git a/test/test_data/tokenizer_golden/golden_corpus.txt b/test/test_data/tokenizer_golden/golden_corpus.txt new file mode 100644 index 00000000..38c7c887 --- /dev/null +++ b/test/test_data/tokenizer_golden/golden_corpus.txt @@ -0,0 +1,20 @@ +人工智能是计算机科学的一个分支 +机器学习是人工智能的核心领域 +深度学习使用神经网络进行模式识别 +大语言模型基于 Transformer 架构 +开源软件促进了全球技术合作 +Rust 语言以内存安全著称 +Python 广泛应用于数据科学 +分布式系统需要处理网络分区问题 +数据库事务保证原子性一致性隔离性持久性 +编程的艺术在于解决复杂问题 +搜索引擎依赖倒排索引加速查询 +自然语言处理技术日新月异 +云计算降低了基础设施成本 +开发者社区推动了技术进步 +版本控制系统是协作的基石 +操作系统管理计算机的硬件资源 +编译器将源代码翻译成机器指令 +算法的时间复杂度决定了执行效率 +数据结构的选择影响程序性能 +网络协议定义了通信的规则 diff --git a/test/test_data/tokenizer_golden/golden_synthetic.txt b/test/test_data/tokenizer_golden/golden_synthetic.txt new file mode 100644 index 00000000..65b14474 --- /dev/null +++ b/test/test_data/tokenizer_golden/golden_synthetic.txt @@ -0,0 +1,38 @@ +Hello World +hello world +HELLO WORLD +Hello 世界 +你好世界 +中国人民共和国 +我爱北京天安门 +北京是中华人民共和国的首都 +南京市长江大桥 +他来到了网易杭研大厦 +小明硕士毕业于中国科学院计算所,后在日本京都大学深造 +工信处女干事每月经过下属科室都要亲口交代24口交换机等技术性器件的安装工作 +结婚的和尚未结婚的 +程序员用Python和Rust写代码 +this is a test 这是一个测试 +Rust tantivy 全文索引 +C++ 到 Rust 的 FFI 桥接 +cpp cppjieba jieba-rs +分词器 tokenizer +全文 search +倒排索引 inverted index +paimon-cpp tantivy-fts +100个中文字符被分词器处理 +超长词最长词最长词最长词最长词最长词最长词 +... +!@#$%^&*() +"hello" +'quoted' +content +{json: "value"} +[1,2,3] +line1 +line2 +CJK 标点、。！？ +全角：ＡＢＣ１２３ +ABC123 混合数字字母 +abc123 +ABC123 diff --git a/test/test_data/tokenizer_golden/known_diffs.txt b/test/test_data/tokenizer_golden/known_diffs.txt new file mode 100644 index 00000000..23073bd3 --- /dev/null +++ b/test/test_data/tokenizer_golden/known_diffs.txt @@ -0,0 +1,18 @@ +abc_123 +foo.bar.baz +https://example.com/path?q=1 +email@example.com +192.168.1.1 +2026-04-20 +12:34:56 +$100 ¥200 €300 +100% +3.14 +-1 -2 -3 +a b c d e + + tab tab +mixed space tab +空 白 和 tab + leading and trailing +中英混合 Mixed CN EN diff --git a/third_party/tantivy_ffi/Cargo.lock b/third_party/tantivy_ffi/Cargo.lock new file mode 100644 index 00000000..be9056ad --- /dev/null +++ b/third_party/tantivy_ffi/Cargo.lock @@ -0,0 +1,1859 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "allocator-api2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cbindgen" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799" +dependencies = [ + "clap", + "heck", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "tempfile", + "toml", +] + +[[package]] +name = "cc" +version = "1.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cedarwood" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90" +dependencies = [ + "smallvec", +] + +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "croaring" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0e813b58ac55ac5ccea5ec63beb8c80f37dedd78da3f594c848313415a08c8c" +dependencies = [ + "allocator-api2 0.4.0", + "croaring-sys", +] + +[[package]] +name = "croaring-sys" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f34e9ee8e65c0d46c9d0fe55ce80b477d0bfae4c786c6694687b9c70e8267027" +dependencies = [ + "cc", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix 0.38.44", + "windows-sys 0.52.0", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2 0.2.21", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2 0.2.21", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "include-flate" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e233413926ef735f7d87024466cfda5a4b87467730846bd82ea7d504121347" +dependencies = [ + "include-flate-codegen", + "include-flate-compress", +] + +[[package]] +name = "include-flate-codegen" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7148f24ef8922cc0e5574ebb908729ccdd3a110c440a45165733fedadd9969" +dependencies = [ + "include-flate-compress", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "include-flate-compress" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74783a9ed407e844e99d5e7a57bd650acbfa124cf6e97ffd790ba59d8ab8e7ff" +dependencies = [ + "libflate", + "zstd", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jieba-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c676b32a471d3cfae8dac2ad2f8334cd52e53377733cca8c1fb0a5062fec192" +dependencies = [ + "phf_codegen", +] + +[[package]] +name = "jieba-rs" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5dd552bbb95d578520ee68403bf8aaf0dbbb2ce55b0854d019f9350ad61040a" +dependencies = [ + "cedarwood", + "fxhash", + "include-flate", + "jieba-macros", + "lazy_static", + "phf", + "regex", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "libc" +version = "0.2.185" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" + +[[package]] +name = "libflate" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df" +dependencies = [ + "adler32", + "crc32fast", + "dary_heap", + "libflate_lz77", + "no_std_io2", +] + +[[package]] +name = "libflate_lz77" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd" +dependencies = [ + "hashbrown 0.16.1", + "no_std_io2", + "rle-decode-fast", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + +[[package]] +name = "measure_time" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +dependencies = [ + "instant", + "log", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + +[[package]] +name = "no_std_io2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b51ed7824b6e07d354605f4abb3d9d300350701299da96642ee084f5ce631550" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "ownedbytes" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "paimon_tantivy_ffi" +version = "0.1.0" +dependencies = [ + "cbindgen", + "croaring", + "jieba-rs", + "log", + "tantivy", + "tempfile", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +dependencies = [ + "serde", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tantivy" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "num_cpus", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror", + "time", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +dependencies = [ + "nom", +] + +[[package]] +name = "tantivy-sstable" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +dependencies = [ + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +dependencies = [ + "murmurhash32", + "rand_distr", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.1", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/third_party/tantivy_ffi/Cargo.toml b/third_party/tantivy_ffi/Cargo.toml new file mode 100644 index 00000000..68fe24dd --- /dev/null +++ b/third_party/tantivy_ffi/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "paimon_tantivy_ffi" +version = "0.1.0" +edition = "2021" +description = "C FFI layer wrapping tantivy + jieba-rs for paimon-cpp tantivy-fts global index" +license = "Apache-2.0" +publish = false + +[lib] +name = "paimon_tantivy_ffi" +# staticlib: linked by CMake + Corrosion into libpaimon_tantivy_ffi.a +# rlib: lets `cargo test` use native Rust linkage when building test binaries +crate-type = ["staticlib", "rlib"] + +[dependencies] +tantivy = "0.22" +jieba-rs = "0.7" +croaring = "2.0" +log = "0.4" +tempfile = "3" + +[build-dependencies] +cbindgen = "0.29" + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +panic = "abort" + +[profile.dev] +# Errors propagate across the FFI via status codes; a Rust panic must abort +# rather than unwind across the FFI boundary. +panic = "abort" diff --git a/third_party/tantivy_ffi/build.rs b/third_party/tantivy_ffi/build.rs new file mode 100644 index 00000000..107d3f42 --- /dev/null +++ b/third_party/tantivy_ffi/build.rs @@ -0,0 +1,40 @@ +//! build.rs: runs cbindgen to generate the C header paimon_tantivy_ffi.h. +//! +//! Output path: $OUT_DIR/paimon_tantivy_ffi.h +//! Corrosion (on the CMake side) reads OUT_DIR from cargo metadata and adds the +//! header to the C++ include path. + +use std::env; +use std::path::PathBuf; + +fn main() { + let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let header_path = out_dir.join("paimon_tantivy_ffi.h"); + + let cfg = cbindgen::Config::from_file(PathBuf::from(&crate_dir).join("cbindgen.toml")) + .expect("cbindgen.toml must exist at crate root"); + + match cbindgen::Builder::new() + .with_crate(&crate_dir) + .with_config(cfg) + .generate() + { + Ok(bindings) => { + bindings.write_to_file(&header_path); + println!( + "cargo:rerun-if-changed={}", + PathBuf::from(&crate_dir).join("src").display() + ); + println!("cargo:rerun-if-changed=cbindgen.toml"); + // Expose the header directory to Corrosion / the upstream CMake build. + println!("cargo:include={}", out_dir.display()); + eprintln!("cbindgen: wrote {}", header_path.display()); + } + Err(e) => { + // cbindgen failure is not necessarily fatal (e.g. CI skips it when the + // Rust code is unchanged); log a warning and continue. + eprintln!("cbindgen generation failed: {e:?}"); + } + } +} diff --git a/third_party/tantivy_ffi/cbindgen.toml b/third_party/tantivy_ffi/cbindgen.toml new file mode 100644 index 00000000..646051b0 --- /dev/null +++ b/third_party/tantivy_ffi/cbindgen.toml @@ -0,0 +1,66 @@ +# cbindgen config: Rust FFI -> C header generator. +# Invoked by build.rs, outputs to $OUT_DIR/paimon_tantivy_ffi.h. +# CMake picks up $OUT_DIR via Corrosion and adds it to the C++ include path. + +language = "C" + +# Header banner written at the top of the generated file. +header = """ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * \"License\"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * AUTO-GENERATED by cbindgen from Rust sources under third_party/tantivy_ffi - DO NOT EDIT. + * + * C ABI for paimon_tantivy_ffi. See docs/dev/tantivy_ffi_design.md for contract. + */ +#pragma once +""" + +include_guard = "PAIMON_TANTIVY_FFI_H" +cpp_compat = true +pragma_once = false # already written by hand in the header banner above +documentation = true +documentation_style = "c" +line_length = 100 +tab_width = 4 + +[export] +# No type prefix (Rust type names already carry the PaimonTantivy... prefix). +# Function names already start with paimon_tantivy_ (named that way in the Rust source). +prefix = "" +# Force-export types that only appear as handles / return values; cbindgen does +# not export them by default when no FFI function directly takes/returns them, +# so list them explicitly here. +include = ["PaimonTantivyStatus"] + +[export.rename] +# Rust enum name -> C typedef name (to avoid duplicated prefixes, etc.) + +[fn] +prefix = "" +args = "auto" +rename_args = "None" + +[enum] +rename_variants = "ScreamingSnakeCase" +prefix_with_name = true +derive_helper_methods = false + +[parse] +parse_deps = false diff --git a/third_party/tantivy_ffi/rust-toolchain.toml b/third_party/tantivy_ffi/rust-toolchain.toml new file mode 100644 index 00000000..8a8c3664 --- /dev/null +++ b/third_party/tantivy_ffi/rust-toolchain.toml @@ -0,0 +1,11 @@ +# Pin the Rust toolchain used to build paimon_tantivy_ffi. Without this, +# Corrosion's FindRust.cmake invokes `rustup which rustc --toolchain ''` +# which fails on fresh CMake configure (no rust-toolchain → empty toolchain +# name → rustup rejects it). See docs/dev/execute.md Stage 11 for context. +# +# Only the `channel` is pinned — no extra components, because rustup in +# CI/containers may lack network access to fetch clippy/rustfmt, and build +# doesn't need them. +[toolchain] +channel = "stable" +profile = "minimal" diff --git a/third_party/tantivy_ffi/src/buffer.rs b/third_party/tantivy_ffi/src/buffer.rs new file mode 100644 index 00000000..13e9f43f --- /dev/null +++ b/third_party/tantivy_ffi/src/buffer.rs @@ -0,0 +1,111 @@ +//! `paimon_tantivy_buffer_t`: Rust-allocated byte buffer returned to C++. +//! +//! Contract: +//! - Buffer is allocated by Rust (as a `Box<[u8]>`) +//! - C++ reads `data[0..len]`, **must not** write past len +//! - C++ must call `paimon_tantivy_buffer_free()` exactly once per non-empty buffer +//! - Empty (len=0) buffer has null `data`; buffer_free accepts it as no-op +//! +//! This struct is #[repr(C)] so cbindgen generates a matching C struct. + +use std::ptr; + +#[repr(C)] +pub struct PaimonTantivyBuffer { + /// Pointer to `len` bytes. Null iff len == 0. + pub data: *mut u8, + /// Number of valid bytes. + pub len: usize, + /// Internal capacity hint for Rust-side reconstruction. C++ treats as opaque. + pub capacity: usize, +} + +impl PaimonTantivyBuffer { + /// Build a buffer from owned bytes; consumes the Vec. + pub(crate) fn from_vec(mut v: Vec) -> Self { + if v.is_empty() { + return Self::empty(); + } + v.shrink_to_fit(); + let len = v.len(); + let capacity = v.capacity(); + let data = v.as_mut_ptr(); + std::mem::forget(v); + Self { data, len, capacity } + } + + pub(crate) fn empty() -> Self { + Self { + data: ptr::null_mut(), + len: 0, + capacity: 0, + } + } +} + +/// Free a buffer returned by any Rust FFI function. Safe to call on an empty +/// buffer (len=0 / data=null). Must only be called once per buffer. +/// +/// SAFETY: `buf` must be either null, or point to a live `paimon_tantivy_buffer_t` +/// produced by this crate and not yet freed. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_buffer_free(buf: *mut PaimonTantivyBuffer) { + if buf.is_null() { + return; + } + let b = unsafe { &mut *buf }; + if b.len != 0 && !b.data.is_null() { + // Reconstruct the Vec and drop it + let v = unsafe { Vec::from_raw_parts(b.data, b.len, b.capacity) }; + drop(v); + } + b.data = ptr::null_mut(); + b.len = 0; + b.capacity = 0; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_has_null_data() { + let b = PaimonTantivyBuffer::empty(); + assert!(b.data.is_null()); + assert_eq!(b.len, 0); + } + + #[test] + fn from_vec_roundtrip() { + let src = vec![1u8, 2, 3, 4, 5]; + let src_clone = src.clone(); + let mut b = PaimonTantivyBuffer::from_vec(src); + assert_eq!(b.len, 5); + assert!(!b.data.is_null()); + let view: &[u8] = unsafe { std::slice::from_raw_parts(b.data, b.len) }; + assert_eq!(view, src_clone.as_slice()); + unsafe { paimon_tantivy_buffer_free(&mut b) }; + assert!(b.data.is_null()); + assert_eq!(b.len, 0); + } + + #[test] + fn free_null_is_noop() { + unsafe { paimon_tantivy_buffer_free(std::ptr::null_mut()) }; + } + + #[test] + fn free_empty_is_noop() { + let mut b = PaimonTantivyBuffer::empty(); + unsafe { paimon_tantivy_buffer_free(&mut b) }; + } + + #[test] + fn stress_alloc_free() { + // LSAN would catch any leak + for i in 0..5_000usize { + let mut b = PaimonTantivyBuffer::from_vec(vec![42u8; i.min(256)]); + unsafe { paimon_tantivy_buffer_free(&mut b) }; + } + } +} diff --git a/third_party/tantivy_ffi/src/callback_directory.rs b/third_party/tantivy_ffi/src/callback_directory.rs new file mode 100644 index 00000000..fabeb3cb --- /dev/null +++ b/third_party/tantivy_ffi/src/callback_directory.rs @@ -0,0 +1,515 @@ +//! PaimonCallbackDirectory: streaming tantivy `Directory` backed by C FFI +//! callbacks. Replaces the V1 `PaimonDirectory` (RamDirectory wrapper) with a +//! callback-driven design that mirrors Java paimon-tantivy-jni's `JniDirectory`. +//! +//! ## Why callback-based? +//! +//! Loading the entire archive (100MB+) into `RamDirectory` at reader +//! construction would give ~2x archive peak RAM and pay the whole download +//! cost up front even for small queries. This directory keeps just the +//! `HashMap` layout and issues pread calls through the FFI +//! callback whenever tantivy asks for bytes — peak RAM is ~KB, startup is +//! ~header size. +//! +//! ## Concurrency +//! +//! `read_at` is serialized via `stream_mutex` (same as Java JniDir's +//! `stream_lock`). pread-style callbacks in principle allow concurrent reads, +//! but some `paimon::InputStream` subclasses (notably `JindoInputStream`) +//! have shared-state races, so we play it safe. + +use std::collections::HashMap; +use std::ffi::c_void; +use std::fmt; +use std::io; +use std::ops::Range; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use tantivy::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError}; +use tantivy::directory::{ + AntiCallToken, Directory, DirectoryLock, FileHandle, Lock, OwnedBytes, TerminatingWrite, + WatchCallback, WatchHandle, WritePtr, +}; +use tantivy::HasLen; + +// ========================================================================= +// FFI types +// ========================================================================= + +/// pread-style callback table passed from C++ at reader construction. +/// +/// `ctx` is an opaque pointer to C++'s `StreamCtx` (holding a +/// `paimon::InputStream`). Rust never dereferences it — only forwards it +/// into the callback functions. `release` is called exactly once when the +/// last `Arc` is dropped. +#[repr(C)] +pub struct PaimonStreamCallbacks { + pub ctx: *mut c_void, + pub read_at: + extern "C" fn(ctx: *mut c_void, offset: u64, len: usize, out_buf: *mut u8) -> i32, + pub release: extern "C" fn(ctx: *mut c_void), +} + +// ========================================================================= +// Internal state +// ========================================================================= + +#[derive(Clone, Debug)] +struct FileMeta { + offset: u64, + length: u64, +} + +/// RAII wrapper owning the FFI callbacks. On drop, invokes `release(ctx)`. +/// Shared across clones of `PaimonCallbackDirectory` via `Arc`. +struct CallbackCtx { + callbacks: PaimonStreamCallbacks, +} + +impl Drop for CallbackCtx { + fn drop(&mut self) { + // Calling an extern "C" fn pointer from safe Rust is legal; the + // contract safety relies on the C++ side providing a valid ctx. + (self.callbacks.release)(self.callbacks.ctx); + } +} + +// Safety: callbacks.ctx is treated as opaque; C++ owner is responsible for +// the ctx being usable across threads. Rust's stream_mutex serializes +// read_at calls, and release is only invoked once (when Arc refcount hits 0). +unsafe impl Send for CallbackCtx {} +unsafe impl Sync for CallbackCtx {} + +// ========================================================================= +// PaimonCallbackDirectory +// ========================================================================= + +#[derive(Clone)] +pub struct PaimonCallbackDirectory { + /// name → (offset, length) in the stream. Immutable after construction. + layout: Arc>, + /// FFI callbacks + their ctx lifetime. + ctx: Arc, + /// tantivy writes small atomic files (`.lock`, in some paths `meta.json`) + /// via `atomic_write`; we keep them in memory instead of pushing back + /// through C++ (read-only archive). Shared across clones. + atomic_data: Arc>>>, + /// Serialize seek+read (mirrors Java JniDir's `stream_lock`) to guard + /// against shared-state races in some InputStream subclasses. + stream_mutex: Arc>, +} + +impl fmt::Debug for PaimonCallbackDirectory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PaimonCallbackDirectory") + .field("files", &self.layout.keys().collect::>()) + .finish() + } +} + +impl PaimonCallbackDirectory { + /// Construct a new directory from the C++-parsed archive layout + callbacks. + /// The ctx ownership transfers to this Directory; `release` is invoked on + /// drop of the last clone. + pub fn new( + entries: Vec<(String, u64, u64)>, + callbacks: PaimonStreamCallbacks, + ) -> Self { + let mut layout = HashMap::with_capacity(entries.len()); + for (name, offset, length) in entries { + layout.insert(PathBuf::from(name), FileMeta { offset, length }); + } + Self { + layout: Arc::new(layout), + ctx: Arc::new(CallbackCtx { callbacks }), + atomic_data: Arc::new(Mutex::new(HashMap::new())), + stream_mutex: Arc::new(Mutex::new(())), + } + } + + /// Perform an FFI pread. Serialized via `stream_mutex`. + fn pread(&self, offset: u64, len: usize) -> io::Result> { + let _guard = self.stream_mutex.lock().map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("stream_mutex poisoned: {e}")) + })?; + let mut buf = vec![0u8; len]; + // Calling extern "C" fn pointer — safe from Rust's POV (ABI is C); + // the contract safety (ctx validity, buffer ownership) is on the C++ side. + let rc = + (self.ctx.callbacks.read_at)(self.ctx.callbacks.ctx, offset, len, buf.as_mut_ptr()); + if rc != 0 { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("pread callback rc={rc} offset={offset} len={len}"), + )); + } + Ok(buf) + } + + /// Sorted file names, for diagnostic / test use. + #[cfg(test)] + pub(crate) fn file_names(&self) -> Vec { + let mut names: Vec = self + .layout + .keys() + .map(|p| p.to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } +} + +// ========================================================================= +// FileHandle +// ========================================================================= + +#[derive(Clone)] +struct PaimonCallbackFileHandle { + directory: PaimonCallbackDirectory, + file_offset: u64, + file_length: u64, +} + +impl fmt::Debug for PaimonCallbackFileHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PaimonCallbackFileHandle") + .field("offset", &self.file_offset) + .field("length", &self.file_length) + .finish() + } +} + +impl HasLen for PaimonCallbackFileHandle { + fn len(&self) -> usize { + self.file_length as usize + } +} + +impl FileHandle for PaimonCallbackFileHandle { + fn read_bytes(&self, range: Range) -> io::Result { + let start = self.file_offset + range.start as u64; + let len = range.end - range.start; + let data = self.directory.pread(start, len)?; + Ok(OwnedBytes::new(data)) + } +} + +// ========================================================================= +// Directory trait (13 methods for tantivy 0.22) +// ========================================================================= + +impl Directory for PaimonCallbackDirectory { + fn get_file_handle(&self, path: &Path) -> Result, OpenReadError> { + let meta = self + .layout + .get(path) + .ok_or_else(|| OpenReadError::FileDoesNotExist(path.to_path_buf()))?; + Ok(Arc::new(PaimonCallbackFileHandle { + directory: self.clone(), + file_offset: meta.offset, + file_length: meta.length, + })) + } + + fn exists(&self, path: &Path) -> Result { + let in_layout = self.layout.contains_key(path); + let in_atomic = self.atomic_data.lock().unwrap().contains_key(path); + Ok(in_layout || in_atomic) + } + + fn atomic_read(&self, path: &Path) -> Result, OpenReadError> { + if let Some(data) = self.atomic_data.lock().unwrap().get(path) { + return Ok(data.clone()); + } + let meta = self + .layout + .get(path) + .ok_or_else(|| OpenReadError::FileDoesNotExist(path.to_path_buf()))?; + self.pread(meta.offset, meta.length as usize) + .map_err(|e| OpenReadError::wrap_io_error(e, path.to_path_buf())) + } + + fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { + self.atomic_data + .lock() + .unwrap() + .insert(path.to_path_buf(), data.to_vec()); + Ok(()) + } + + fn delete(&self, _path: &Path) -> Result<(), DeleteError> { + // read-only archive: ignore + Ok(()) + } + + fn open_write(&self, _path: &Path) -> Result { + // tantivy needs this for lock files when opening an index; provide a + // dummy in-memory writer (same trick as Java JniDirectory). + let buf: Vec = Vec::new(); + Ok(io::BufWriter::new(Box::new(VecTerminatingWrite(buf)))) + } + + fn sync_directory(&self) -> io::Result<()> { + Ok(()) + } + + fn acquire_lock(&self, _lock: &Lock) -> Result { + // Read-only: no actual locking. + Ok(DirectoryLock::from(Box::new(()))) + } + + fn watch(&self, _watch_callback: WatchCallback) -> tantivy::Result { + Ok(WatchHandle::empty()) + } +} + +/// Throwaway writer for `open_write` — tantivy creates it for lock files but +/// the bytes never matter in a read-only archive. +struct VecTerminatingWrite(Vec); + +impl io::Write for VecTerminatingWrite { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl TerminatingWrite for VecTerminatingWrite { + fn terminate_ref(&mut self, _token: AntiCallToken) -> io::Result<()> { + Ok(()) + } +} + +// ========================================================================= +// Test support (pub(crate) — used by reader.rs tests too) +// ========================================================================= + +#[cfg(test)] +pub(crate) mod test_support { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Mock backend: an in-memory buffer serving pread requests. Counters + /// expose behavior for test assertions (read count / release count). + pub(crate) struct MockBackend { + pub data: Vec, + pub read_count: AtomicUsize, + pub release_count: AtomicUsize, + } + + extern "C" fn mock_read_at( + ctx: *mut c_void, + offset: u64, + len: usize, + out_buf: *mut u8, + ) -> i32 { + let backend = unsafe { &*(ctx as *const MockBackend) }; + backend.read_count.fetch_add(1, Ordering::SeqCst); + let data = &backend.data; + let end = (offset as usize).saturating_add(len); + if end > data.len() { + return 1; // out of range + } + unsafe { + std::ptr::copy_nonoverlapping(data.as_ptr().add(offset as usize), out_buf, len); + } + 0 + } + + extern "C" fn mock_release(ctx: *mut c_void) { + // Reclaim the strong ref that `Arc::into_raw` leaked at construction. + let backend = unsafe { Arc::from_raw(ctx as *const MockBackend) }; + backend.release_count.fetch_add(1, Ordering::SeqCst); + // `arc` drops here → decrement; test still holds its own clone. + } + + /// Build a mock-backed directory for tests. Returns (dir, backend clone). + /// The backend Arc is shared — drop the directory to trigger release. + pub(crate) fn build_mock_directory( + data: Vec, + entries: Vec<(String, u64, u64)>, + ) -> (PaimonCallbackDirectory, Arc) { + let backend = Arc::new(MockBackend { + data, + read_count: AtomicUsize::new(0), + release_count: AtomicUsize::new(0), + }); + let ctx_ptr = Arc::into_raw(backend.clone()) as *mut c_void; + let cb = PaimonStreamCallbacks { + ctx: ctx_ptr, + read_at: mock_read_at, + release: mock_release, + }; + let dir = PaimonCallbackDirectory::new(entries, cb); + (dir, backend) + } + + /// Build mock callbacks (+ a backend clone) without wrapping them in a + /// directory, for tests that drive the FFI entry points directly. + pub(crate) fn make_mock_callbacks(data: Vec) -> (PaimonStreamCallbacks, Arc) { + let backend = Arc::new(MockBackend { + data, + read_count: AtomicUsize::new(0), + release_count: AtomicUsize::new(0), + }); + let ctx_ptr = Arc::into_raw(backend.clone()) as *mut c_void; + let cb = PaimonStreamCallbacks { + ctx: ctx_ptr, + read_at: mock_read_at, + release: mock_release, + }; + (cb, backend) + } + + /// Parse a packed archive blob (BE, no version header, matching + /// `writer::pack_index_dir`) and build a mock-backed directory. Used by + /// `reader.rs::tests` since writer.finish currently still returns a Vec. + pub(crate) fn build_directory_from_archive( + packed: Vec, + ) -> (PaimonCallbackDirectory, Arc) { + let entries = parse_archive_header(&packed); + build_mock_directory(packed, entries) + } + + /// Parse the archive header — mirrors the layout that + /// C++ `ArchiveLayout::Parse` produces in production. + fn parse_archive_header(bytes: &[u8]) -> Vec<(String, u64, u64)> { + let mut off = 0usize; + let file_count = i32::from_be_bytes(bytes[off..off + 4].try_into().unwrap()) as usize; + off += 4; + let mut entries = Vec::with_capacity(file_count); + for _ in 0..file_count { + let nlen = i32::from_be_bytes(bytes[off..off + 4].try_into().unwrap()) as usize; + off += 4; + let name = + std::str::from_utf8(&bytes[off..off + nlen]).unwrap().to_owned(); + off += nlen; + let flen = i64::from_be_bytes(bytes[off..off + 8].try_into().unwrap()) as u64; + off += 8; + let data_offset = off as u64; + entries.push((name, data_offset, flen)); + off += flen as usize; + } + entries + } +} + +#[cfg(test)] +mod tests { + use super::test_support::*; + use super::*; + + #[test] + fn file_handle_reads_correct_bytes() { + let data = b"hello world".to_vec(); + let entries = vec![("foo.txt".to_string(), 0, 11)]; + let (dir, _backend) = build_mock_directory(data, entries); + + let handle = dir.get_file_handle(Path::new("foo.txt")).unwrap(); + let bytes = handle.read_bytes(0..5).unwrap(); + assert_eq!(&bytes[..], b"hello"); + let bytes = handle.read_bytes(6..11).unwrap(); + assert_eq!(&bytes[..], b"world"); + } + + #[test] + fn missing_file_returns_error() { + let (dir, _backend) = build_mock_directory(vec![], vec![]); + let err = dir.get_file_handle(Path::new("nonexistent")).unwrap_err(); + match err { + OpenReadError::FileDoesNotExist(p) => { + assert_eq!(p.to_string_lossy(), "nonexistent") + } + other => panic!("expected FileDoesNotExist, got {other:?}"), + } + } + + #[test] + fn pread_out_of_range_propagates_error() { + let data = b"short".to_vec(); + let entries = vec![("bad.txt".to_string(), 0, 100)]; // length exceeds data + let (dir, _backend) = build_mock_directory(data, entries); + let handle = dir.get_file_handle(Path::new("bad.txt")).unwrap(); + let err = handle.read_bytes(0..100).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::Other); + } + + #[test] + fn atomic_write_read_roundtrip_and_exists() { + let (dir, _backend) = build_mock_directory(vec![], vec![]); + dir.atomic_write(Path::new(".lock"), b"locked").unwrap(); + let data = dir.atomic_read(Path::new(".lock")).unwrap(); + assert_eq!(data, b"locked"); + assert!(dir.exists(Path::new(".lock")).unwrap()); + assert!(!dir.exists(Path::new("gone")).unwrap()); + } + + #[test] + fn release_called_exactly_once_on_last_drop() { + let entries = vec![("a".to_string(), 0, 5)]; + let (dir, backend) = build_mock_directory(b"hello".to_vec(), entries); + assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 0); + drop(dir); + assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[test] + fn cloned_directory_shares_ctx_and_atomic_data() { + let (dir, backend) = build_mock_directory(vec![], vec![]); + let dir2 = dir.clone(); + dir.atomic_write(Path::new("x"), b"hello").unwrap(); + assert!(dir2.exists(Path::new("x")).unwrap()); // shared atomic_data + drop(dir); + assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 0); // ctx still held by dir2 + drop(dir2); + assert_eq!(backend.release_count.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[test] + fn concurrent_pread_results_correct_under_stream_mutex() { + use std::thread; + + let data: Vec = (0..1000).map(|i| (i % 256) as u8).collect(); + let entries = vec![("data".to_string(), 0, 1000)]; + let (dir, backend) = build_mock_directory(data.clone(), entries); + let handle: Arc = + dir.get_file_handle(Path::new("data")).unwrap(); + + let threads: Vec<_> = (0..8) + .map(|_| { + let h = handle.clone(); + let expected = data.clone(); + thread::spawn(move || { + for _ in 0..20 { + let bytes = h.read_bytes(100..200).unwrap(); + assert_eq!(&bytes[..], &expected[100..200]); + } + }) + }) + .collect(); + + for t in threads { + t.join().unwrap(); + } + assert_eq!( + backend.read_count.load(std::sync::atomic::Ordering::SeqCst), + 8 * 20 + ); + } + + #[test] + fn file_names_sorted() { + let entries = vec![ + ("z.idx".to_string(), 0, 10), + ("a.meta".to_string(), 10, 20), + ("m.term".to_string(), 30, 5), + ]; + let (dir, _backend) = build_mock_directory(vec![0u8; 100], entries); + let names = dir.file_names(); + assert_eq!(names, vec!["a.meta", "m.term", "z.idx"]); + } +} diff --git a/third_party/tantivy_ffi/src/error.rs b/third_party/tantivy_ffi/src/error.rs new file mode 100644 index 00000000..6be463c7 --- /dev/null +++ b/third_party/tantivy_ffi/src/error.rs @@ -0,0 +1,137 @@ +//! Error model for paimon_tantivy_ffi. +//! +//! Contract: +//! - Every fallible FFI function returns `paimon_tantivy_status_t` +//! - Failure sets `last_error` (thread-local) with human-readable text +//! - C++ calls `paimon_tantivy_last_error()` after a non-OK status to fetch text +//! - Pointer returned by `last_error()` is thread-local and valid until the +//! next failing FFI call on the same thread. C++ must NOT free it. + +use std::cell::RefCell; +use std::ffi::c_char; +use std::ffi::CString; + +/// Status codes. Values are stable ABI; append-only. +#[repr(i32)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum PaimonTantivyStatus { + Ok = 0, + InvalidArgument = 1, + NotFound = 2, + IoError = 3, + Unsupported = 4, + TokenizerError = 5, + QueryParseError = 6, + IndexFormatError = 7, + InternalError = 99, +} + +thread_local! { + /// Pre-allocated empty string so `paimon_tantivy_last_error()` can always + /// return a valid non-null pointer. + static LAST_ERROR: RefCell = RefCell::new(CString::new("").unwrap()); +} + +/// Record an error message for the current thread. Called by fallible FFI +/// functions right before returning a non-OK status. +pub(crate) fn set_last_error(msg: impl Into) { + // Interior nul bytes would make CString::new fail; strip them as a safety net. + let s: String = msg.into().replace('\0', "\u{FFFD}"); + LAST_ERROR.with(|cell| { + // CString::new clones the bytes and appends a nul terminator. + *cell.borrow_mut() = CString::new(s).unwrap_or_else(|_| CString::new("").unwrap()); + }); +} + +/// Clear the current thread's error slot. Called at the top of fallible APIs +/// so a subsequent successful call doesn't return stale text. +#[allow(dead_code)] +pub(crate) fn clear_last_error() { + LAST_ERROR.with(|cell| { + *cell.borrow_mut() = CString::new("").unwrap(); + }); +} + +/// Macro that wraps a `Result`-returning block: sets last_error on +/// Err and returns the given status code; returns Ok value on success. +#[macro_export] +macro_rules! ffi_try { + ($expr:expr, $err_status:expr) => {{ + match $expr { + Ok(v) => v, + Err(e) => { + $crate::error::set_last_error(format!("{e}")); + return $err_status; + } + } + }}; +} + +/// Return the last error text for the calling thread. Always non-null; returns +/// pointer to "" when there is no error recorded yet. Pointer is thread-local; +/// C++ must NOT free it; treat as valid until the next failing FFI call on +/// the same thread. +#[no_mangle] +pub extern "C" fn paimon_tantivy_last_error() -> *const c_char { + LAST_ERROR.with(|cell| cell.borrow().as_ptr()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CStr; + + #[test] + fn initial_last_error_is_empty() { + let ptr = paimon_tantivy_last_error(); + assert!(!ptr.is_null()); + let s = unsafe { CStr::from_ptr(ptr) }.to_str().unwrap(); + assert_eq!(s, ""); + } + + #[test] + fn set_then_retrieve() { + set_last_error("boom"); + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + assert_eq!(s, "boom"); + } + + #[test] + fn clear_resets_to_empty() { + set_last_error("x"); + clear_last_error(); + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + assert_eq!(s, ""); + } + + #[test] + fn embedded_nul_is_stripped() { + set_last_error("a\0b"); + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + assert_eq!(s, "a\u{FFFD}b"); + } + + #[test] + fn thread_local_isolation() { + set_last_error("main"); + let t = std::thread::spawn(|| { + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + s.to_owned() + }) + .join() + .unwrap(); + assert_eq!(t, ""); + let s = unsafe { CStr::from_ptr(paimon_tantivy_last_error()) } + .to_str() + .unwrap(); + assert_eq!(s, "main"); + } +} diff --git a/third_party/tantivy_ffi/src/handle.rs b/third_party/tantivy_ffi/src/handle.rs new file mode 100644 index 00000000..6ec776a2 --- /dev/null +++ b/third_party/tantivy_ffi/src/handle.rs @@ -0,0 +1,106 @@ +//! Opaque handle helpers. +//! +//! Contract: +//! - Rust creates handles with `Box::into_raw(Box::new(T))` +//! - C++ must free with the matching `xxx_free(*mut T)` function, once +//! - Functions accepting handles treat null as invalid argument + +use std::ffi::c_void; + +/// Consume `T`, return a raw opaque pointer suitable for C++. +#[inline] +pub(crate) fn into_handle(value: T) -> *mut T { + Box::into_raw(Box::new(value)) +} + +/// Reconstitute a `Box` from an FFI-provided pointer and drop it. +/// SAFETY: caller must pass a pointer previously returned by `into_handle::`, +/// and must not use it again after this call. +#[inline] +pub(crate) unsafe fn free_handle(handle: *mut T) { + if handle.is_null() { + return; + } + drop(unsafe { Box::from_raw(handle) }); +} + +/// Borrow an `&T` from an FFI-provided pointer. Returns None on null. +/// SAFETY: caller must ensure the pointer was previously returned by +/// `into_handle::` and is still alive (not freed). +#[inline] +pub(crate) unsafe fn borrow_handle<'a, T>(handle: *const T) -> Option<&'a T> { + if handle.is_null() { + None + } else { + Some(unsafe { &*handle }) + } +} + +/// Borrow `&mut T` from an FFI-provided pointer. Returns None on null. +/// SAFETY: same as `borrow_handle`, plus caller must ensure there is no +/// concurrent access via another pointer (writer/reader handles are +/// documented as thread-unsafe). +#[inline] +pub(crate) unsafe fn borrow_handle_mut<'a, T>(handle: *mut T) -> Option<&'a mut T> { + if handle.is_null() { + None + } else { + Some(unsafe { &mut *handle }) + } +} + +/// Opaque ctx pointer from C++ (passed through to Rust Directory callbacks). +/// Type-erased on purpose: only C++ side knows the concrete type. +pub(crate) type OpaqueCtx = *mut c_void; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn into_then_free() { + struct X(i32); + let h: *mut X = into_handle(X(42)); + assert!(!h.is_null()); + unsafe { free_handle(h) }; + // no leak (LSAN would catch if compiled with sanitizers) + } + + #[test] + fn free_null_is_noop() { + let h: *mut i32 = std::ptr::null_mut(); + unsafe { free_handle(h) }; + } + + #[test] + fn borrow_roundtrip() { + let h = into_handle(42i32); + unsafe { + assert_eq!(*borrow_handle(h as *const i32).unwrap(), 42); + *borrow_handle_mut(h).unwrap() = 7; + assert_eq!(*borrow_handle(h as *const i32).unwrap(), 7); + free_handle(h); + } + } + + #[test] + fn borrow_null_is_none() { + unsafe { + assert!(borrow_handle::(std::ptr::null()).is_none()); + assert!(borrow_handle_mut::(std::ptr::null_mut()).is_none()); + } + } + + #[test] + fn stress_many_create_destroy() { + // smoke stress: many allocations, no leak + for i in 0..10_000 { + let h = into_handle(vec![i; 8]); + unsafe { + let v = borrow_handle(h as *const Vec).unwrap(); + assert_eq!(v.len(), 8); + free_handle(h); + } + } + } +} diff --git a/third_party/tantivy_ffi/src/lib.rs b/third_party/tantivy_ffi/src/lib.rs new file mode 100644 index 00000000..fc544c90 --- /dev/null +++ b/third_party/tantivy_ffi/src/lib.rs @@ -0,0 +1,79 @@ +//! paimon_tantivy_ffi: C ABI layer for tantivy + jieba-rs, +//! consumed by paimon-cpp's `tantivy-fulltext` global index. +//! +//! Modules: error / handle / buffer / log (common FFI layer), tokenizer, +//! writer, callback directory, and reader (query). + +#![deny(unsafe_op_in_unsafe_fn)] + +use std::ffi::c_char; + +pub mod error; +pub mod handle; +pub mod buffer; +pub mod log_bridge; +pub mod tokenizer; +pub mod writer; +pub mod callback_directory; +pub mod reader; + +// Re-export public FFI symbols at crate root so cbindgen picks them up. +pub use buffer::{paimon_tantivy_buffer_free, PaimonTantivyBuffer}; +pub use error::{paimon_tantivy_last_error, PaimonTantivyStatus}; +pub use log_bridge::{ + paimon_tantivy_clear_log_callback, paimon_tantivy_set_log_callback, PaimonTantivyLogFn, +}; +pub use tokenizer::{ + paimon_tantivy_tokenizer_free, paimon_tantivy_tokenizer_new, + paimon_tantivy_tokenizer_tokenize, PaimonJiebaTokenizer, +}; +pub use writer::{ + paimon_tantivy_writer_add, paimon_tantivy_writer_finish_streaming, + paimon_tantivy_writer_free, paimon_tantivy_writer_new, PaimonTantivyWriter, + PaimonWriteCallbacks, +}; +pub use callback_directory::{PaimonCallbackDirectory, PaimonStreamCallbacks}; +pub use reader::{ + paimon_tantivy_reader_free, paimon_tantivy_reader_new_streaming, + paimon_tantivy_reader_search, PaimonTantivyReader, +}; + +/// Semantic version of this crate, **'static lifetime**; C++ must NOT free. +/// Format: `""` (git sha postfix can be added later via build.rs). +/// Returned as a NUL-terminated UTF-8 C string. +#[no_mangle] +pub extern "C" fn paimon_tantivy_version() -> *const c_char { + concat!(env!("CARGO_PKG_VERSION"), "\0").as_ptr() as *const c_char +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CStr; + + #[test] + fn version_is_non_empty() { + let ptr = paimon_tantivy_version(); + assert!(!ptr.is_null()); + let s = unsafe { CStr::from_ptr(ptr) }.to_str().unwrap(); + assert!(!s.is_empty(), "version must be non-empty"); + assert!(s.contains('.'), "version must look like semver, got {s:?}"); + } + + #[test] + fn tantivy_and_jieba_are_linked() { + let _ = tantivy::schema::Schema::builder(); + let _ = jieba_rs::Jieba::new(); + } + + #[test] + fn croaring_serialize_roundtrip() { + use croaring::Bitmap; + let mut b = Bitmap::new(); + b.add(42); + b.add(100); + let bytes = b.serialize::(); + let b2 = Bitmap::deserialize::(&bytes); + assert_eq!(b.cardinality(), b2.cardinality()); + } +} diff --git a/third_party/tantivy_ffi/src/log_bridge.rs b/third_party/tantivy_ffi/src/log_bridge.rs new file mode 100644 index 00000000..7f4ab00f --- /dev/null +++ b/third_party/tantivy_ffi/src/log_bridge.rs @@ -0,0 +1,103 @@ +//! Log bridge: tantivy internally emits log records via the `log` crate +//! (via `tantivy::debug` / `info` etc.). This module registers a global +//! `log::Log` implementation that forwards records to a C callback. +//! +//! Contract: +//! - C++ calls `paimon_tantivy_set_log_callback(cb)` once at process startup +//! - Passing null unregisters (reverts to stderr) +//! - Callback receives (level, msg_ptr, msg_len); pointer is non-null, +//! UTF-8, NOT null-terminated, valid only for the duration of the call +//! - Level mapping: 0=trace 1=debug 2=info 3=warn 4=error +//! - Callback must be thread-safe: tantivy writes from worker threads +//! +//! NOTE: tantivy uses `tracing` in newer versions and `log` in others. +//! Our current `tantivy = "0.22"` uses `log`. +//! If a future upgrade switches to `tracing`, install a `tracing-log` +//! bridge here. + +use std::ffi::c_char; +use std::sync::atomic::{AtomicPtr, Ordering}; + +pub type PaimonTantivyLogFn = extern "C" fn(level: i32, msg: *const c_char, len: usize); + +static CALLBACK: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut()); + +struct LogBridge; + +impl log::Log for LogBridge { + fn enabled(&self, _: &log::Metadata) -> bool { + true + } + + fn log(&self, record: &log::Record) { + let level = match record.level() { + log::Level::Trace => 0, + log::Level::Debug => 1, + log::Level::Info => 2, + log::Level::Warn => 3, + log::Level::Error => 4, + }; + let msg = format!("[{}] {}", record.target(), record.args()); + let ptr = CALLBACK.load(Ordering::Acquire); + if ptr.is_null() { + // Fallback: stderr + eprintln!("{msg}"); + return; + } + // SAFETY: ptr was installed as PaimonTantivyLogFn via transmute below + let cb: PaimonTantivyLogFn = unsafe { std::mem::transmute(ptr) }; + cb(level, msg.as_ptr() as *const c_char, msg.len()); + } + + fn flush(&self) {} +} + +static LOGGER: LogBridge = LogBridge; + +/// Install a non-null callback. First call also registers `LogBridge` as +/// the global `log` crate sink. Subsequent calls swap the callback atomically. +/// Thread-safety: safe to call from any thread. +/// +/// Note: we use separate `set`/`clear` functions instead of `Option` +/// because cbindgen translates `Option` into an opaque struct +/// rather than a nullable C function pointer. +#[no_mangle] +pub extern "C" fn paimon_tantivy_set_log_callback(cb: PaimonTantivyLogFn) { + let ptr = cb as *mut (); + CALLBACK.store(ptr, Ordering::Release); + let _ = log::set_logger(&LOGGER); + log::set_max_level(log::LevelFilter::Info); +} + +/// Clear the installed callback (revert to Rust-side stderr fallback). +#[no_mangle] +pub extern "C" fn paimon_tantivy_clear_log_callback() { + CALLBACK.store(std::ptr::null_mut(), Ordering::Release); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + // Simple test callback that counts invocations + static COUNT: AtomicUsize = AtomicUsize::new(0); + extern "C" fn counting_cb(_: i32, _: *const c_char, _: usize) { + COUNT.fetch_add(1, Ordering::SeqCst); + } + + #[test] + fn install_then_log() { + COUNT.store(0, Ordering::SeqCst); + paimon_tantivy_set_log_callback(counting_cb); + log::info!("hello"); + assert!(COUNT.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn clear_reverts_to_stderr() { + paimon_tantivy_set_log_callback(counting_cb); + paimon_tantivy_clear_log_callback(); + log::warn!("goes to stderr"); + } +} diff --git a/third_party/tantivy_ffi/src/reader.rs b/third_party/tantivy_ffi/src/reader.rs new file mode 100644 index 00000000..4bd8e0fa --- /dev/null +++ b/third_party/tantivy_ffi/src/reader.rs @@ -0,0 +1,1298 @@ +//! PaimonTantivyReader: query side of tantivy-fulltext. +//! +//! Constructs a tantivy Index from a packed-blob produced by writer.rs (via +//! PaimonDirectory), registers the same `paimon_jieba` tokenizer, and runs +//! one of 5 search types (mirrors `paimon::FullTextSearch::SearchType`): +//! +//! 1 MATCH_ALL — tokenize query, BooleanQuery (Must) +//! 2 MATCH_ANY — tokenize query, BooleanQuery (Should) +//! 3 PHRASE — tokenize query, PhraseQuery +//! 4 PREFIX — RegexQuery `.*` (no tokenization, mirrors lucene-fts) +//! 5 WILDCARD — RegexQuery from glob pattern (`*` → `.*`, `?` → `.`, others escaped) +//! +//! For paimon-java compatibility, row_id is stored as an explicit u64 field +//! (`fast` for O(1) retrieval). Reader translates tantivy DocAddress → row_id +//! via `fast_fields().u64("row_id").first(doc_id)` per segment. +//! +//! FFI return format (little-endian, **doc identifiers are u64 row_ids**): +//! `[u8 has_scores | u64 count | u64 row_id[count] | optional f32 score[count]]` + +use std::ffi::{c_char, CStr}; +use std::path::Path; + +use croaring::{Portable, Treemap}; +use tantivy::collector::{Collector, SegmentCollector}; +use tantivy::columnar::Column; +use tantivy::query::{BooleanQuery, Occur, PhraseQuery, Query, RegexQuery, TermQuery}; +use tantivy::schema::{Field, IndexRecordOption}; +use tantivy::{DocAddress, DocId, Index, IndexReader, ReloadPolicy, Score, SegmentOrdinal, + SegmentReader, Term}; + +use crate::buffer::PaimonTantivyBuffer; +use crate::callback_directory::{PaimonCallbackDirectory, PaimonStreamCallbacks}; +use crate::error::{set_last_error, PaimonTantivyStatus}; +use crate::handle::{borrow_handle_mut, free_handle, into_handle}; +use crate::tokenizer::{PaimonJiebaTokenizer, TokenizeMode}; +use crate::writer::{PAIMON_ROW_ID_FIELD_NAME, PAIMON_TEXT_FIELD_NAME, PAIMON_TOKENIZER_NAME}; + +/// Numeric encoding of `paimon::FullTextSearch::SearchType`. Kept in sync +/// with include/paimon/predicate/full_text_search.h. +#[repr(i32)] +#[derive(Clone, Copy, Debug)] +pub enum SearchType { + MatchAll = 1, + MatchAny = 2, + Phrase = 3, + Prefix = 4, + Wildcard = 5, +} + +impl SearchType { + fn from_i32(v: i32) -> Option { + match v { + 1 => Some(Self::MatchAll), + 2 => Some(Self::MatchAny), + 3 => Some(Self::Phrase), + 4 => Some(Self::Prefix), + 5 => Some(Self::Wildcard), + _ => None, + } + } +} + +pub struct PaimonTantivyReader { + /// Held alive so `IndexReader::searcher()` + `index.tokenizers()` stay + /// usable for the reader's lifetime. + index: Index, + reader: IndexReader, + text_field: Field, + /// Name of the tokenizer the `text` field is actually bound to in the open + /// index's schema (read from `meta.json` at construction time). Query-side + /// tokenization looks this up in `index.tokenizers()` every time + tokenizer_name: String, +} + +impl PaimonTantivyReader { + /// Construct a reader from a pre-built callback-backed Directory. + /// Layout (file names + offsets + lengths) must come from the caller + /// (C++ side `ArchiveLayout::Parse`); Rust does not re-parse the archive. + pub fn new( + directory: PaimonCallbackDirectory, + mode: TokenizeMode, + with_position: bool, + dict_dir: &Path, + ) -> Result { + let index = Index::open(directory) + .map_err(|e| format!("tantivy::Index::open: {e}"))?; + + // Resolve fields by their fixed names (schema is `row_id` + `text`). + let schema = index.schema(); + let text_field = schema.get_field(PAIMON_TEXT_FIELD_NAME).map_err(|e| { + format!("tantivy index missing '{PAIMON_TEXT_FIELD_NAME}' field: {e}") + })?; + + // Read the tokenizer name the `text` field was actually written with + // (lives in meta.json's schema). Auto-aligns cpp query-side tokenizer + // with whatever the writer side used. + let tokenizer_name = match schema.get_field_entry(text_field).field_type() { + tantivy::schema::FieldType::Str(text_options) => text_options + .get_indexing_options() + .map(|io| io.tokenizer().to_string()) + .unwrap_or_else(|| "default".to_string()), + other => { + return Err(format!( + "text field has non-TEXT type: {other:?} (schema corrupted?)" + )); + } + }; + + // Only register paimon_jieba if the index actually uses it. The + // tantivy-builtin "default" / "raw" / "en_stem" etc. are pre-registered + // by the TokenizerManager — no setup needed for those. + if tokenizer_name == PAIMON_TOKENIZER_NAME { + // `Path::is_empty` is unstable; check via OsStr. + if dict_dir.as_os_str().is_empty() { + return Err(format!( + "paimon_jieba tokenizer required by archive schema but dict dir \ + is empty — set the PAIMON_JIEBA_DICT_DIR env var to a directory \ + containing jieba.dict.utf8 / hmm_model.utf8 / user.dict.utf8 / \ + idf.utf8 / stop_words.utf8" + )); + } + let jieba = PaimonJiebaTokenizer::new(dict_dir, mode, with_position) + .map_err(|e| format!("create paimon_jieba tokenizer: {e}"))?; + index.tokenizers().register(PAIMON_TOKENIZER_NAME, jieba); + } else { + // For other known-safe names we trust tantivy's builtin registry. + // `mode` / `dict_dir` are unused in this branch — no-op; we still + // require them in the ABI for backward-compat with the jieba case. + let _ = (mode, dict_dir); + } + + // Sanity: the tokenizer MUST be resolvable now; otherwise query-time + // lookup fails mid-flight. + if index.tokenizers().get(&tokenizer_name).is_none() { + return Err(format!( + "tokenizer {tokenizer_name:?} referenced by text field is not \ + registered; add it to TokenizerManager before opening the reader" + )); + } + + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::Manual) + .try_into() + .map_err(|e| format!("build IndexReader: {e}"))?; + + Ok(Self { + index, + reader, + text_field, + tokenizer_name, + }) + } + + /// Tokenize the query string using the *same* tokenizer the index's text + /// field was built with. Looks up `self.tokenizer_name` in the index's + /// `TokenizerManager` — which was populated by `new()` with either + /// `paimon_jieba` (if cpp wrote the index) or a tantivy builtin like + /// `default` (if paimon-java wrote it). + fn tokenize_query(&self, query: &str) -> Vec { + // `TokenizerManager::get` returns a fresh clone per call — safe to use + // across threads / calls. If the tokenizer was missing we'd have + // failed in `new()`; we still defend with `unwrap_or_default`. + let mut analyzer = match self.index.tokenizers().get(&self.tokenizer_name) { + Some(a) => a, + None => return Vec::new(), + }; + let mut stream = analyzer.token_stream(query); + let mut out = Vec::new(); + while stream.advance() { + out.push(stream.token().text.clone()); + } + out + } + + fn build_match_query(&self, query: &str, occur: Occur) -> Result, String> { + let terms = self.tokenize_query(query); + if terms.is_empty() { + return Err(format!("query {query:?} produced no tokens after analysis")); + } + if terms.len() == 1 { + let term = Term::from_field_text(self.text_field, &terms[0]); + return Ok(Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs))); + } + let clauses: Vec<(Occur, Box)> = terms + .iter() + .map(|t| { + let term = Term::from_field_text(self.text_field, t); + let q: Box = + Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs)); + (occur, q) + }) + .collect(); + Ok(Box::new(BooleanQuery::new(clauses))) + } + + fn build_phrase_query(&self, query: &str) -> Result, String> { + let terms = self.tokenize_query(query); + if terms.is_empty() { + return Err(format!("phrase query {query:?} produced no tokens")); + } + if terms.len() == 1 { + // PhraseQuery requires >=2 terms in tantivy; degrade to TermQuery. + let term = Term::from_field_text(self.text_field, &terms[0]); + return Ok(Box::new(TermQuery::new(term, IndexRecordOption::WithFreqsAndPositions))); + } + let tantivy_terms: Vec = terms + .iter() + .map(|t| Term::from_field_text(self.text_field, t)) + .collect(); + Ok(Box::new(PhraseQuery::new(tantivy_terms))) + } + + fn build_prefix_query(&self, query: &str) -> Result, String> { + if query.is_empty() { + return Err("prefix query is empty".into()); + } + // Mirror lucene-fts: don't tokenize prefix; match indexed term bytes + // starting with the given prefix verbatim. + let pattern = format!("{}.*", regex_escape(query)); + RegexQuery::from_pattern(&pattern, self.text_field) + .map(|q| Box::new(q) as Box) + .map_err(|e| format!("RegexQuery from prefix {query:?}: {e}")) + } + + fn build_wildcard_query(&self, query: &str) -> Result, String> { + if query.is_empty() { + return Err("wildcard query is empty".into()); + } + let pattern = wildcard_to_regex(query); + RegexQuery::from_pattern(&pattern, self.text_field) + .map(|q| Box::new(q) as Box) + .map_err(|e| format!("RegexQuery from wildcard {query:?} (pattern {pattern}): {e}")) + } + + fn build_query(&self, search_type: SearchType, query: &str) -> Result, String> { + match search_type { + SearchType::MatchAll => self.build_match_query(query, Occur::Must), + SearchType::MatchAny => self.build_match_query(query, Occur::Should), + SearchType::Phrase => self.build_phrase_query(query), + SearchType::Prefix => self.build_prefix_query(query), + SearchType::Wildcard => self.build_wildcard_query(query), + } + } + + /// Return all matching row_ids (no scoring, no limit, no pre_filter). + /// row_ids come from the explicit `row_id` u64 fast field, supporting + /// multi-segment indexes (e.g. produced by paimon-java without force-merge). + pub fn search_all(&self, search_type: SearchType, query: &str) -> Result, String> { + let q = self.build_query(search_type, query)?; + let searcher = self.reader.searcher(); + let mut ids: Vec = searcher + .search(&*q, &RowIdCollector) + .map_err(|e| format!("tantivy search: {e}"))?; + ids.sort_unstable(); + ids.dedup(); + Ok(ids) + } + + /// 4-path dispatch on `(with_score, limit)` — see `docs/dev/tantivy_bm25_score_contract.md` + /// §4. + /// + /// | with_score | limit | path | collector | sort | truncate | output score | + /// |------------|--------|------|------------------------|----------------|----------|--------------| + /// | false | None | A | RowIdCollector | row_id asc | — | ❌ | + /// | false | Some(n)| B | AllScoredCollector | score desc | top n | ❌ (dropped) | + /// | true | None | C | AllScoredCollector | row_id asc | — | ✅ | + /// | true | Some(n)| D | AllScoredCollector | score desc | top n | ✅ | + /// + /// Pre-filter is a `Treemap` of paimon row_ids (not tantivy doc_ids), applied BEFORE + /// truncation so high-score matches outside the filter don't crowd out valid ones. + /// + /// **v0.2 contract change**: previously `limit.is_some()` implicitly triggered scoring; now + /// scoring is gated solely by `with_score`. + pub fn search_with_limit_and_filter( + &self, + search_type: SearchType, + query: &str, + with_score: bool, + limit: Option, + pre_filter: Option<&Treemap>, + min_score: Option, + ) -> Result)>, String> { + let q = self.build_query(search_type, query)?; + let searcher = self.reader.searcher(); + match (with_score, limit) { + // Path A: all rows, no score. RowIdCollector reads the `row_id` fast + // field inline per segment (opened once), avoiding a DocSetCollector + // HashSet and per-doc handle — hot path for high-cardinality counts. + (false, None) => { + let mut row_ids: Vec = searcher + .search(&*q, &RowIdCollector) + .map_err(|e| format!("tantivy search: {e}"))?; + if let Some(filter) = pre_filter { + row_ids.retain(|id| filter.contains(*id)); + } + row_ids.sort_unstable(); + row_ids.dedup(); + Ok(row_ids.into_iter().map(|id| (id, None)).collect()) + } + // Path B: any N matches, unscored. Used by SR's `WHERE MATCH ... LIMIT N` (no + // ORDER BY): pushes the limit down so each shard stops collecting once N hits + // are gathered per segment instead of materialising the full posting list. + // If the caller wants top-N by BM25 they should set `with_score=true` (Path D) + // and ignore the score values. + (false, Some(n)) => { + if n == 0 { + return Ok(Vec::new()); + } + if min_score.is_some() { + // min_score requires scoring — fall back to collect_scored path + let mut filtered = self.collect_scored(&*q, &searcher, pre_filter)?; + if let Some(threshold) = min_score { + filtered.retain(|(s, _)| *s > threshold); + } + let truncated = Self::sort_by_score_desc_truncate(filtered, n); + Ok(truncated.into_iter().map(|(_, id)| (id, None)).collect()) + } else if let Some(filter) = pre_filter { + // pre_filter present: it MUST be applied to the full match set + // before truncation. LimitedDocSetCollector stops after the + // first N raw matches, which could all be filtered out while + // valid matches exist further down the posting list — that + // would under-return (fewer than N, or even empty). So collect + // every matching row_id (filter-aware), then truncate to N. + let mut row_ids: Vec = searcher + .search(&*q, &RowIdCollector) + .map_err(|e| format!("tantivy search: {e}"))?; + row_ids.retain(|id| filter.contains(*id)); + row_ids.sort_unstable(); + row_ids.dedup(); + row_ids.truncate(n); + Ok(row_ids.into_iter().map(|id| (id, None)).collect()) + } else { + // No pre_filter: fast path — stop collecting once N matches are + // gathered per segment instead of materialising the full posting list. + let collector = LimitedDocSetCollector::new(n); + let mut docset = searcher + .search(&*q, &collector) + .map_err(|e| format!("tantivy search: {e}"))?; + let mut by_segment: std::collections::HashMap> = + std::collections::HashMap::new(); + for addr in docset.drain(..) { + by_segment.entry(addr.segment_ord).or_default().push(addr.doc_id); + } + let mut row_ids: Vec = Vec::new(); + for (segment_ord, doc_ids) in by_segment.iter() { + let segment_reader = searcher.segment_reader(*segment_ord); + let fast = segment_reader + .fast_fields() + .u64(PAIMON_ROW_ID_FIELD_NAME) + .map_err(|e| format!("fast_fields().u64('row_id') on segment {}: {e}", + segment_ord))?; + for &doc_id in doc_ids { + row_ids.push(fast.first(doc_id).unwrap_or(0)); + } + } + row_ids.sort_unstable(); + row_ids.dedup(); + row_ids.truncate(n); + Ok(row_ids.into_iter().map(|id| (id, None)).collect()) + } + } + // Path C: all rows + all scores, sorted by row_id asc to match the + // BitmapScoredGlobalIndexResult contract (bitmap iter order == score order). + (true, None) => { + let mut filtered = self.collect_scored(&*q, &searcher, pre_filter)?; + if let Some(threshold) = min_score { + filtered.retain(|(s, _)| *s > threshold); + } + filtered.sort_unstable_by(|a, b| a.1.cmp(&b.1)); + Ok(filtered.into_iter().map(|(s, id)| (id, Some(s))).collect()) + } + // Path D: top-N by BM25 with scores. + (true, Some(n)) => { + if n == 0 { + return Ok(Vec::new()); + } + let mut filtered = self.collect_scored(&*q, &searcher, pre_filter)?; + if let Some(threshold) = min_score { + filtered.retain(|(s, _)| *s > threshold); + } + let truncated = Self::sort_by_score_desc_truncate(filtered, n); + Ok(truncated.into_iter().map(|(s, id)| (id, Some(s))).collect()) + } + } + } + + /// Helper for paths B/C/D: run AllScoredCollector, translate doc_id → row_id, apply pre_filter. + /// Groups results by segment so the fast field column handle is opened once per segment + /// (same rationale as Path A — avoids per-match Column allocation). + fn collect_scored( + &self, + q: &dyn Query, + searcher: &tantivy::Searcher, + pre_filter: Option<&Treemap>, + ) -> Result, String> { + let scored = searcher + .search(q, &AllScoredCollector) + .map_err(|e| format!("tantivy search: {e}"))?; + let mut by_segment: std::collections::HashMap> = + std::collections::HashMap::new(); + for (s, addr) in scored.into_iter() { + by_segment.entry(addr.segment_ord).or_default().push((s, addr.doc_id)); + } + let mut result: Vec<(Score, u64)> = Vec::new(); + for (segment_ord, entries) in by_segment.iter() { + let segment_reader = searcher.segment_reader(*segment_ord); + let fast = segment_reader + .fast_fields() + .u64(PAIMON_ROW_ID_FIELD_NAME) + .map_err(|e| format!("fast_fields().u64('row_id') on segment {}: {e}", + segment_ord))?; + for &(score, doc_id) in entries { + let rid = fast.first(doc_id).unwrap_or(0); + if pre_filter.map_or(true, |t| t.contains(rid)) { + result.push((score, rid)); + } + } + } + Ok(result) + } + + /// Helper for paths B/D: sort (score, row_id) by score desc with row_id asc tie-break, + /// then truncate to `n` items. + fn sort_by_score_desc_truncate(mut v: Vec<(Score, u64)>, n: usize) -> Vec<(Score, u64)> { + v.sort_unstable_by(|a, b| { + b.0.partial_cmp(&a.0) + .unwrap_or(std::cmp::Ordering::Equal) + .then(a.1.cmp(&b.1)) + }); + v.truncate(n); + v + } + + #[cfg(test)] + pub(crate) fn tokenizer_name(&self) -> &str { + &self.tokenizer_name + } + + #[cfg(test)] + pub(crate) fn debug_index(&self) -> &Index { + &self.index + } +} + +/// Escape regex metacharacters, but leave the input as a verbatim literal. +fn regex_escape(input: &str) -> String { + let mut out = String::with_capacity(input.len() + 4); + for ch in input.chars() { + match ch { + '.' | '+' | '*' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '^' | '$' | '\\' => { + out.push('\\'); + out.push(ch); + } + _ => out.push(ch), + } + } + out +} + +/// Translate a glob-style wildcard ('*' = any, '?' = single char) into a +/// regex pattern, escaping all other regex metacharacters. +fn wildcard_to_regex(input: &str) -> String { + let mut out = String::with_capacity(input.len() + 4); + for ch in input.chars() { + match ch { + '*' => out.push_str(".*"), + '?' => out.push('.'), + '.' | '+' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '^' | '$' | '\\' => { + out.push('\\'); + out.push(ch); + } + _ => out.push(ch), + } + } + out +} + +/// Collector that reads the explicit `row_id` u64 fast field directly into a +/// `Vec`, opening the column once per segment in `for_segment`. Replaces +/// the DocSetCollector → HashSet → per-doc translate path for unscored queries. +struct RowIdCollector; + +struct RowIdSegmentCollector { + row_id: Column, + ids: Vec, +} + +impl SegmentCollector for RowIdSegmentCollector { + type Fruit = Vec; + + fn collect(&mut self, doc: DocId, _score: Score) { + self.ids.push(self.row_id.first(doc).unwrap_or(0)); + } + + fn harvest(self) -> Vec { + self.ids + } +} + +impl Collector for RowIdCollector { + type Fruit = Vec; + type Child = RowIdSegmentCollector; + + fn for_segment( + &self, _ord: SegmentOrdinal, segment: &SegmentReader, + ) -> tantivy::Result { + let row_id = segment.fast_fields().u64(PAIMON_ROW_ID_FIELD_NAME)?; + Ok(RowIdSegmentCollector { row_id, ids: Vec::new() }) + } + + fn requires_scoring(&self) -> bool { + false + } + + fn merge_fruits(&self, segs: Vec>) -> tantivy::Result> { + Ok(segs.into_iter().flatten().collect()) + } +} + +/// Collector that returns at most `limit` DocAddresses across all segments, +/// no scoring. Shared atomic counter caps the global total so per-shard +/// transfer stays bounded for plain `LIMIT N` queries (no ORDER BY). +struct LimitedDocSetCollector { + limit: usize, + counter: std::sync::Arc, +} + +impl LimitedDocSetCollector { + fn new(limit: usize) -> Self { + Self { limit, counter: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)) } + } +} + +struct LimitedDocSetSegmentCollector { + segment_ord: SegmentOrdinal, + docs: Vec, + counter: std::sync::Arc, + limit: u64, +} + +impl SegmentCollector for LimitedDocSetSegmentCollector { + type Fruit = Vec; + + fn collect(&mut self, doc: DocId, _score: Score) { + // Best-effort cap: if multiple segments are scanned concurrently the + // atomic ensures we never accept more than `limit` rows total. + let prev = self.counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if prev < self.limit { + self.docs.push(doc); + } + } + + fn harvest(self) -> Self::Fruit { + let segment_ord = self.segment_ord; + self.docs.into_iter().map(|d| DocAddress::new(segment_ord, d)).collect() + } +} + +impl Collector for LimitedDocSetCollector { + type Fruit = Vec; + type Child = LimitedDocSetSegmentCollector; + + fn for_segment( + &self, segment_ord: SegmentOrdinal, _segment: &SegmentReader, + ) -> tantivy::Result { + Ok(LimitedDocSetSegmentCollector { + segment_ord, + docs: Vec::new(), + counter: self.counter.clone(), + limit: self.limit as u64, + }) + } + + fn requires_scoring(&self) -> bool { false } + + fn merge_fruits( + &self, segment_fruits: Vec>, + ) -> tantivy::Result> { + let mut result: Vec = segment_fruits.into_iter().flatten().collect(); + result.truncate(self.limit); + Ok(result) + } +} + +/// Custom Collector that returns ALL matching (score, DocAddress) tuples, +/// without truncation. tantivy's stock `TopDocs::with_limit(N)` would force +/// us to either pick N upfront (wrong when pre_filter rejects high-score +/// docs) or pass `usize::MAX` (which still enforces a binary heap on every +/// push). Our collector is just a plain Vec append, then merge. +struct AllScoredCollector; + +struct AllScoredSegmentCollector { + segment_ord: SegmentOrdinal, + docs: Vec<(Score, DocId)>, +} + +impl SegmentCollector for AllScoredSegmentCollector { + type Fruit = Vec<(Score, DocAddress)>; + + fn collect(&mut self, doc: DocId, score: Score) { + self.docs.push((score, doc)); + } + + fn harvest(self) -> Self::Fruit { + let segment_ord = self.segment_ord; + self.docs + .into_iter() + .map(|(s, d)| (s, DocAddress::new(segment_ord, d))) + .collect() + } +} + +impl Collector for AllScoredCollector { + type Fruit = Vec<(Score, DocAddress)>; + type Child = AllScoredSegmentCollector; + + fn for_segment( + &self, + segment_ord: SegmentOrdinal, + _segment: &SegmentReader, + ) -> tantivy::Result { + Ok(AllScoredSegmentCollector { + segment_ord, + docs: Vec::new(), + }) + } + + fn requires_scoring(&self) -> bool { + true + } + + fn merge_fruits( + &self, + segment_fruits: Vec>, + ) -> tantivy::Result> { + Ok(segment_fruits.into_iter().flatten().collect()) + } +} + +// ============================ FFI surface ============================ + +/// Construct a streaming reader from a layout table + pread callbacks. +/// +/// The layout arrays (names / offsets / lengths) are produced by C++-side +/// `ArchiveLayout::Parse` after reading only the archive header bytes. Payload +/// bytes are fetched lazily through `callbacks.read_at` as tantivy reads. +/// +/// # Arguments +/// * `file_names` — array of `file_count` UTF-8 NUL-terminated C strings +/// * `file_offsets` / `file_lengths` — u64 arrays (archive-absolute offsets and lengths) +/// * `file_count` — number of entries in each of the three arrays +/// * `callbacks` — pread + release callbacks; `ctx` ownership transfers to Rust +/// * `mode_cstr` — tokenize mode ("mp"/"mix"/"full"/"query"; "hmm" → Unsupported) +/// * `with_position` — whether text field was indexed with positions +/// * `dict_dir_cstr` — paimon_jieba dictionary directory +/// * `out` — receives the reader handle on success +/// +/// Releases an FFI stream ctx on drop unless disarmed. Gives +/// `paimon_tantivy_reader_new_streaming` a single ownership rule: Rust owns ctx +/// from entry and releases it on any error before the directory takes over. +struct StreamCtxReleaseGuard { + release: extern "C" fn(*mut std::os::raw::c_void), + ctx: *mut std::os::raw::c_void, + armed: bool, +} + +impl Drop for StreamCtxReleaseGuard { + fn drop(&mut self) { + if self.armed { + (self.release)(self.ctx); + } + } +} + +/// # Safety +/// All pointer args must be valid for the duration of the call. Ownership of +/// `callbacks.ctx` transfers to Rust on entry: on any error this function +/// releases it, and on success it lives until the reader handle is freed. The +/// caller must NOT release ctx itself after calling this function. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_reader_new_streaming( + file_names: *const *const c_char, + file_offsets: *const u64, + file_lengths: *const u64, + file_count: usize, + callbacks: PaimonStreamCallbacks, + mode_cstr: *const c_char, + with_position: bool, + dict_dir_cstr: *const c_char, + out: *mut *mut PaimonTantivyReader, +) -> PaimonTantivyStatus { + // Single, uniform ownership rule: Rust owns ctx from entry. This guard + // releases it on every error path until ownership moves into the directory + // (which then releases on its own drop). Prevents the C++ caller and Rust + // from both releasing the same ctx on the post-directory failure path. + let mut ctx_guard = StreamCtxReleaseGuard { + release: callbacks.release, + ctx: callbacks.ctx, + armed: true, + }; + + if mode_cstr.is_null() || dict_dir_cstr.is_null() || out.is_null() { + set_last_error("paimon_tantivy_reader_new_streaming: null mandatory argument"); + return PaimonTantivyStatus::InvalidArgument; + } + if file_count > 0 + && (file_names.is_null() || file_offsets.is_null() || file_lengths.is_null()) + { + set_last_error("file_names/offsets/lengths must be non-null when file_count > 0"); + return PaimonTantivyStatus::InvalidArgument; + } + + let mode_str = match unsafe { CStr::from_ptr(mode_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("mode not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let dict_dir = match unsafe { CStr::from_ptr(dict_dir_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("dict_dir not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let mode = match TokenizeMode::parse(mode_str) { + Some(m) => m, + None => { + set_last_error(format!( + "unknown tokenize mode {mode_str:?}; expected mp/mix/full/query" + )); + return PaimonTantivyStatus::InvalidArgument; + } + }; + + // Copy the C string array into owned Rust entries so the directory doesn't + // depend on caller-supplied lifetime. + let mut entries: Vec<(String, u64, u64)> = Vec::with_capacity(file_count); + for i in 0..file_count { + let name_ptr = unsafe { *file_names.add(i) }; + if name_ptr.is_null() { + set_last_error(format!("file_names[{i}] is null")); + return PaimonTantivyStatus::InvalidArgument; + } + let name = match unsafe { CStr::from_ptr(name_ptr) }.to_str() { + Ok(s) => s.to_owned(), + Err(e) => { + set_last_error(format!("file_names[{i}] not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let offset = unsafe { *file_offsets.add(i) }; + let length = unsafe { *file_lengths.add(i) }; + entries.push((name, offset, length)); + } + + // Ownership of ctx transfers to the directory from here on (it releases on + // its own drop, whether it fails below or lives inside the returned reader). + ctx_guard.armed = false; + let directory = PaimonCallbackDirectory::new(entries, callbacks); + + match PaimonTantivyReader::new(directory, mode, with_position, Path::new(dict_dir)) { + Ok(r) => { + unsafe { *out = into_handle(r) }; + PaimonTantivyStatus::Ok + } + Err(e) => { + let unsupported = e.contains("'hmm' is not supported"); + let bad_format = e.contains("tantivy::Index::open") + || e.contains("missing 'text' field"); + set_last_error(e); + if unsupported { + PaimonTantivyStatus::Unsupported + } else if bad_format { + PaimonTantivyStatus::IndexFormatError + } else { + PaimonTantivyStatus::InternalError + } + } + } +} + +/// Run a query and emit results into `out`. +/// +/// Output bytes (little-endian): +/// `[u8 has_scores | u64 count | u64 row_ids[count] | optional f32 scores[count]]` +/// +/// `has_scores=1` iff `limit >= 0` (caller asked for scoring + limit). +/// +/// `limit < 0` ⇒ no limit, no scoring; sorted ascending by row_id. +/// `limit >= 0` ⇒ top-N by descending score (pre_filter applied first). +/// `pre_filter_bytes`: serialized croaring `Roaring64Map::write` (portable), +/// containing paimon **row_ids** (not tantivy doc_ids); null+0 = no filter. +/// +/// SAFETY: `reader` must be a live handle; `query` and `pre_filter_bytes` +/// may be null+0 or readable slices; `out` non-null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_reader_search( + reader: *mut PaimonTantivyReader, + search_type: i32, + query: *const c_char, + query_len: usize, + with_score: bool, + limit: i32, + pre_filter_bytes: *const c_char, + pre_filter_len: usize, + min_score: f32, + out: *mut PaimonTantivyBuffer, +) -> PaimonTantivyStatus { + if out.is_null() { + set_last_error("reader_search: out is null"); + return PaimonTantivyStatus::InvalidArgument; + } + let Some(r) = (unsafe { borrow_handle_mut::(reader) }) else { + set_last_error("reader_search: null reader handle"); + return PaimonTantivyStatus::InvalidArgument; + }; + let st = match SearchType::from_i32(search_type) { + Some(s) => s, + None => { + set_last_error(format!("unknown search_type {search_type}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + if query.is_null() && query_len != 0 { + set_last_error("query is null but len > 0"); + return PaimonTantivyStatus::InvalidArgument; + } + let query_str = if query_len == 0 { + "" + } else { + let slice = unsafe { std::slice::from_raw_parts(query as *const u8, query_len) }; + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + set_last_error(format!("query not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + } + }; + + let pre_filter: Option = if pre_filter_bytes.is_null() && pre_filter_len == 0 { + None + } else if pre_filter_bytes.is_null() { + set_last_error("pre_filter_bytes is null but len > 0"); + return PaimonTantivyStatus::InvalidArgument; + } else { + let slice = unsafe { + std::slice::from_raw_parts(pre_filter_bytes as *const u8, pre_filter_len) + }; + match Treemap::try_deserialize::(slice) { + Some(t) => Some(t), + None => { + set_last_error(format!( + "pre_filter not a valid Roaring64Map portable serialization ({} bytes)", + pre_filter_len + )); + return PaimonTantivyStatus::InvalidArgument; + } + } + }; + + let limit_opt: Option = if limit < 0 { None } else { Some(limit as usize) }; + let min_score_opt: Option = if min_score > 0.0 { Some(min_score) } else { None }; + + match r.search_with_limit_and_filter(st, query_str, with_score, limit_opt, pre_filter.as_ref(), min_score_opt) + { + Ok(rows) => { + // v0.2: has_scores is decoupled from limit — it equals with_score directly. + let has_scores = with_score; + let count = rows.len() as u64; + // 1 byte has_scores + 8 bytes count + 8 bytes per row_id + optional 4 bytes per score + let mut buf = Vec::with_capacity( + 1 + 8 + rows.len() * 8 + if has_scores { rows.len() * 4 } else { 0 }, + ); + buf.push(if has_scores { 1u8 } else { 0u8 }); + buf.extend_from_slice(&count.to_le_bytes()); + for (id, _) in &rows { + buf.extend_from_slice(&id.to_le_bytes()); // u64 row_id LE + } + if has_scores { + for (_, score) in &rows { + let s = score.unwrap_or(0.0); + buf.extend_from_slice(&s.to_le_bytes()); + } + } + unsafe { *out = PaimonTantivyBuffer::from_vec(buf) }; + PaimonTantivyStatus::Ok + } + Err(e) => { + let parse_err = e.contains("RegexQuery from") + || e.contains("phrase query") + || e.contains("produced no tokens"); + set_last_error(e); + if parse_err { + PaimonTantivyStatus::QueryParseError + } else { + PaimonTantivyStatus::InternalError + } + } + } +} + +/// Destroy a reader handle. Safe on null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_reader_free(reader: *mut PaimonTantivyReader) { + unsafe { free_handle(reader) }; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::callback_directory::test_support::{build_directory_from_archive, make_mock_callbacks}; + use crate::writer::PaimonTantivyWriter; + use std::path::PathBuf; + + // When PaimonTantivyReader::new fails *after* the callback directory (and + // thus ctx) is constructed, Rust must release ctx exactly once — never zero + // (leak) and never twice (the C++ caller no longer releases on failure, so a + // second release would be a double-free). file_count=0 yields an index with + // no meta.json, which fails to open after the directory is built. + #[test] + fn reader_new_streaming_releases_ctx_once_on_failure() { + let (cb, backend) = make_mock_callbacks(Vec::new()); + let mode = std::ffi::CString::new("mix").unwrap(); + let dict = std::ffi::CString::new("").unwrap(); + let mut out: *mut PaimonTantivyReader = std::ptr::null_mut(); + let st = unsafe { + paimon_tantivy_reader_new_streaming( + std::ptr::null(), + std::ptr::null(), + std::ptr::null(), + 0, + cb, + mode.as_ptr(), + true, + dict.as_ptr(), + &mut out, + ) + }; + assert!(!matches!(st, PaimonTantivyStatus::Ok)); + assert!(out.is_null()); + assert_eq!( + backend.release_count.load(std::sync::atomic::Ordering::SeqCst), + 1 + ); + } + + fn dict_dir() -> PathBuf { + std::env::var("PAIMON_JIEBA_DICT_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| PathBuf::from("/tmp/nonexistent-dict")) + } + + fn build(docs: &[&str]) -> Vec { + let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap(); + for (i, d) in docs.iter().enumerate() { + w.add(i as u64, d).unwrap(); + } + w.finish().unwrap().1 + } + + fn open(packed: &[u8]) -> PaimonTantivyReader { + // Simulate production flow: parse archive header → build layout → + // back PaimonCallbackDirectory with a mock pread that reads from the + // packed Vec. Once C++ `ArchiveLayout::Parse` is in place, prod + // uses the same PaimonCallbackDirectory path. + let (dir, _backend) = build_directory_from_archive(packed.to_vec()); + PaimonTantivyReader::new(dir, TokenizeMode::Mix, true, &dict_dir()).unwrap() + } + + #[test] + fn match_all_single_term() { + let bytes = build(&["hello world", "hello there", "world peace"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAll, "hello").unwrap(); + assert_eq!(ids, vec![0u64, 1]); + } + + #[test] + fn match_all_two_terms_intersection() { + let bytes = build(&["hello world", "hello there", "world peace"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAll, "hello world").unwrap(); + assert_eq!(ids, vec![0u64]); + } + + #[test] + fn match_any_two_terms_union() { + let bytes = build(&["hello world", "hello there", "world peace"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAny, "hello peace").unwrap(); + assert_eq!(ids, vec![0u64, 1, 2]); + } + + #[test] + fn phrase_only_consecutive() { + let bytes = build(&["hello world there", "world hello there"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::Phrase, "hello world").unwrap(); + assert_eq!(ids, vec![0u64]); + } + + #[test] + fn prefix_matches_indexed_terms() { + let bytes = build(&["unordered user-defined doc id"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::Prefix, "unorder").unwrap(); + assert_eq!(ids, vec![0u64]); + } + + #[test] + fn wildcard_with_star() { + let bytes = build(&["unordered", "ordered", "border"]); + let r = open(&bytes); + let ids = r.search_all(SearchType::Wildcard, "*order*").unwrap(); + assert_eq!(ids, vec![0u64, 1, 2]); + } + + #[test] + fn empty_query_for_match_returns_query_parse_error() { + let bytes = build(&["hello"]); + let r = open(&bytes); + let err = r.search_all(SearchType::MatchAll, "").unwrap_err(); + assert!(err.contains("no tokens"), "got: {err}"); + } + + #[test] + fn wildcard_helper_escapes_dots() { + assert_eq!(wildcard_to_regex("a*b"), "a.*b"); + assert_eq!(wildcard_to_regex("a?b"), "a.b"); + assert_eq!(wildcard_to_regex("a.b"), r"a\.b"); + assert_eq!(wildcard_to_regex("*a*"), ".*a.*"); + } + + // ----- limit + pre_filter + scoring (row_id-based) ----- + + #[test] + fn limit_returns_top_n_with_scores() { + let bytes = build(&[ + "doc", // 0: low score (1 occurrence) + "doc doc doc doc doc", // 1: high score (5 occurrences) + "doc doc", // 2: medium score + ]); + let r = open(&bytes); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "doc", true, Some(2), None, None) + .unwrap(); + assert_eq!(rows.len(), 2); + // doc 1 has highest TF, expect first + assert_eq!(rows[0].0, 1u64); + assert!(rows[0].1.is_some()); + assert!(rows[1].1.is_some()); + // Scores monotonically decreasing + assert!(rows[0].1.unwrap() >= rows[1].1.unwrap()); + } + + #[test] + fn no_limit_returns_all_unscored() { + let bytes = build(&["hello world", "world hello", "world peace"]); + let r = open(&bytes); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "world", false, None, None, None) + .unwrap(); + let ids: Vec = rows.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![0u64, 1, 2]); + assert!(rows.iter().all(|(_, s)| s.is_none())); + } + + #[test] + fn pre_filter_no_limit_intersects() { + let bytes = build(&["alpha beta", "alpha gamma", "beta gamma"]); + let r = open(&bytes); + // pre_filter = {0, 2}; query "alpha" matches {0, 1}; expect intersection {0} + let mut tm = Treemap::new(); + tm.add(0); + tm.add(2); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "alpha", false, None, Some(&tm), None) + .unwrap(); + let ids: Vec = rows.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![0u64]); + } + + #[test] + fn pre_filter_with_limit_filters_before_topn() { + // doc 0 has highest TF for "doc" but is NOT in pre_filter → must NOT + // be in result, even with limit=1. + let bytes = build(&[ + "doc doc doc doc doc", // 0: highest TF, but excluded + "doc doc", // 1: medium TF, included + "doc", // 2: low TF, excluded + ]); + let r = open(&bytes); + let mut tm = Treemap::new(); + tm.add(1); // only doc 1 passes pre_filter + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "doc", true, Some(10), Some(&tm), None) + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].0, 1u64); + } + + #[test] + fn unscored_limit_with_pre_filter_applies_filter_before_truncate() { + // Regression (review finding #1): with_score=false + limit=N + pre_filter + // must apply the filter to the FULL match set before truncating to N. + // All three docs match "doc" but only row_id 2 (the LAST one) passes the + // pre_filter; a truncate-before-filter impl (LimitedDocSetCollector that + // stops at N raw matches, then filters) would collect doc 0, filter it + // out, and wrongly return empty instead of {2}. + let bytes = build(&["doc", "doc", "doc"]); + let r = open(&bytes); + let mut tm = Treemap::new(); + tm.add(2); // only row_id 2 passes the pre_filter + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "doc", false, Some(1), Some(&tm), None) + .unwrap(); + let ids: Vec = rows.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![2u64], "pre_filter must be applied before LIMIT truncation"); + assert!(rows.iter().all(|(_, s)| s.is_none())); + } + + #[test] + fn empty_pre_filter_returns_empty() { + let bytes = build(&["alpha", "beta"]); + let r = open(&bytes); + let tm = Treemap::new(); // empty + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "alpha", false, None, Some(&tm), None) + .unwrap(); + assert!(rows.is_empty()); + } + + #[test] + fn limit_zero_returns_empty_without_running_query() { + let bytes = build(&["alpha", "beta"]); + let r = open(&bytes); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "alpha", true, Some(0), None, None) + .unwrap(); + assert!(rows.is_empty()); + } + + // ----- row_id is independent of doc_id ----- + + #[test] + fn pre_filter_uses_row_id_not_doc_id() { + // Build with non-contiguous row_ids so doc_id ≠ row_id. Then verify + // pre_filter operates on row_id values, not internal tantivy doc_ids. + let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap(); + w.add(100, "alpha").unwrap(); + w.add(200, "alpha").unwrap(); + w.add(300, "alpha").unwrap(); + let bytes = w.finish().unwrap().1; + let r = open(&bytes); + + // pre_filter = {200} as row_id (doc_id would be 1) + let mut tm = Treemap::new(); + tm.add(200); + let rows = r + .search_with_limit_and_filter(SearchType::MatchAll, "alpha", false, None, Some(&tm), None) + .unwrap(); + let ids: Vec = rows.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![200u64], "pre_filter must operate on row_id, not doc_id"); + } + + #[test] + fn search_returns_caller_supplied_row_ids() { + // Same setup: row_ids 100/200/300, verify search_all returns those values. + let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap(); + w.add(100, "doc").unwrap(); + w.add(200, "doc").unwrap(); + w.add(300, "doc").unwrap(); + let bytes = w.finish().unwrap().1; + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAll, "doc").unwrap(); + assert_eq!(ids, vec![100u64, 200, 300]); + } + + #[test] + fn tokenizer_name_reflects_paimon_jieba_schema_for_cpp_written_index() { + // cpp-written index: PaimonTantivyWriter binds the text field to + // `paimon_jieba`. Reader must pick that up from meta.json (not hardcode). + let bytes = build(&["hello world"]); + let r = open(&bytes); + assert_eq!(r.tokenizer_name(), PAIMON_TOKENIZER_NAME); + + // tokenize sanity: jieba mode="mix" picks `hello` + `world` from ASCII. + let q = r.tokenize_query("hello world"); + assert_eq!(q, vec!["hello".to_string(), "world".to_string()]); + } + + #[test] + fn tokenizer_name_reflects_default_schema_for_externally_written_index() { + // Simulate a paimon-java-shaped index: text field bound to the + // builtin `default` tokenizer (SimpleTokenizer + LowerCaser), not jieba. + // Build it directly via tantivy (bypassing PaimonTantivyWriter's jieba + // schema) so we can prove the reader auto-switches to the builtin. + use crate::callback_directory::test_support::build_mock_directory; + use tantivy::directory::Directory; + use tantivy::schema::{IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions}; + use tantivy::{doc, Index}; + + // Build a minimal index with field "text" bound to "default". + let mut sb = Schema::builder(); + let row_id_f = sb.add_u64_field( + "row_id", + NumericOptions::default().set_stored().set_indexed().set_fast(), + ); + let text_opts = TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer("default") // ← key: match paimon-java's TEXT default + .set_index_option(IndexRecordOption::WithFreqsAndPositions), + ); + let text_f = sb.add_text_field("text", text_opts); + let schema = sb.build(); + let tmp = tempfile::Builder::new() + .prefix("paimon-tantivy-dyn-tk-") + .tempdir() + .unwrap(); + let index = Index::create_in_dir(tmp.path(), schema).unwrap(); + let mut writer = index.writer(15_000_000).unwrap(); + writer + .add_document(doc!(row_id_f => 0u64, text_f => "Hello World")) + .unwrap(); + writer + .add_document(doc!(row_id_f => 1u64, text_f => "Apple.Banana")) + .unwrap(); + writer.commit().unwrap(); + writer.wait_merging_threads().unwrap(); + + // Pack the index dir into our archive format so the callback directory + // can serve it. Reuse writer.rs's format by streaming entries manually. + let mut data = Vec::new(); + let mut entries = Vec::<(String, u64, u64)>::new(); + let dir_iter = std::fs::read_dir(tmp.path()).unwrap(); + let mut files: Vec<_> = dir_iter + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().ok().map_or(false, |t| t.is_file())) + .filter(|e| !e.file_name().to_string_lossy().starts_with('.')) + .collect(); + files.sort_by_key(|e| e.file_name()); + data.extend_from_slice(&(files.len() as i32).to_be_bytes()); + for e in &files { + let name = e.file_name().to_string_lossy().into_owned(); + let bytes = std::fs::read(e.path()).unwrap(); + data.extend_from_slice(&(name.len() as i32).to_be_bytes()); + data.extend_from_slice(name.as_bytes()); + data.extend_from_slice(&(bytes.len() as i64).to_be_bytes()); + let off = data.len() as u64; + data.extend_from_slice(&bytes); + entries.push((name, off, bytes.len() as u64)); + } + + let (dir, _backend) = build_mock_directory(data, entries); + let r = PaimonTantivyReader::new(dir, TokenizeMode::Mix, true, &dict_dir()).unwrap(); + + // Reader must pick up `default` from schema, not hardcode `paimon_jieba`. + assert_eq!(r.tokenizer_name(), "default"); + + // Query tokenization now goes through tantivy's builtin default + // (SimpleTokenizer + LowerCaser): + // "Apple.Banana" → ["apple", "banana"] (dot is non-alnum, split) + // "Hello World" → ["hello", "world"] (space split + lowercase) + let q1 = r.tokenize_query("Hello World"); + assert_eq!(q1, vec!["hello".to_string(), "world".to_string()]); + let q2 = r.tokenize_query("Apple.Banana"); + assert_eq!(q2, vec!["apple".to_string(), "banana".to_string()]); + + // And the search path works across tokenizer: + let ids = r.search_all(SearchType::MatchAll, "hello").unwrap(); + assert_eq!(ids, vec![0u64]); + let ids = r.search_all(SearchType::MatchAll, "apple").unwrap(); + assert_eq!(ids, vec![1u64]); + } + + #[test] + fn reader_aggregates_row_ids_across_segments() { + // Multi-thread default writer + many docs => may produce multiple + // segments before force-merge. After finish(), force-merge collapses + // to one segment, but this test still validates the row_id retrieval + // path works for ≥1 segment. + let mut w = PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir(), "paimon_jieba").unwrap(); + for i in 0..200u64 { + w.add(i * 7, &format!("docmark_{i} apple")).unwrap(); + } + let bytes = w.finish().unwrap().1; + let r = open(&bytes); + let ids = r.search_all(SearchType::MatchAll, "apple").unwrap(); + assert_eq!(ids.len(), 200); + for i in 0..200u64 { + assert!(ids.contains(&(i * 7)), "missing row_id={}", i * 7); + } + } +} diff --git a/third_party/tantivy_ffi/src/tokenizer.rs b/third_party/tantivy_ffi/src/tokenizer.rs new file mode 100644 index 00000000..e3e69f24 --- /dev/null +++ b/third_party/tantivy_ffi/src/tokenizer.rs @@ -0,0 +1,447 @@ +//! PaimonJiebaTokenizer: tantivy Tokenizer impl wrapping jieba-rs. +//! +//! Contract: +//! - Behavior-equivalent with `JiebaAnalyzer` in src/paimon/global_index/lucene/ +//! - 5 modes: mp / hmm / mix / full / query +//! - `hmm` is Unsupported (jieba-rs has no standalone HMM entry point) +//! - `mp` accepts cut(hmm=false) but does not replicate cppjieba's +//! max_word_len truncation +//! - Normalize: skip pure whitespace, skip stop_words, lowercase ASCII-only tokens +//! - Token offsets: byte offsets into the original UTF-8 string +//! - `with_position=false`: all tokens emitted at `position=0` (disables PhraseQuery) +//! - Custom dict dir: loads `jieba.dict.utf8` (+optional `user.dict.utf8`) from +//! `$PAIMON_JIEBA_DICT_DIR`; stop_words.utf8 loaded if present + +use std::collections::HashSet; +use std::ffi::{c_char, CStr}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::sync::Arc; + +use jieba_rs::Jieba; +use tantivy::tokenizer::{Token, TokenStream, Tokenizer}; + +use crate::buffer::PaimonTantivyBuffer; +use crate::error::{set_last_error, PaimonTantivyStatus}; +use crate::handle::{borrow_handle, free_handle, into_handle}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TokenizeMode { + Mp, + Hmm, + Mix, + Full, + Query, +} + +impl TokenizeMode { + pub(crate) fn parse(s: &str) -> Option { + match s { + "mp" => Some(Self::Mp), + "hmm" => Some(Self::Hmm), + "mix" => Some(Self::Mix), + "full" => Some(Self::Full), + "query" => Some(Self::Query), + _ => None, + } + } +} + +#[derive(Clone)] +pub struct PaimonJiebaTokenizer { + jieba: Arc, + mode: TokenizeMode, + with_position: bool, + stop_words: Arc>, +} + +impl PaimonJiebaTokenizer { + pub fn new( + dict_dir: &Path, + mode: TokenizeMode, + with_position: bool, + ) -> Result { + if mode == TokenizeMode::Hmm { + return Err( + "tokenize mode 'hmm' is not supported (jieba-rs does not expose standalone HMM)" + .into(), + ); + } + let jieba = load_jieba(dict_dir)?; + let stop_words = load_stop_words(dict_dir); + Ok(Self { + jieba: Arc::new(jieba), + mode, + with_position, + stop_words: Arc::new(stop_words), + }) + } + + /// Directly tokenize, returning a Vec of (offset_start, offset_end, text) tuples. + /// Used both by the tantivy Tokenizer impl and the standalone `tokenize` FFI. + pub fn tokenize_raw(&self, text: &str) -> Vec<(usize, usize, String)> { + // Use jieba-rs's cut variants which return Vec<&'a str>; compute byte offsets + // via pointer arithmetic (each &str is a slice of the original). + let cuts: Vec<&str> = match self.mode { + TokenizeMode::Mp => self.jieba.cut(text, false), + TokenizeMode::Hmm => Vec::new(), // unreachable (caught in new()) + TokenizeMode::Mix => self.jieba.cut(text, true), + TokenizeMode::Full => self.jieba.cut_all(text), + TokenizeMode::Query => self.jieba.cut_for_search(text, true), + }; + + let text_start = text.as_ptr() as usize; + let mut out = Vec::with_capacity(cuts.len()); + for piece in cuts { + // skip pure whitespace + if piece.chars().all(char::is_whitespace) { + continue; + } + // skip stop words (compare original case) + if self.stop_words.contains(piece) { + continue; + } + // offset calc + let start = piece.as_ptr() as usize - text_start; + let end = start + piece.len(); + // lowercase only if pure ASCII alphanumeric (match cppjieba Normalize behavior) + let token_text = if is_ascii_alnum(piece) { + piece.to_ascii_lowercase() + } else { + piece.to_string() + }; + out.push((start, end, token_text)); + } + out + } +} + +fn is_ascii_alnum(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric()) +} + +fn load_jieba(dict_dir: &Path) -> Result { + let main_dict = dict_dir.join("jieba.dict.utf8"); + let mut jieba = if main_dict.exists() { + let file = File::open(&main_dict) + .map_err(|e| format!("open {}: {e}", main_dict.display()))?; + let mut rdr = BufReader::new(file); + Jieba::with_dict(&mut rdr).map_err(|e| format!("load jieba dict: {e:?}"))? + } else { + // No custom dict; use jieba-rs builtin + Jieba::new() + }; + // Optional user dict. cppjieba's user.dict.utf8 is lenient: lines are + // `word [freq] [tag]` where freq can be omitted (e.g. ` nz`), but + // jieba-rs's load_dict strictly requires `word freq [tag]` and fails if + // freq is not an integer. We parse line-by-line with `add_word` to stay + // compatible. + let user_dict = dict_dir.join("user.dict.utf8"); + if user_dict.exists() { + let file = File::open(&user_dict) + .map_err(|e| format!("open {}: {e}", user_dict.display()))?; + for (n, line_res) in BufReader::new(file).lines().enumerate() { + let line = match line_res { + Ok(l) => l, + Err(_) => continue, // skip unreadable lines + }; + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + let mut it = trimmed.split_whitespace(); + let word = it.next().unwrap(); // non-empty guaranteed + let next = it.next(); + let freq = next.and_then(|s| s.parse::().ok()); + let tag = match (freq, next) { + (Some(_), _) => it.next(), // [tag] + (None, tok) => tok, // (no freq) + }; + // `add_word` returns the assigned frequency; ignore it. For lines + // with bogus content we silently keep going, matching cppjieba's + // tolerant behavior. + let _ = jieba.add_word(word, freq, tag); + let _ = n; // keep for potential debug + } + } + Ok(jieba) +} + +fn load_stop_words(dict_dir: &Path) -> HashSet { + let path = dict_dir.join("stop_words.utf8"); + let mut out = HashSet::new(); + if let Ok(f) = File::open(&path) { + for line in BufReader::new(f).lines().map_while(Result::ok) { + let w = line.trim(); + if !w.is_empty() { + out.insert(w.to_owned()); + } + } + } + out +} + +// ----------------- tantivy Tokenizer integration ----------------- + +pub struct PaimonJiebaTokenStream { + tokens: Vec, + index: usize, +} + +impl TokenStream for PaimonJiebaTokenStream { + fn advance(&mut self) -> bool { + self.index += 1; + self.index <= self.tokens.len() + } + + fn token(&self) -> &Token { + &self.tokens[self.index - 1] + } + + fn token_mut(&mut self) -> &mut Token { + &mut self.tokens[self.index - 1] + } +} + +impl Tokenizer for PaimonJiebaTokenizer { + type TokenStream<'a> = PaimonJiebaTokenStream; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + let raw = self.tokenize_raw(text); + let tokens: Vec = raw + .into_iter() + .enumerate() + .map(|(i, (s, e, t))| Token { + offset_from: s, + offset_to: e, + position: if self.with_position { i } else { 0 }, + text: t, + position_length: 1, + }) + .collect(); + PaimonJiebaTokenStream { tokens, index: 0 } + } +} + +// ----------------- FFI surface ----------------- + +/// Create a tokenizer handle. Returns OK and writes *out on success; returns +/// status and sets last_error on failure. +/// +/// SAFETY: `mode_cstr` and `dict_dir_cstr` must be NUL-terminated UTF-8; +/// `out` must be a valid non-null pointer. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_tokenizer_new( + mode_cstr: *const c_char, + with_position: bool, + dict_dir_cstr: *const c_char, + out: *mut *mut PaimonJiebaTokenizer, +) -> PaimonTantivyStatus { + if mode_cstr.is_null() || dict_dir_cstr.is_null() || out.is_null() { + set_last_error("paimon_tantivy_tokenizer_new: null argument"); + return PaimonTantivyStatus::InvalidArgument; + } + let mode_s = match unsafe { CStr::from_ptr(mode_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("mode not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let dict_s = match unsafe { CStr::from_ptr(dict_dir_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("dict_dir not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let mode = match TokenizeMode::parse(mode_s) { + Some(m) => m, + None => { + set_last_error(format!( + "unknown tokenize mode {mode_s:?}; expected one of mp/hmm/mix/full/query" + )); + return PaimonTantivyStatus::InvalidArgument; + } + }; + match PaimonJiebaTokenizer::new(Path::new(dict_s), mode, with_position) { + Ok(t) => { + unsafe { *out = into_handle(t) }; + PaimonTantivyStatus::Ok + } + Err(e) => { + let is_hmm_unsupported = e.contains("'hmm' is not supported"); + set_last_error(e); + if is_hmm_unsupported { + PaimonTantivyStatus::Unsupported + } else { + PaimonTantivyStatus::TokenizerError + } + } + } +} + +/// Free a tokenizer handle. Safe on null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_tokenizer_free(tok: *mut PaimonJiebaTokenizer) { + unsafe { free_handle(tok) }; +} + +/// Tokenize a string and return a newline-delimited list of tokens as bytes. +/// Used for golden-sample tests (easy to diff from C++). +/// +/// Output format: +/// `\t\t\t\n` for each token. +/// +/// SAFETY: `tok` must be a valid handle; `text` must point to `text_len` UTF-8 bytes; +/// `out` must be non-null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_tokenizer_tokenize( + tok: *const PaimonJiebaTokenizer, + text: *const c_char, + text_len: usize, + out: *mut PaimonTantivyBuffer, +) -> PaimonTantivyStatus { + if out.is_null() { + set_last_error("paimon_tantivy_tokenizer_tokenize: out is null"); + return PaimonTantivyStatus::InvalidArgument; + } + let Some(tokenizer) = (unsafe { borrow_handle::(tok) }) else { + set_last_error("paimon_tantivy_tokenizer_tokenize: null tokenizer handle"); + return PaimonTantivyStatus::InvalidArgument; + }; + if text.is_null() && text_len != 0 { + set_last_error("text is null but len > 0"); + return PaimonTantivyStatus::InvalidArgument; + } + let text_str = if text_len == 0 { + "" + } else { + let slice = unsafe { std::slice::from_raw_parts(text as *const u8, text_len) }; + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + set_last_error(format!("text not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + } + }; + let raw = tokenizer.tokenize_raw(text_str); + let mut buf = String::new(); + for (i, (s, e, t)) in raw.iter().enumerate() { + let pos = if tokenizer.with_position { i } else { 0 }; + buf.push_str(&format!("{s}\t{e}\t{pos}\t{t}\n")); + } + let bytes = buf.into_bytes(); + unsafe { *out = PaimonTantivyBuffer::from_vec(bytes) }; + PaimonTantivyStatus::Ok +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + + fn dict_dir_from_env() -> std::path::PathBuf { + std::env::var("PAIMON_JIEBA_DICT_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|_| std::path::PathBuf::from("/tmp/nonexistent-dict")) + } + + #[test] + fn mode_parse() { + for (s, m) in [ + ("mp", TokenizeMode::Mp), + ("hmm", TokenizeMode::Hmm), + ("mix", TokenizeMode::Mix), + ("full", TokenizeMode::Full), + ("query", TokenizeMode::Query), + ] { + assert_eq!(TokenizeMode::parse(s), Some(m)); + } + assert!(TokenizeMode::parse("bogus").is_none()); + } + + #[test] + fn hmm_mode_returns_unsupported() { + let tok = PaimonJiebaTokenizer::new( + &dict_dir_from_env(), + TokenizeMode::Hmm, + true, + ); + match tok { + Err(e) => assert!(e.contains("'hmm' is not supported"), "got: {e}"), + Ok(_) => panic!("expected Err"), + } + } + + #[test] + fn tokenize_mix_default_dict_smoke() { + // If no custom dict dir, jieba-rs builtin is used. + let t = PaimonJiebaTokenizer::new(Path::new("/tmp/nonexistent-dict"), TokenizeMode::Mix, true) + .unwrap(); + let raw = t.tokenize_raw("他来到了网易杭研大厦"); + let texts: Vec<&str> = raw.iter().map(|(_, _, s)| s.as_str()).collect(); + assert!(texts.contains(&"网易")); + assert!(texts.contains(&"大厦")); + } + + #[test] + fn ascii_alnum_is_lowercased() { + let t = PaimonJiebaTokenizer::new(Path::new("/tmp/nx"), TokenizeMode::Mix, true).unwrap(); + let raw = t.tokenize_raw("Hello World 中国"); + let texts: Vec<&str> = raw.iter().map(|(_, _, s)| s.as_str()).collect(); + assert!(texts.contains(&"hello")); + assert!(texts.contains(&"world")); + assert!(texts.contains(&"中国")); + } + + #[test] + fn with_position_false_emits_zero_position() { + let t = PaimonJiebaTokenizer::new(Path::new("/tmp/nx"), TokenizeMode::Mix, false).unwrap(); + let raw = t.tokenize_raw("中国人"); + // Can't check position on raw tuples; check via tantivy Token stream: + let mut t2 = t.clone(); + let mut stream = ::token_stream(&mut t2, "中国人"); + let mut positions = Vec::new(); + while stream.advance() { + positions.push(stream.token().position); + } + assert!(!raw.is_empty()); + assert!(positions.iter().all(|&p| p == 0)); + } + + #[test] + fn ffi_roundtrip() { + let dict = dict_dir_from_env(); + let dict_str = dict.to_str().unwrap(); + let mode = CString::new("mix").unwrap(); + let dict_c = CString::new(dict_str).unwrap(); + let mut handle: *mut PaimonJiebaTokenizer = std::ptr::null_mut(); + unsafe { + let st = paimon_tantivy_tokenizer_new( + mode.as_ptr(), + true, + dict_c.as_ptr(), + &mut handle, + ); + assert_eq!(st, PaimonTantivyStatus::Ok); + assert!(!handle.is_null()); + + let input = "Hello 中国"; + let input_c = CString::new(input).unwrap(); + let mut buf = PaimonTantivyBuffer::empty(); + let st2 = paimon_tantivy_tokenizer_tokenize( + handle, + input_c.as_ptr(), + input.len(), + &mut buf, + ); + assert_eq!(st2, PaimonTantivyStatus::Ok); + assert!(buf.len > 0); + crate::buffer::paimon_tantivy_buffer_free(&mut buf); + paimon_tantivy_tokenizer_free(handle); + } + } +} diff --git a/third_party/tantivy_ffi/src/writer.rs b/third_party/tantivy_ffi/src/writer.rs new file mode 100644 index 00000000..3ea5ab4b --- /dev/null +++ b/third_party/tantivy_ffi/src/writer.rs @@ -0,0 +1,773 @@ +//! PaimonTantivyWriter: Writer for tantivy-fulltext global index. +//! +//! Contract (see docs/dev/tantivy_java_compat_plan.md §2.5 + §5.1 J2): +//! - `writer_new(field_name, mode, with_position, dict_dir, out)` — create on a +//! private tmp dir backed by MmapDirectory + PaimonJiebaTokenizer. +//! `field_name` is **ignored** by the Rust schema (kept for FFI ABI +//! compatibility); schema field names are fixed (`row_id`, `text`) to match +//! paimon-java `paimon-tantivy-jni/rust/src/lib.rs:55-66`. +//! - `writer_add(writer, row_id, text, len)` — add a single document with the +//! caller-supplied `row_id` (u64) and a TEXT field +//! - `writer_finish(writer, out_row_count, out_buf)` — commit + force-merge to +//! single segment + pack all on-disk index files into a Rust-allocated buffer +//! - `writer_free(writer)` — destroy (RAII removes tmp dir) +//! +//! Packing format (big-endian, **cross-readable with paimon-java archive**; +//! see `paimon-tantivy-index/README.md` §Archive File Format): +//! `[i32 BE file_count | +//! (i32 BE name_len | name_bytes | i64 BE file_len | file_bytes)*]` + +use std::ffi::{c_char, c_void, CStr}; +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use tantivy::schema::{ + Field, IndexRecordOption, NumericOptions, Schema, TextFieldIndexing, TextOptions, +}; +use tantivy::{doc, Index, IndexWriter, TantivyDocument}; +use tempfile::TempDir; + +use crate::error::{set_last_error, PaimonTantivyStatus}; +use crate::handle::{borrow_handle_mut, free_handle, into_handle}; +use crate::tokenizer::{PaimonJiebaTokenizer, TokenizeMode}; + +/// Schema field names. Fixed to match paimon-java's tantivy schema so that +/// indexes are cross-readable. Both fields are required. +pub const PAIMON_ROW_ID_FIELD_NAME: &str = "row_id"; +pub const PAIMON_TEXT_FIELD_NAME: &str = "text"; + +/// Name registered with the tantivy `TokenizerManager`. Reader must register +/// the same name to make stored term dictionaries readable. +pub const PAIMON_TOKENIZER_NAME: &str = "paimon_jieba"; + +/// Heap budget for the in-process IndexWriter (50 MB; tantivy minimum is ~3 MB). +/// Default multi-threaded writer (`Index::writer(heap)`) splits this budget +/// across `min(num_cpus, MAX_NUM_THREAD=8)` worker threads. +const WRITER_HEAP_SIZE: usize = 50_000_000; + +pub struct PaimonTantivyWriter { + /// Owned tmp dir; cleaned up when this struct drops. + tmpdir: TempDir, + /// `row_id` u64 field (stored + indexed + fast). Reader retrieves the + /// caller-supplied row_id via `fast_fields().u64("row_id").first(doc_id)`. + row_id_field: Field, + /// `text` TEXT field tokenized via the registered jieba tokenizer. + text_field: Field, + /// tantivy index instance, file-backed in `tmpdir`. + index: Index, + /// Active writer; consumed by `wait_merging_threads()` in `finish`. + writer: Option, + /// Documents added since construction. + row_count: i64, +} + +impl PaimonTantivyWriter { + pub fn new( + field_name: &str, + mode: TokenizeMode, + with_position: bool, + dict_dir: &Path, + tokenizer_name: &str, + ) -> Result { + if field_name.is_empty() { + return Err("field_name must be non-empty".into()); + } + // Schema is fixed to match paimon-java: row_id (u64 + // stored+indexed+fast) + text (TEXT). The caller-supplied `field_name` + // parameter is currently ignored by the Rust schema (kept for FFI + // backward-compatibility); the C++ side still uses it to extract the + // right column from arrow batches. + let _ = field_name; // intentionally unused on the Rust side + let mut schema_builder = Schema::builder(); + let row_id_field = schema_builder.add_u64_field( + PAIMON_ROW_ID_FIELD_NAME, + NumericOptions::default() + .set_stored() + .set_indexed() + .set_fast(), + ); + let index_option = if with_position { + IndexRecordOption::WithFreqsAndPositions + } else { + IndexRecordOption::Basic + }; + // Empty input falls back to tantivy's built-in "default" (SimpleTokenizer), + // matching the cpp-side default in `tantivy_defs.h::kDefaultTantivyWriteTokenizer`. + // Cross-read with paimon-java works out of the box; CJK callers must + // pass "paimon_jieba" explicitly. + let effective_tokenizer = if tokenizer_name.is_empty() { + "default" + } else { + tokenizer_name + }; + let text_options = TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer(effective_tokenizer) + .set_index_option(index_option), + ); + let text_field = schema_builder.add_text_field(PAIMON_TEXT_FIELD_NAME, text_options); + let schema = schema_builder.build(); + + let tmpdir = tempfile::Builder::new() + .prefix("paimon-tantivy-") + .tempdir() + .map_err(|e| format!("create tmp dir: {e}"))?; + + let index = Index::create_in_dir(tmpdir.path(), schema) + .map_err(|e| format!("create tantivy index: {e}"))?; + // When caller picks "paimon_jieba" we construct + register the jieba + // tokenizer. For any tantivy built-in name ("default", "whitespace", + // "raw", "en_stem", ...) tantivy's TokenizerManager already has it + // registered via `TokenizerManager::default()`; no-op here. This lets + // paimon-cpp emit archives cross-readable by paimon-java's default + // TEXT tokenizer path. + if effective_tokenizer == PAIMON_TOKENIZER_NAME { + let tokenizer = PaimonJiebaTokenizer::new(dict_dir, mode, with_position) + .map_err(|e| format!("create tokenizer: {e}"))?; + index + .tokenizers() + .register(PAIMON_TOKENIZER_NAME, tokenizer); + } + + // Default multi-threaded writer (schema stores row_id explicitly so + // we no longer need single-threaded ordering invariants). tantivy will + // use min(num_cpus, MAX_NUM_THREAD=8) workers, splitting heap budget. + let writer: IndexWriter = index + .writer(WRITER_HEAP_SIZE) + .map_err(|e| format!("create index writer: {e}"))?; + + Ok(Self { + tmpdir, + row_id_field, + text_field, + index, + writer: Some(writer), + row_count: 0, + }) + } + + pub fn add(&mut self, row_id: u64, text: &str) -> Result<(), String> { + let writer = self + .writer + .as_mut() + .ok_or_else(|| "writer already finished".to_string())?; + let document: TantivyDocument = doc!( + self.row_id_field => row_id, + self.text_field => text, + ); + writer + .add_document(document) + .map_err(|e| format!("add document: {e}"))?; + self.row_count += 1; + Ok(()) + } + + /// Commit + force-merge + GC on-disk index. Extracted from `finish_*` + /// so both streaming and test paths can share it. + fn commit_and_merge(&mut self) -> Result<(), String> { + let mut writer = self + .writer + .take() + .ok_or_else(|| "writer already finished".to_string())?; + writer.commit().map_err(|e| format!("commit: {e}"))?; + + let segment_metas = self + .index + .searchable_segment_metas() + .map_err(|e| format!("list segments: {e}"))?; + if segment_metas.len() > 1 { + let segment_ids: Vec<_> = segment_metas.iter().map(|m| m.id()).collect(); + writer + .merge(&segment_ids) + .wait() + .map_err(|e| format!("merge: {e}"))?; + } + writer + .garbage_collect_files() + .wait() + .map_err(|e| format!("garbage_collect_files: {e}"))?; + writer + .wait_merging_threads() + .map_err(|e| format!("wait_merging_threads: {e}"))?; + Ok(()) + } + + /// Streaming finish (production path): commit + force-merge + push archive + /// bytes through the FFI callback in fixed-size chunks (see + /// WRITER_STREAM_BUFFER_SIZE). Peak RAM independent of archive size — one + /// heap buffer + a few KB metadata. + pub fn finish_streaming( + &mut self, + cb: &PaimonWriteCallbacks, + ) -> Result { + self.commit_and_merge()?; + let ctx = cb.ctx; + let write_fn = cb.write; + pack_index_dir_stream(self.tmpdir.path(), |bytes| { + // Calling extern "C" fn pointer is safe; C++ side owns ctx validity. + let rc = (write_fn)(ctx, bytes.as_ptr(), bytes.len()); + if rc != 0 { + return Err(format!("write callback rc={rc} len={}", bytes.len())); + } + Ok(()) + })?; + Ok(self.row_count) + } + + /// Test-only convenience: collect streaming output into a `Vec`. + /// Rust unit tests / integration tests use this; production path is + /// `finish_streaming`. + #[cfg(test)] + pub(crate) fn finish(&mut self) -> Result<(i64, Vec), String> { + self.commit_and_merge()?; + let mut out: Vec = Vec::new(); + pack_index_dir_stream(self.tmpdir.path(), |bytes| { + out.extend_from_slice(bytes); + Ok(()) + })?; + Ok((self.row_count, out)) + } + + #[cfg(test)] + pub(crate) fn tmpdir_path(&self) -> &Path { + self.tmpdir.path() + } +} + +// ========================================================================= +// Streaming pack +// ========================================================================= + +/// Streaming pack buffer size. 1MB matches the buffer size data-lake storage +/// backends (e.g. Pangu) use for good throughput, still far below any archive +/// size we care about. Heap-allocated (see pack_index_dir_stream), so the size +/// does not affect stack usage. +const WRITER_STREAM_BUFFER_SIZE: usize = 1024 * 1024; + +/// Callback table passed from C++ for streaming writer output. +/// +/// `ctx` is an opaque pointer to C++'s `WriteCtx` (holding a `paimon::OutputStream`). +/// `write` is called in-order by Rust (not concurrently) to push bytes. +#[repr(C)] +pub struct PaimonWriteCallbacks { + pub ctx: *mut c_void, + /// Returns 0 on success, non-zero to signal C++ side error (Rust aborts pack). + pub write: extern "C" fn(ctx: *mut c_void, data: *const u8, len: usize) -> i32, +} + +/// Walk tempdir + pack into the Java-compatible archive format, pushing each +/// chunk through `write_fn`. Peak RAM = one WRITER_STREAM_BUFFER_SIZE heap +/// buffer + a few KB of entry metadata (name + PathBuf + u64 length). Mirrors +/// Java `TantivyFullTextGlobalIndexWriter.packIndex` but with a bigger buffer. +/// +/// Archive format (BE, no version): `[i32 file_count | (i32 name_len, name, +/// i64 file_len, file_bytes)*]`. Files sorted alphabetically for deterministic +/// output; `.`-prefixed (lock) files and non-regular entries skipped. +fn pack_index_dir_stream(dir: &Path, mut write_fn: F) -> Result<(), String> +where + F: FnMut(&[u8]) -> Result<(), String>, +{ + let entries = collect_dir_entries(dir)?; + + // Header: BE i32 file_count + write_fn(&(entries.len() as i32).to_be_bytes())?; + + let mut buf = vec![0u8; WRITER_STREAM_BUFFER_SIZE]; + for (name, path, file_len) in &entries { + // Per-entry header: name_len, name, data_len + write_fn(&(name.len() as i32).to_be_bytes())?; + write_fn(name.as_bytes())?; + write_fn(&(*file_len as i64).to_be_bytes())?; + + // Payload: fixed-size buffer loop + let mut f = File::open(path) + .map_err(|e| format!("open {}: {e}", path.display()))?; + let mut pushed: u64 = 0; + loop { + let n = f + .read(&mut buf) + .map_err(|e| format!("read {}: {e}", path.display()))?; + if n == 0 { + break; + } + write_fn(&buf[..n])?; + pushed += n as u64; + } + if pushed != *file_len { + return Err(format!( + "file {} changed size during packing: header said {}, streamed {}", + name, file_len, pushed + )); + } + } + Ok(()) +} + +/// Enumerate the tempdir: sorted (name, path, len) for regular non-`.lock` files. +fn collect_dir_entries(dir: &Path) -> Result, String> { + let mut entries: Vec<(String, PathBuf, u64)> = Vec::new(); + let read_dir = + std::fs::read_dir(dir).map_err(|e| format!("read tmp dir {}: {e}", dir.display()))?; + for entry_res in read_dir { + let entry = entry_res.map_err(|e| format!("read entry: {e}"))?; + let name = match entry.file_name().into_string() { + Ok(n) => n, + Err(_) => continue, + }; + if name.starts_with('.') { + continue; + } + let ft = entry + .file_type() + .map_err(|e| format!("file_type for {}: {e}", entry.path().display()))?; + if !ft.is_file() { + continue; + } + let len = entry + .metadata() + .map_err(|e| format!("metadata for {}: {e}", entry.path().display()))? + .len(); + entries.push((name, entry.path(), len)); + } + entries.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(entries) +} + +// ============================ FFI surface ============================ + +/// Create a writer handle on a private tmp dir. +/// +/// SAFETY: all C-string args must be NUL-terminated UTF-8; `out` non-null. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_writer_new( + field_name_cstr: *const c_char, + mode_cstr: *const c_char, + with_position: bool, + dict_dir_cstr: *const c_char, + tokenizer_cstr: *const c_char, + out: *mut *mut PaimonTantivyWriter, +) -> PaimonTantivyStatus { + if field_name_cstr.is_null() + || mode_cstr.is_null() + || dict_dir_cstr.is_null() + || tokenizer_cstr.is_null() + || out.is_null() + { + set_last_error("paimon_tantivy_writer_new: null argument"); + return PaimonTantivyStatus::InvalidArgument; + } + let field_name = match unsafe { CStr::from_ptr(field_name_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("field_name not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let mode_str = match unsafe { CStr::from_ptr(mode_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("mode not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let dict_dir = match unsafe { CStr::from_ptr(dict_dir_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("dict_dir not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let tokenizer_name = match unsafe { CStr::from_ptr(tokenizer_cstr) }.to_str() { + Ok(s) => s, + Err(e) => { + set_last_error(format!("tokenizer not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + }; + let mode = match TokenizeMode::parse(mode_str) { + Some(m) => m, + None => { + set_last_error(format!( + "unknown tokenize mode {mode_str:?}; expected one of mp/hmm/mix/full/query" + )); + return PaimonTantivyStatus::InvalidArgument; + } + }; + match PaimonTantivyWriter::new( + field_name, + mode, + with_position, + Path::new(dict_dir), + tokenizer_name, + ) { + Ok(w) => { + unsafe { *out = into_handle(w) }; + PaimonTantivyStatus::Ok + } + Err(e) => { + // hmm-mode rejection bubbles through tokenizer construction. + let unsupported = e.contains("'hmm' is not supported"); + set_last_error(e); + if unsupported { + PaimonTantivyStatus::Unsupported + } else { + PaimonTantivyStatus::InternalError + } + } + } +} + +/// Add a single document. `text` need not be NUL-terminated; treat as a slice +/// of `text_len` UTF-8 bytes. Empty text (len=0) inserts an empty-text doc. +/// `row_id` is the caller-supplied paimon row id (u64), stored in a fast field +/// for retrieval by the reader. +/// +/// SAFETY: `writer` must be a live handle from `writer_new`. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_writer_add( + writer: *mut PaimonTantivyWriter, + row_id: u64, + text: *const c_char, + text_len: usize, +) -> PaimonTantivyStatus { + let Some(w) = (unsafe { borrow_handle_mut::(writer) }) else { + set_last_error("paimon_tantivy_writer_add: null writer handle"); + return PaimonTantivyStatus::InvalidArgument; + }; + if text.is_null() && text_len != 0 { + set_last_error("text is null but len > 0"); + return PaimonTantivyStatus::InvalidArgument; + } + let text_str = if text_len == 0 { + "" + } else { + let slice = unsafe { std::slice::from_raw_parts(text as *const u8, text_len) }; + match std::str::from_utf8(slice) { + Ok(s) => s, + Err(e) => { + set_last_error(format!("text not utf-8: {e}")); + return PaimonTantivyStatus::InvalidArgument; + } + } + }; + match w.add(row_id, text_str) { + Ok(()) => PaimonTantivyStatus::Ok, + Err(e) => { + set_last_error(e); + PaimonTantivyStatus::InternalError + } + } +} + +/// Commit + force-merge + stream archive bytes through `callbacks.write` in +/// fixed-size chunks (see WRITER_STREAM_BUFFER_SIZE). May only be called once +/// per writer; subsequent calls return InvalidArgument with +/// last_error="writer already finished". Peak Rust RAM ≈ one buffer + entry +/// metadata (independent of archive size). +/// +/// The callback is invoked **serially** (not concurrently) within this call; +/// C++ side can write directly to paimon OutputStream without locking. +/// +/// SAFETY: `writer` must be a live handle; `out_row_count` non-null. +/// `callbacks.write` / `callbacks.ctx` must remain valid for the duration of +/// the call (callback is consumed in-place, not retained). +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_writer_finish_streaming( + writer: *mut PaimonTantivyWriter, + callbacks: PaimonWriteCallbacks, + out_row_count: *mut i64, +) -> PaimonTantivyStatus { + if out_row_count.is_null() { + set_last_error("paimon_tantivy_writer_finish_streaming: null out_row_count"); + return PaimonTantivyStatus::InvalidArgument; + } + let Some(w) = (unsafe { borrow_handle_mut::(writer) }) else { + set_last_error("paimon_tantivy_writer_finish_streaming: null writer handle"); + return PaimonTantivyStatus::InvalidArgument; + }; + match w.finish_streaming(&callbacks) { + Ok(rows) => { + unsafe { *out_row_count = rows }; + PaimonTantivyStatus::Ok + } + Err(e) => { + let already_finished = e == "writer already finished"; + let io_err = e.starts_with("write callback rc=") + || e.starts_with("open ") + || e.starts_with("read "); + set_last_error(e); + if already_finished { + PaimonTantivyStatus::InvalidArgument + } else if io_err { + PaimonTantivyStatus::IoError + } else { + PaimonTantivyStatus::InternalError + } + } + } +} + +/// Destroy a writer handle. Safe on null. Tmp dir is removed via Drop. +#[no_mangle] +pub unsafe extern "C" fn paimon_tantivy_writer_free(writer: *mut PaimonTantivyWriter) { + unsafe { free_handle(writer) }; +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + + /// Test dict dir for jieba; defaults to a non-existent path so jieba-rs uses + /// its built-in dict (which is enough for these smoke tests). + fn dict_dir_from_env() -> std::path::PathBuf { + std::env::var("PAIMON_JIEBA_DICT_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|_| std::path::PathBuf::from("/tmp/nonexistent-dict")) + } + + #[test] + fn empty_field_name_rejected() { + let err = PaimonTantivyWriter::new("", TokenizeMode::Mix, true, Path::new("/tmp/nx"), "paimon_jieba") + .err() + .unwrap(); + assert!(err.contains("field_name"), "got: {err}"); + } + + #[test] + fn hmm_mode_rejected() { + let err = + PaimonTantivyWriter::new("f0", TokenizeMode::Hmm, true, Path::new("/tmp/nx"), "paimon_jieba") + .err() + .unwrap(); + assert!(err.contains("'hmm' is not supported"), "got: {err}"); + } + + #[test] + fn create_add_finish_roundtrip() { + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + w.add(0, "hello world").unwrap(); + w.add(1, "中国人民").unwrap(); + w.add(2, "").unwrap(); // empty doc + let (rows, bytes) = w.finish().unwrap(); + assert_eq!(rows, 3); + assert!(bytes.len() > 4); + + // Validate header (Java-compatible: BE int32 file_count, no version) + let file_count = i32::from_be_bytes(bytes[0..4].try_into().unwrap()); + assert!(file_count > 0, "expected >0 packed files"); + + // Walk entries (BE) + let mut off: usize = 4; + let mut names = Vec::new(); + for _ in 0..file_count { + let nlen = i32::from_be_bytes(bytes[off..off + 4].try_into().unwrap()) as usize; + off += 4; + let name = std::str::from_utf8(&bytes[off..off + nlen]).unwrap().to_owned(); + off += nlen; + let flen = i64::from_be_bytes(bytes[off..off + 8].try_into().unwrap()) as usize; + off += 8; + assert!(off + flen <= bytes.len(), "file {name} extends past buffer"); + off += flen; + names.push(name); + } + assert_eq!(off, bytes.len(), "trailing bytes after pack"); + // tantivy must produce at least meta.json + assert!(names.iter().any(|n| n == "meta.json"), "names={names:?}"); + } + + #[test] + fn schema_field_names_are_fixed() { + // Schema must be `row_id` (u64) + `text` (TEXT) regardless of caller's + // field_name argument — matches paimon-java for cross-readability. + let w = + PaimonTantivyWriter::new("ignored_name", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba") + .unwrap(); + let schema = w.index.schema(); + assert!(schema.get_field(PAIMON_ROW_ID_FIELD_NAME).is_ok(), + "schema must have row_id field"); + assert!(schema.get_field(PAIMON_TEXT_FIELD_NAME).is_ok(), + "schema must have text field"); + // Caller-supplied name must NOT appear + assert!(schema.get_field("ignored_name").is_err(), + "caller-supplied field_name must be ignored"); + } + + #[test] + fn archive_uses_big_endian_no_version_header() { + // Strong guard: header must be BE int32 file_count, NOT LE int32 + // version=1 + LE int32 file_count. Any regression to LE/version-header + // would silently break paimon-java cross-read. + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + w.add(0, "hello").unwrap(); + let (_, bytes) = w.finish().unwrap(); + let header_be = i32::from_be_bytes(bytes[0..4].try_into().unwrap()); + let header_le = i32::from_le_bytes(bytes[0..4].try_into().unwrap()); + // BE file_count is small (single-segment force-merge: ~6-7 files) + assert!(header_be > 0 && header_be < 100, + "expected sensible BE file_count, got BE={header_be} LE={header_le}"); + // LE-decoded header would be a huge number (e.g. 0x06000000), ensuring + // we did NOT regress to the old LE+version layout. + assert_ne!(header_be, header_le, "buffer must be BE-encoded"); + } + + #[test] + fn multi_thread_writer_default() { + // Schema stores row_id explicitly so we no longer enforce + // single-threaded writer. Just verify many docs across threads land + // correctly and force-merge collapses to a single segment. + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + for i in 0..200u64 { + w.add(i, &format!("row {i} apple banana")).unwrap(); + } + let (rows, bytes) = w.finish().unwrap(); + assert_eq!(rows, 200); + assert!(bytes.len() > 4); + // After force-merge there must be exactly one meta.json + segment files. + let file_count = i32::from_be_bytes(bytes[0..4].try_into().unwrap()); + assert!(file_count >= 2, "force-merged single segment needs ≥ 2 files (meta + segment), got {file_count}"); + } + + #[test] + fn finish_twice_errors() { + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + w.add(0, "hi").unwrap(); + let _ = w.finish().unwrap(); + let err = w.finish().err().unwrap(); + assert!(err.contains("already finished"), "got: {err}"); + } + + /// Mock collector for FFI streaming tests: push bytes into a Box> + /// pointed to by `ctx`. (No Arc / atomic needed — test is single-threaded.) + extern "C" fn mock_write_collect(ctx: *mut c_void, data: *const u8, len: usize) -> i32 { + let vec = unsafe { &mut *(ctx as *mut Vec) }; + let slice = unsafe { std::slice::from_raw_parts(data, len) }; + vec.extend_from_slice(slice); + 0 + } + + /// Mock that counts the largest single `write` call — sanity check that + /// Rust streams in chunks bounded by the buffer (+ small header fields). + extern "C" fn mock_write_max_chunk( + ctx: *mut c_void, + _data: *const u8, + len: usize, + ) -> i32 { + let max = unsafe { &mut *(ctx as *mut usize) }; + if len > *max { + *max = len; + } + 0 + } + + #[test] + fn ffi_full_path_streaming() { + unsafe { + let field = CString::new("f0").unwrap(); + let mode = CString::new("mix").unwrap(); + let dict = CString::new(dict_dir_from_env().to_str().unwrap()).unwrap(); + let tokenizer = CString::new("paimon_jieba").unwrap(); + let mut handle: *mut PaimonTantivyWriter = std::ptr::null_mut(); + let st = paimon_tantivy_writer_new( + field.as_ptr(), + mode.as_ptr(), + true, + dict.as_ptr(), + tokenizer.as_ptr(), + &mut handle, + ); + assert_eq!(st, PaimonTantivyStatus::Ok); + assert!(!handle.is_null()); + + let txt = "hello world"; + let st = + paimon_tantivy_writer_add(handle, 42u64, txt.as_ptr() as *const c_char, txt.len()); + assert_eq!(st, PaimonTantivyStatus::Ok); + + // Streaming finish: collect bytes into a Vec via FFI callback + let mut out: Vec = Vec::new(); + let cb = PaimonWriteCallbacks { + ctx: &mut out as *mut _ as *mut c_void, + write: mock_write_collect, + }; + let mut rows: i64 = 0; + let st = paimon_tantivy_writer_finish_streaming(handle, cb, &mut rows); + assert_eq!(st, PaimonTantivyStatus::Ok); + assert_eq!(rows, 1); + // BE file_count at byte 0,> 0 + let file_count = i32::from_be_bytes(out[0..4].try_into().unwrap()); + assert!(file_count > 0); + + // double finish must error + let mut out2: Vec = Vec::new(); + let cb2 = PaimonWriteCallbacks { + ctx: &mut out2 as *mut _ as *mut c_void, + write: mock_write_collect, + }; + let mut rows2: i64 = 0; + let st = paimon_tantivy_writer_finish_streaming(handle, cb2, &mut rows2); + assert_eq!(st, PaimonTantivyStatus::InvalidArgument); + + paimon_tantivy_writer_free(handle); + } + } + + #[test] + fn streaming_chunk_size_bounded_by_buffer() { + // After force-merge, a 200-doc index still streams in chunks bounded by + // WRITER_STREAM_BUFFER_SIZE (payload) or small header-field chunks. + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + for i in 0..200u64 { + w.add(i, &format!("row {i} apple banana")).unwrap(); + } + let mut max_chunk: usize = 0; + let cb = PaimonWriteCallbacks { + ctx: &mut max_chunk as *mut _ as *mut c_void, + write: mock_write_max_chunk, + }; + let rows = w.finish_streaming(&cb).unwrap(); + assert_eq!(rows, 200); + assert!( + max_chunk <= WRITER_STREAM_BUFFER_SIZE, + "streaming chunk size {} exceeded buffer {}", + max_chunk, + WRITER_STREAM_BUFFER_SIZE + ); + } + + #[test] + fn streaming_write_callback_error_propagates() { + extern "C" fn always_fail(_ctx: *mut c_void, _data: *const u8, _len: usize) -> i32 { + 7 + } + let mut w = + PaimonTantivyWriter::new("f0", TokenizeMode::Mix, true, &dict_dir_from_env(), "paimon_jieba").unwrap(); + w.add(0, "hello").unwrap(); + let cb = PaimonWriteCallbacks { + ctx: std::ptr::null_mut(), + write: always_fail, + }; + let err = w.finish_streaming(&cb).unwrap_err(); + assert!(err.contains("write callback rc=7"), "got: {err}"); + } + + #[test] + fn ffi_null_writer_invalid() { + unsafe { + let txt = "x"; + let st = paimon_tantivy_writer_add( + std::ptr::null_mut(), + 0u64, + txt.as_ptr() as *const c_char, + txt.len(), + ); + assert_eq!(st, PaimonTantivyStatus::InvalidArgument); + } + } +} From 82ffaaf54874cf60960dbfe15cea75ee723455c7 Mon Sep 17 00:00:00 2001 From: gripleaf <425797155@qq.com> Date: Mon, 22 Jun 2026 09:37:02 +0800 Subject: [PATCH 066/138] feat(parquet): support parquet metadata cache --- docs/source/user_guide.rst | 1 + .../user_guide/parquet_metadata_cache.rst | 110 ++++++++++++++++ include/paimon/cache/cache.h | 1 + include/paimon/format/reader_builder.h | 7 + .../core/manifest/manifest_file_test.cpp | 5 +- .../core/operation/abstract_split_read.cpp | 1 + .../page_filtered_row_group_reader_test.cpp | 6 +- .../parquet/parquet_file_batch_reader.cpp | 8 +- .../parquet/parquet_file_batch_reader.h | 14 +- .../parquet_file_batch_reader_test.cpp | 124 +++++++++++++++++- .../format/parquet/parquet_reader_builder.h | 115 +++++++++++++++- .../parquet/predicate_pushdown_test.cpp | 6 +- ...st_utils.h => counting_cache_test_utils.h} | 39 ++++-- test/inte/scan_inte_test.cpp | 8 +- test/inte/write_and_read_inte_test.cpp | 90 +++++++++++++ 15 files changed, 494 insertions(+), 41 deletions(-) create mode 100644 docs/source/user_guide/parquet_metadata_cache.rst rename src/paimon/testing/utils/{manifest_cache_test_utils.h => counting_cache_test_utils.h} (68%) diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index d7597f48..5d503beb 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -28,6 +28,7 @@ User Guide user_guide/snapshot user_guide/manifest user_guide/manifest_cache + user_guide/parquet_metadata_cache user_guide/data_types user_guide/primary_key_table user_guide/append_only_table diff --git a/docs/source/user_guide/parquet_metadata_cache.rst b/docs/source/user_guide/parquet_metadata_cache.rst new file mode 100644 index 00000000..d0e9626e --- /dev/null +++ b/docs/source/user_guide/parquet_metadata_cache.rst @@ -0,0 +1,110 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Parquet Metadata Cache +====================== + +Overview +-------- + +paimon-cpp can cache serialized Parquet metadata footer bytes for Parquet data files. +The cache is used by ``ParquetReaderBuilder`` before opening the Arrow Parquet +reader. On a cache miss, paimon-cpp loads the Parquet file metadata, serializes +it as a complete metadata footer, and stores those bytes in the public +``Cache`` abstraction. On a cache hit, paimon-cpp parses the cached footer bytes into +``parquet::FileMetaData`` and passes the metadata to the Parquet reader. + +The cache stores serialized metadata footer bytes instead of caching a +``parquet::FileMetaData`` instance. This keeps the cache value compact and +similar to manifest cache values: the cache weight follows the actual cached +bytes, while the Parquet library still owns metadata parsing and validation. + +This optimization is useful when the same Parquet files are opened repeatedly +in the same process, for example repeated ``get`` or ``scan`` requests over the +same snapshot. On a cache hit, the read path avoids reading the Parquet footer +bytes from the filesystem again. paimon-cpp still parses the cached footer bytes +into ``parquet::FileMetaData`` for each reader open. Data pages, page indexes, +and column chunks are still read from the file as usual. + +Configuration +------------- + +Parquet metadata caching is disabled by default. Embedding applications that +need it can provide a custom ``Cache`` implementation and inject it through +``ScanContextBuilder`` or ``ReadContextBuilder``. Parquet reader builders +receive the cache from the read context and create cache keys with +``CacheKind::DATA_FILE_FOOTER`` internally. + +The cache key represents the file footer and is created from the file URI with +position ``-1`` and length ``-1``. Callers do not need to construct this key +directly; they only need to route ``CacheKind::DATA_FILE_FOOTER`` entries to an +appropriate cache backend. + +Example: + +.. code-block:: cpp + + class RoutingCache : public paimon::Cache { + public: + RoutingCache(std::shared_ptr default_cache, + std::shared_ptr parquet_metadata_cache) + : default_cache_(std::move(default_cache)), + parquet_metadata_cache_(std::move(parquet_metadata_cache)) {} + + paimon::Result> Get( + const std::shared_ptr& key, + std::function>( + const std::shared_ptr&)> supplier) override { + return Select(key)->Get(key, std::move(supplier)); + } + + // Put(), Invalidate(), InvalidateAll(), and Size() route in the same way. + + private: + std::shared_ptr Select( + const std::shared_ptr& key) const { + return key && key->GetKind() == paimon::CacheKind::DATA_FILE_FOOTER + ? parquet_metadata_cache_ + : default_cache_; + } + + std::shared_ptr default_cache_; + std::shared_ptr parquet_metadata_cache_; + }; + + auto cache = std::make_shared( + std::make_shared(), + std::make_shared()); + + paimon::ScanContextBuilder scan_builder(table_path); + scan_builder.WithCache(cache); + + paimon::ReadContextBuilder read_builder(table_path); + read_builder.WithCache(cache); + +Passing ``nullptr`` or omitting ``WithCache()`` leaves Parquet metadata caching +disabled. If a file URI cannot be obtained, paimon-cpp also bypasses the cache +and opens the Parquet file normally. + +Future Optimizations +-------------------- + +- Add hit, miss, bypass, and eviction metrics for Parquet metadata cache. +- Add single-flight loading for high-concurrency misses on the same Parquet + file. +- Evaluate sharing cached metadata footer bytes with page-index prefetch logic when + those read paths can use the same cache abstraction. diff --git a/include/paimon/cache/cache.h b/include/paimon/cache/cache.h index fba26a99..1edcb472 100644 --- a/include/paimon/cache/cache.h +++ b/include/paimon/cache/cache.h @@ -34,6 +34,7 @@ class CacheValue; enum class CacheKind { DEFAULT, MANIFEST, + DATA_FILE_FOOTER, }; class PAIMON_EXPORT CacheKey { diff --git a/include/paimon/format/reader_builder.h b/include/paimon/format/reader_builder.h index ca2021c3..7ef1bf8f 100644 --- a/include/paimon/format/reader_builder.h +++ b/include/paimon/format/reader_builder.h @@ -25,6 +25,7 @@ #include "paimon/type_fwd.h" namespace paimon { +class Cache; /// Create a file batch reader based on the file path. Allows you to specify memory pool. class PAIMON_EXPORT ReaderBuilder { @@ -34,6 +35,12 @@ class PAIMON_EXPORT ReaderBuilder { /// Set memory pool to use. virtual ReaderBuilder* WithMemoryPool(const std::shared_ptr& pool) = 0; + /// Inject a cache for reader-specific immutable metadata. + virtual ReaderBuilder* WithCache(const std::shared_ptr& cache) { + (void)cache; + return this; + } + /// Build a file batch reader based on the created `InputStream`. virtual Result> Build( const std::shared_ptr& path) const = 0; diff --git a/src/paimon/core/manifest/manifest_file_test.cpp b/src/paimon/core/manifest/manifest_file_test.cpp index 499cc756..8bb2c22e 100644 --- a/src/paimon/core/manifest/manifest_file_test.cpp +++ b/src/paimon/core/manifest/manifest_file_test.cpp @@ -43,7 +43,7 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/binary_row_generator.h" -#include "paimon/testing/utils/manifest_cache_test_utils.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -279,7 +279,8 @@ TEST_F(ManifestFileTest, TestManifestCacheIsDisabledWithoutInjectedCache) { TEST_F(ManifestFileTest, TestManifestCacheReusesCachedBytes) { auto pool = GetDefaultPool(); auto counting_file_system = std::make_shared(); - auto manifest_cache = std::make_shared(); + auto manifest_cache = + std::make_shared(CacheKind::MANIFEST, 64 * 1024 * 1024); ASSERT_OK_AND_ASSIGN(std::shared_ptr file_format, FileFormatFactory::Get("orc", {})); std::string root_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index b00a08ef..0f19fd80 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -119,6 +119,7 @@ Result> AbstractSplitRead::PrepareReaderBuilder( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, file_format->CreateReaderBuilder(options_.GetReadBatchSize())); reader_builder->WithMemoryPool(pool_); + reader_builder->WithCache(options_.GetCache()); return reader_builder; } diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 5c1cb89c..0186f309 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -121,9 +121,9 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { std::map options; options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "true"; - ASSERT_OK_AND_ASSIGN( - auto batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), arrow_pool_, options, batch_size)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, batch_size, + /*file_metadata=*/nullptr, arrow_pool_)); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate, diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index f1223706..456c4d3f 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -73,8 +73,9 @@ ParquetFileBatchReader::ParquetFileBatchReader( Result> ParquetFileBatchReader::Create( std::shared_ptr&& input_stream, - const std::shared_ptr& pool, - const std::map& options, int32_t batch_size) { + const std::map& options, int32_t batch_size, + std::shared_ptr<::parquet::FileMetaData> file_metadata, + const std::shared_ptr& pool) { try { assert(input_stream); PAIMON_ASSIGN_OR_RAISE(::parquet::ReaderProperties reader_properties, @@ -84,7 +85,8 @@ Result> ParquetFileBatchReader::Create( CreateArrowReaderProperties(pool, options, batch_size)); ::parquet::arrow::FileReaderBuilder file_reader_builder; - PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.Open(input_stream, reader_properties)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + file_reader_builder.Open(input_stream, reader_properties, std::move(file_metadata))); std::unique_ptr<::parquet::arrow::FileReader> file_reader; PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_builder.memory_pool(pool.get()) diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 393bc385..7bfc2e1e 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -53,6 +53,9 @@ namespace io { class RandomAccessFile; } // namespace io } // namespace arrow +namespace parquet { +class FileMetaData; +} // namespace parquet namespace paimon { class Metrics; class Predicate; @@ -65,8 +68,13 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { public: static Result> Create( std::shared_ptr&& input_stream, + const std::map& options, int32_t batch_size, + std::shared_ptr<::parquet::FileMetaData> file_metadata, + const std::shared_ptr& pool); + + static Result<::parquet::ReaderProperties> CreateReaderProperties( const std::shared_ptr& pool, - const std::map& options, int32_t batch_size); + const std::map& options); // For timestamp type, we return the schema stored in file, e.g., second in parquet file will // store as milli. @@ -130,10 +138,6 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { const std::map& options, const std::shared_ptr& arrow_pool); - static Result<::parquet::ReaderProperties> CreateReaderProperties( - const std::shared_ptr& pool, - const std::map& options); - static Result<::parquet::ArrowReaderProperties> CreateArrowReaderProperties( const std::shared_ptr& pool, const std::map& options, int32_t batch_size); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index 33107779..2041166e 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -18,6 +18,7 @@ #include "paimon/format/parquet/parquet_file_batch_reader.h" +#include #include #include #include @@ -43,12 +44,14 @@ #include "paimon/defs.h" #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_format_writer.h" +#include "paimon/format/parquet/parquet_reader_builder.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/reader/batch_reader.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" #include "paimon/testing/utils/timezone_guard.h" @@ -67,6 +70,47 @@ std::string SerializeSchemaToString(const std::shared_ptr& schema static_cast(serialized->size())); } +class FailedUriInputStream : public InputStream { + public: + explicit FailedUriInputStream(const std::shared_ptr& input) : input_(input) {} + + Status Seek(int64_t offset, SeekOrigin origin) override { + return input_->Seek(offset, origin); + } + + Result GetPos() const override { + return input_->GetPos(); + } + + Result Read(char* buffer, int64_t size) override { + return input_->Read(buffer, size); + } + + Result Read(char* buffer, int64_t size, int64_t offset) override { + return input_->Read(buffer, size, offset); + } + + void ReadAsync(char* buffer, int64_t size, int64_t offset, + std::function&& callback) override { + return input_->ReadAsync(buffer, size, offset, std::move(callback)); + } + + Result GetUri() const override { + return Status::Invalid("failed to get uri"); + } + + Result Length() const override { + return input_->Length(); + } + + Status Close() override { + return input_->Close(); + } + + private: + std::shared_ptr input_; +}; + class ParquetFileBatchReaderTest : public ::testing::Test, public ::testing::WithParamInterface { public: @@ -154,7 +198,8 @@ class ParquetFileBatchReaderTest : public ::testing::Test, const std::optional& selection_bitmap, int32_t batch_size) const { EXPECT_OK_AND_ASSIGN( auto parquet_batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), pool_, options, batch_size)); + ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size, + /*file_metadata=*/nullptr, pool_)); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); EXPECT_TRUE(arrow_status.ok()); @@ -162,7 +207,7 @@ class ParquetFileBatchReaderTest : public ::testing::Test, return parquet_batch_reader; } - private: + protected: std::string file_path_; std::unique_ptr dir_; std::shared_ptr fs_; @@ -172,6 +217,75 @@ class ParquetFileBatchReaderTest : public ::testing::Test, std::shared_ptr struct_array_; }; +TEST_F(ParquetFileBatchReaderTest, TestParquetMetadataCacheReusesSerializedFooter) { + WriteArray(file_path_, struct_array_, schema_, /*write_batch_size=*/struct_array_->length(), + /*enable_dictionary=*/false, + /*max_row_group_length=*/struct_array_->length()); + + auto cache = std::make_shared(CacheKind::DATA_FILE_FOOTER, + 128 * 1024 * 1024); + auto open_reader = [&]() -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, fs_->Open(file_path_)); + std::map options; + ParquetReaderBuilder builder(options, batch_size_); + builder.WithMemoryPool(GetDefaultPool())->WithCache(cache); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + builder.Build(input_stream)); + auto parquet_reader = dynamic_cast(reader.release()); + if (parquet_reader == nullptr) { + return Status::Invalid("failed to cast FileBatchReader to ParquetFileBatchReader"); + } + return std::unique_ptr(parquet_reader); + }; + + ASSERT_OK_AND_ASSIGN(auto reader1, open_reader()); + ASSERT_OK_AND_ASSIGN(auto schema1, reader1->GetFileSchema()); + ASSERT_TRUE(schema1); + ASSERT_TRUE(schema1->release); + schema1->release(schema1.get()); + ASSERT_EQ(1, cache->GetCount()); + ASSERT_EQ(1, cache->SupplierCallCount()); + ASSERT_EQ(1, cache->Size()); + ASSERT_EQ(CacheKind::DATA_FILE_FOOTER, cache->LastKind()); + + ASSERT_OK_AND_ASSIGN(auto reader2, open_reader()); + ASSERT_OK_AND_ASSIGN(auto schema2, reader2->GetFileSchema()); + ASSERT_TRUE(schema2); + ASSERT_TRUE(schema2->release); + schema2->release(schema2.get()); + ASSERT_EQ(2, cache->GetCount()); + ASSERT_EQ(1, cache->SupplierCallCount()); + ASSERT_EQ(1, cache->Size()); + ASSERT_EQ(CacheKind::DATA_FILE_FOOTER, cache->LastKind()); +} + +TEST_F(ParquetFileBatchReaderTest, TestParquetMetadataCacheBypassesWhenGetUriFails) { + WriteArray(file_path_, struct_array_, schema_, /*write_batch_size=*/struct_array_->length(), + /*enable_dictionary=*/false, + /*max_row_group_length=*/struct_array_->length()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs_->Open(file_path_)); + auto failed_uri_input_stream = std::make_shared(input_stream); + auto cache = std::make_shared(CacheKind::DATA_FILE_FOOTER, + 128 * 1024 * 1024); + + std::map options; + ParquetReaderBuilder builder(options, batch_size_); + builder.WithMemoryPool(GetDefaultPool())->WithCache(cache); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + builder.Build(failed_uri_input_stream)); + auto parquet_reader = dynamic_cast(reader.get()); + ASSERT_TRUE(parquet_reader); + ASSERT_OK_AND_ASSIGN(auto file_schema, parquet_reader->GetFileSchema()); + ASSERT_TRUE(file_schema); + ASSERT_TRUE(file_schema->release); + file_schema->release(file_schema.get()); + + ASSERT_EQ(0, cache->GetCount()); + ASSERT_EQ(0, cache->SupplierCallCount()); + ASSERT_EQ(0, cache->Size()); +} + TEST_F(ParquetFileBatchReaderTest, TestReadBinaryWrittenFromBinaryAndLargeBinary) { auto check_binary_read_result = [&](const std::shared_ptr& write_type, const std::string& file_name) { @@ -238,9 +352,9 @@ TEST_F(ParquetFileBatchReaderTest, TestSetReadSchema) { auto in_stream = std::make_unique(std::move(input_stream), pool_, length); std::map options; - ASSERT_OK_AND_ASSIGN( - auto parquet_batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), pool_, options, batch_size_)); + ASSERT_OK_AND_ASSIGN(auto parquet_batch_reader, + ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size_, + /*file_metadata=*/nullptr, pool_)); // test GetFileSchema() ASSERT_OK_AND_ASSIGN(auto c_file_schema, parquet_batch_reader->GetFileSchema()); auto arrow_file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); diff --git a/src/paimon/format/parquet/parquet_reader_builder.h b/src/paimon/format/parquet/parquet_reader_builder.h index dadbb8cf..76e2fd9c 100644 --- a/src/paimon/format/parquet/parquet_reader_builder.h +++ b/src/paimon/format/parquet/parquet_reader_builder.h @@ -18,18 +18,29 @@ #pragma once +#include +#include +#include #include #include #include #include +#include "arrow/buffer.h" +#include "arrow/io/memory.h" +#include "fmt/format.h" +#include "paimon/cache/cache.h" #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/format/parquet/parquet_file_batch_reader.h" +#include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/reader_builder.h" #include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" #include "paimon/reader/file_batch_reader.h" #include "paimon/result.h" +#include "parquet/file_reader.h" +#include "parquet/file_writer.h" namespace paimon::parquet { @@ -43,20 +54,112 @@ class ParquetReaderBuilder : public ReaderBuilder { return this; } + ReaderBuilder* WithCache(const std::shared_ptr& cache) override { + cache_ = cache; + return this; + } + Result> Build( const std::shared_ptr& path) const override { - PAIMON_ASSIGN_OR_RAISE(int64_t file_length, path->Length()); - std::shared_ptr arrow_pool = GetArrowPool(pool_); - auto input_stream = - std::make_unique(path, arrow_pool, file_length); - return ParquetFileBatchReader::Create(std::move(input_stream), arrow_pool, options_, - batch_size_); + try { + PAIMON_ASSIGN_OR_RAISE(int64_t file_length, path->Length()); + std::string file_uri; + if (cache_) { + Result file_uri_result = path->GetUri(); + if (file_uri_result.ok()) { + file_uri = std::move(file_uri_result).value(); + } + } + std::shared_ptr arrow_pool = GetArrowPool(pool_); + auto unique_input_stream = + std::make_unique(path, arrow_pool, file_length); + std::shared_ptr input_stream( + std::move(unique_input_stream)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<::parquet::FileMetaData> file_metadata, + GetCachedParquetMetadata(input_stream, file_uri, arrow_pool)); + return ParquetFileBatchReader::Create(std::move(input_stream), options_, batch_size_, + std::move(file_metadata), arrow_pool); + } + PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetReaderBuilder::Build") } private: + Result SerializeParquetMetadataFooter( + const std::shared_ptr& input_stream, + const ::parquet::ReaderProperties& reader_properties, + const std::shared_ptr& arrow_pool) const { + constexpr int64_t kParquetFooterSize = 8; + + std::shared_ptr<::parquet::FileMetaData> metadata = + ::parquet::ParquetFileReader::Open(input_stream, reader_properties)->metadata(); + if (metadata == nullptr) { + return Status::Invalid("Failed to read parquet metadata"); + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr output_stream, + arrow::io::BufferOutputStream::Create(metadata->size() + kParquetFooterSize, + arrow_pool.get())); + ::parquet::WriteFileMetaData(*metadata, output_stream.get()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr metadata_footer, + output_stream->Finish()); + + MemorySegment segment = + MemorySegment::AllocateHeapMemory(metadata_footer->size(), pool_.get()); + std::memcpy(segment.MutableData(), metadata_footer->data(), metadata_footer->size()); + return segment; + } + + static Result> ParseParquetMetadataFooter( + const MemorySegment& segment, const ::parquet::ReaderProperties& reader_properties) { + if (segment.Data() == nullptr || segment.Size() <= 0) { + return Status::Invalid("Parquet metadata cache value is empty"); + } + + auto buffer = std::make_shared( + reinterpret_cast(segment.Data()), segment.Size()); + auto buffer_reader = std::make_shared(buffer); + std::shared_ptr<::parquet::FileMetaData> metadata = + ::parquet::ParquetFileReader::Open(buffer_reader, reader_properties)->metadata(); + if (metadata == nullptr) { + return Status::Invalid("Failed to parse parquet metadata footer"); + } + return metadata; + } + + Result> GetCachedParquetMetadata( + const std::shared_ptr& input_stream, + const std::string& file_uri, const std::shared_ptr& arrow_pool) const { + if (!cache_ || file_uri.empty()) { + return std::shared_ptr<::parquet::FileMetaData>(); + } + PAIMON_ASSIGN_OR_RAISE( + ::parquet::ReaderProperties reader_properties, + ParquetFileBatchReader::CreateReaderProperties(arrow_pool, options_)); + + auto cache_key = CacheKey::ForKind(file_uri, /*position=*/-1, /*length=*/-1, + CacheKind::DATA_FILE_FOOTER); + auto supplier = + [this, &input_stream, reader_properties, + arrow_pool](const std::shared_ptr&) -> Result> { + PAIMON_ASSIGN_OR_RAISE( + MemorySegment segment, + SerializeParquetMetadataFooter(input_stream, reader_properties, arrow_pool)); + return std::make_shared(segment, CacheCallback()); + }; + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache_value, + cache_->Get(cache_key, supplier)); + if (cache_value == nullptr) { + return Status::Invalid("Parquet metadata cache returned nullptr value"); + } + return ParseParquetMetadataFooter(cache_value->GetSegment(), reader_properties); + } + int32_t batch_size_ = -1; std::shared_ptr pool_; std::map options_; + std::shared_ptr cache_; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/predicate_pushdown_test.cpp b/src/paimon/format/parquet/predicate_pushdown_test.cpp index 4399f21b..64ed5a54 100644 --- a/src/paimon/format/parquet/predicate_pushdown_test.cpp +++ b/src/paimon/format/parquet/predicate_pushdown_test.cpp @@ -112,9 +112,9 @@ class PredicatePushdownTest : public ::testing::Test { std::map options; options[paimon::parquet::PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT] = std::to_string(predicate_node_count_limit); - ASSERT_OK_AND_ASSIGN(auto batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), arrow_pool_, - options, batch_size_)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, batch_size_, + /*file_metadata=*/nullptr, arrow_pool_)); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); ASSERT_TRUE(arrow_status.ok()); diff --git a/src/paimon/testing/utils/manifest_cache_test_utils.h b/src/paimon/testing/utils/counting_cache_test_utils.h similarity index 68% rename from src/paimon/testing/utils/manifest_cache_test_utils.h rename to src/paimon/testing/utils/counting_cache_test_utils.h index fa2c7d41..c366fdf4 100644 --- a/src/paimon/testing/utils/manifest_cache_test_utils.h +++ b/src/paimon/testing/utils/counting_cache_test_utils.h @@ -26,17 +26,22 @@ #include #include -#include "gtest/gtest.h" #include "paimon/cache/cache.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/result.h" namespace paimon::test { -class CountingManifestRoutingCache : public Cache { +class CountingRoutingCache : public Cache { public: - explicit CountingManifestRoutingCache(int64_t max_weight = 64 * 1024 * 1024) { - caches_[CacheKind::MANIFEST] = std::make_shared(max_weight); + CountingRoutingCache(CacheKind kind, int64_t max_weight) { + caches_[kind] = std::make_shared(max_weight); + } + + explicit CountingRoutingCache(const std::map& max_weights) { + for (const auto& [kind, max_weight] : max_weights) { + caches_[kind] = std::make_shared(max_weight); + } } Result> Get( @@ -44,7 +49,9 @@ class CountingManifestRoutingCache : public Cache { std::function>(const std::shared_ptr&)> supplier) override { ++get_count_; - return GetCache(key)->Get( + last_kind_ = key->GetKind(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache, GetCache(key)); + return cache->Get( key, [this, supplier = std::move(supplier)](const std::shared_ptr& supplier_key) -> Result> { @@ -55,11 +62,15 @@ class CountingManifestRoutingCache : public Cache { Status Put(const std::shared_ptr& key, const std::shared_ptr& value) override { - return GetCache(key)->Put(key, value); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache, GetCache(key)); + return cache->Put(key, value); } void Invalidate(const std::shared_ptr& key) override { - GetCache(key)->Invalidate(key); + Result> cache = GetCache(key); + if (cache.ok()) { + cache.value()->Invalidate(key); + } } void InvalidateAll() override { @@ -84,17 +95,23 @@ class CountingManifestRoutingCache : public Cache { return supplier_call_count_; } + CacheKind LastKind() const { + return last_kind_; + } + private: - std::shared_ptr GetCache(const std::shared_ptr& key) const { - EXPECT_EQ(CacheKind::MANIFEST, key->GetKind()); + Result> GetCache(const std::shared_ptr& key) const { auto iter = caches_.find(key->GetKind()); - EXPECT_NE(caches_.end(), iter); - return iter == caches_.end() ? nullptr : iter->second; + if (iter == caches_.end()) { + return Status::Invalid("unexpected cache kind"); + } + return iter->second; } std::map> caches_; int64_t get_count_ = 0; int64_t supplier_call_count_ = 0; + CacheKind last_kind_ = CacheKind::DEFAULT; }; } // namespace paimon::test diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 0a781c5f..a3b267df 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -55,7 +55,7 @@ #include "paimon/table/source/startup_mode.h" #include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/binary_row_generator.h" -#include "paimon/testing/utils/manifest_cache_test_utils.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -66,7 +66,8 @@ class ScanInteTest : public testing::TestWithParam { Result> FinishScanContext(ScanContextBuilder& builder) { if (GetParam() == ManifestCacheMode::Cache) { if (!cache_) { - cache_ = std::make_shared(); + cache_ = + std::make_shared(CacheKind::MANIFEST, 64 * 1024 * 1024); } builder.WithCache(cache_); } @@ -260,7 +261,8 @@ class ScanInteTest : public testing::TestWithParam { }; TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { - auto manifest_cache = std::make_shared(); + auto manifest_cache = + std::make_shared(CacheKind::MANIFEST, 64 * 1024 * 1024); std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; auto run_scan = [&]() -> Result> { diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index b52ba8c2..d1c7956e 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -44,6 +44,7 @@ #include "paimon/table/source/startup_mode.h" #include "paimon/table/source/table_read.h" #include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" @@ -1097,6 +1098,95 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetPageIndexFilter) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } +TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" || file_system != "local") { + return; + } + + auto test_dir = UniqueTestDirectory::Create("local"); + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "parquet"}, + {Options::TARGET_FILE_SIZE, "1048576"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, "local"}, + }; + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir->Str(), schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); + std::string table_path = test_dir->Str() + "/foo.db/bar"; + + std::string data = R"([ + ["banana", 2], + ["dog", 1], + ["lucy", 14], + ["mouse", 100] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_data_type = arrow::struct_(fields_with_row_kind); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_data_type, R"([ + [0, "banana", 2], + [0, "dog", 1], + [0, "lucy", 14], + [0, "mouse", 100] + ])") + .ValueOrDie()); + + auto cache = + std::make_shared(CacheKind::DATA_FILE_FOOTER, 128 * 1024 * 1024); + auto read_once = [&]() -> Result { + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); + if (result_plan->SnapshotId() != std::optional(1)) { + return Status::Invalid("unexpected snapshot id"); + } + if (result_plan->Splits().empty()) { + return Status::Invalid("no splits found"); + } + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.WithCache(cache); + PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(auto read_result, + ReadResultCollector::CollectResult(batch_reader.get())); + if (!read_result) { + return Status::Invalid("read result is null"); + } + return expected->Equals(read_result); + }; + + ASSERT_OK_AND_ASSIGN(bool first_success, read_once()); + ASSERT_TRUE(first_success); + ASSERT_EQ(1, cache->GetCount()); + ASSERT_EQ(1, cache->SupplierCallCount()); + ASSERT_EQ(1, cache->Size()); + ASSERT_EQ(CacheKind::DATA_FILE_FOOTER, cache->LastKind()); + + ASSERT_OK_AND_ASSIGN(bool second_success, read_once()); + ASSERT_TRUE(second_success); + ASSERT_EQ(2, cache->GetCount()); + ASSERT_EQ(1, cache->SupplierCallCount()); + ASSERT_EQ(1, cache->Size()); + ASSERT_EQ(CacheKind::DATA_FILE_FOOTER, cache->LastKind()); +} + INSTANTIATE_TEST_SUITE_P(FileFormatAndFileSystem, WriteAndReadInteTest, ::testing::ValuesIn(GetTestValuesForWriteAndReadInteTest())); From 1c8616b2469e9fa5e946911236b77a78ec32ec23 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Mon, 22 Jun 2026 10:57:09 +0800 Subject: [PATCH 067/138] fix(format): reset avro reader on schema change --- .../format/avro/avro_file_batch_reader.cpp | 19 +++++-- .../avro/avro_file_batch_reader_test.cpp | 49 +++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp index 2a8aa53c..5daf1b89 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader.cpp @@ -144,18 +144,27 @@ Status AvroFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, if (selection_bitmap) { // TODO(menglingda.mld): support bitmap } - previous_first_row_ = std::numeric_limits::max(); - next_row_to_read_ = std::numeric_limits::max(); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_read_schema, arrow::ImportSchema(read_schema)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, ArrowUtils::DataTypeToSchema(file_data_type_)); - PAIMON_ASSIGN_OR_RAISE(read_fields_projection_, + PAIMON_ASSIGN_OR_RAISE(std::set read_fields_projection, CalculateReadFieldsProjection(file_schema, arrow_read_schema->fields())); - array_builder_->Reset(); std::shared_ptr<::arrow::DataType> read_data_type = arrow::struct_(arrow_read_schema->fields()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(array_builder_, + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr array_builder, arrow::MakeBuilder(read_data_type, arrow_pool_.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::avro::DataFileReaderBase> reader, + CreateDataFileReader(input_stream_, pool_)); + + if (reader_) { + reader_->close(); + } + reader_ = std::move(reader); + read_fields_projection_ = std::move(read_fields_projection); + array_builder_ = std::move(array_builder); + previous_first_row_ = std::numeric_limits::max(); + next_row_to_read_ = std::numeric_limits::max(); + close_ = false; return Status::OK(); } diff --git a/src/paimon/format/avro/avro_file_batch_reader_test.cpp b/src/paimon/format/avro/avro_file_batch_reader_test.cpp index a4a26a88..b1849931 100644 --- a/src/paimon/format/avro/avro_file_batch_reader_test.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader_test.cpp @@ -345,6 +345,55 @@ TEST_F(AvroFileBatchReaderTest, TestGetPreviousBatchFirstRowNumber) { ASSERT_TRUE(BatchReader::IsEofBatch(batch5)); } +TEST_F(AvroFileBatchReaderTest, TestSetReadSchemaResetsReaderToFirstRow) { + std::string file_path = PathUtil::JoinPath(dir_->Str(), "file.avro"); + + arrow::FieldVector fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::int32()), + }; + auto file_data_type = arrow::struct_(fields); + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(file_data_type, R"([ + [1, 10], + [2, 20], + [3, 30], + [4, 40] + ])") + .ValueOrDie(); + WriteData(src_array, file_path, /*compression=*/"null"); + + ASSERT_OK_AND_ASSIGN(auto reader_builder, file_format_->CreateReaderBuilder(/*batch_size=*/2)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_path)); + ASSERT_OK_AND_ASSIGN(auto reader, reader_builder->Build(in)); + + ASSERT_OK_AND_ASSIGN(auto first_batch, reader->NextBatch()); + ASSERT_EQ(0, reader->GetPreviousBatchFirstRowNumber().value()); + auto first_array = + arrow::ImportArray(first_batch.first.get(), first_batch.second.get()).ValueOrDie(); + ASSERT_TRUE(first_array->Equals(src_array->Slice(0, 2))) << first_array->ToString(); + + auto read_schema = arrow::schema({arrow::field("f1", arrow::int32())}); + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + ASSERT_OK(reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); + + ASSERT_OK_AND_ASSIGN(auto projected_batch, reader->NextBatch()); + ASSERT_EQ(0, reader->GetPreviousBatchFirstRowNumber().value()); + auto projected_array = + arrow::ImportArray(projected_batch.first.get(), projected_batch.second.get()).ValueOrDie(); + auto expected_projected_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({arrow::field("f1", arrow::int32())}), + R"([ + [10], + [20] + ])") + .ValueOrDie(); + ASSERT_TRUE(projected_array->Equals(expected_projected_array)) << projected_array->ToString(); +} + TEST_F(AvroFileBatchReaderTest, TestGetNumberOfRows) { std::string file_path = PathUtil::JoinPath(dir_->Str(), "file.avro"); From 8fdc2ec3b4ec51c7be14b777b09d6376a6b1af06 Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Mon, 22 Jun 2026 00:23:36 -0700 Subject: [PATCH 068/138] feat(shredding): support shared shredding file reader --- src/paimon/CMakeLists.txt | 2 + .../shredding/map_shared_shredding_utils.cpp | 61 +- .../shredding/map_shared_shredding_utils.h | 30 + .../map_shared_shredding_utils_test.cpp | 124 ++++ .../shared_shredding_file_reader.cpp | 487 +++++++++++++++ .../shredding/shared_shredding_file_reader.h | 87 +++ .../shared_shredding_file_reader_test.cpp | 572 ++++++++++++++++++ 7 files changed, 1352 insertions(+), 11 deletions(-) create mode 100644 src/paimon/common/data/shredding/shared_shredding_file_reader.cpp create mode 100644 src/paimon/common/data/shredding/shared_shredding_file_reader.h create mode 100644 src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 125ef23b..175a766e 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -144,6 +144,7 @@ set(PAIMON_COMMON_SRCS common/data/shredding/map_shared_shredding_context.cpp common/data/shredding/map_shared_shredding_batch_converter.cpp common/data/shredding/map_shared_shredding_column_allocator.cpp + common/data/shredding/shared_shredding_file_reader.cpp common/utils/delta_varint_compressor.cpp common/utils/fields_comparator.cpp common/utils/path_util.cpp @@ -543,6 +544,7 @@ if(PAIMON_BUILD_TESTS) common/data/shredding/map_shared_shredding_column_allocator_test.cpp common/data/shredding/map_shared_shredding_field_dict_test.cpp common/data/shredding/map_shared_shredding_context_test.cpp + common/data/shredding/shared_shredding_file_reader_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp index 15cc549b..8b376a9a 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp @@ -38,8 +38,23 @@ #include "rapidjson/writer.h" namespace paimon { -// ---- Column detection ---- +Result> MapSharedShreddingUtils::GetPhysicalColumnIndices( + const MapSharedShreddingFieldMeta& meta, const std::string& name) { + auto name_iter = meta.name_to_id.find(name); + if (name_iter == meta.name_to_id.end()) { + return Status::Invalid( + fmt::format("cannot find field {} in map shared shredding meta", name)); + } + auto id_iter = meta.field_to_columns.find(name_iter->second); + if (id_iter == meta.field_to_columns.end()) { + return Status::Invalid( + fmt::format("cannot find field id {} in field_to_columns in map shared shredding meta", + name_iter->second)); + } + return id_iter->second; +} +// ---- Column detection ---- bool MapSharedShreddingUtils::IsShreddingKeyMap( const std::shared_ptr& arrow_type) { if (arrow_type->id() != arrow::Type::MAP) { @@ -78,24 +93,38 @@ Result> MapSharedShreddingUtils::Crea } // ---- Schema conversion ---- +std::shared_ptr MapSharedShreddingUtils::BuildSpecificPhysicalStructType( + const std::shared_ptr& value_type, const std::set& physical_col_ids, + bool value_nullable, bool include_overflow) { + std::vector sorted_cols(physical_col_ids.begin(), physical_col_ids.end()); + return InnerBuildSpecificPhysicalStructType(value_type, sorted_cols, value_nullable, + include_overflow); +} std::shared_ptr MapSharedShreddingUtils::BuildPhysicalStructType( const std::shared_ptr& value_type, int32_t num_columns, bool value_nullable) { - arrow::FieldVector struct_fields; - struct_fields.reserve(num_columns + 2); + std::vector sorted_cols(num_columns); + std::iota(sorted_cols.begin(), sorted_cols.end(), 0); + return InnerBuildSpecificPhysicalStructType(value_type, sorted_cols, value_nullable, + /*include_overflow=*/true); +} +std::shared_ptr MapSharedShreddingUtils::InnerBuildSpecificPhysicalStructType( + const std::shared_ptr& value_type, const std::vector& sorted_cols, + bool value_nullable, bool include_overflow) { + arrow::FieldVector struct_fields; + struct_fields.reserve(sorted_cols.size() + 2); struct_fields.push_back( arrow::field(MapSharedShreddingDefine::kFieldMapping, arrow::list(arrow::int32()), true)); - - for (int32_t i = 0; i < num_columns; ++i) { - struct_fields.push_back(arrow::field(MapSharedShreddingDefine::PhysicalColumnName(i), + for (const auto& col : sorted_cols) { + struct_fields.push_back(arrow::field(MapSharedShreddingDefine::PhysicalColumnName(col), value_type, value_nullable)); } - - struct_fields.push_back(arrow::field( - MapSharedShreddingDefine::kOverflow, - arrow::map(arrow::int32(), arrow::field("value", value_type, value_nullable)), true)); - + if (include_overflow) { + struct_fields.push_back(arrow::field( + MapSharedShreddingDefine::kOverflow, + arrow::map(arrow::int32(), arrow::field("value", value_type, value_nullable)), true)); + } return arrow::struct_(std::move(struct_fields)); } @@ -403,6 +432,16 @@ bool MapSharedShreddingUtils::HasShreddingMetadata( return metadata->value(index) == MapShreddingDefine::kStorageLayoutSharedShredding; } +Result MapSharedShreddingUtils::IsOverflowField(const MapSharedShreddingFieldMeta& meta, + const std::string& name) { + auto name_iter = meta.name_to_id.find(name); + if (name_iter == meta.name_to_id.end()) { + return Status::Invalid( + fmt::format("cannot find field {} in map shared shredding meta", name)); + } + return meta.overflow_field_set.count(name_iter->second) > 0; +} + std::function>()> MapSharedShreddingUtils::BuildMetadataFinalizer( const std::shared_ptr& converter, diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.h b/src/paimon/common/data/shredding/map_shared_shredding_utils.h index fd5d76d4..a017c8dd 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -48,6 +49,13 @@ class MapSharedShreddingUtils { MapSharedShreddingUtils() = delete; ~MapSharedShreddingUtils() = delete; + /// Returns the physical column indices for the given field name from the shredding meta. + /// @param meta The shredding field meta parsed from file footer. + /// @param name The field name to look up. + /// @return Vector of physical column indices assigned to this field, + /// or Status::Invalid if the field name or field id is not found. + static Result> GetPhysicalColumnIndices( + const MapSharedShreddingFieldMeta& meta, const std::string& name); // ---- Column detection ---- /// Checks whether a given arrow field is MAP (the type prerequisite for shredding). @@ -83,6 +91,15 @@ class MapSharedShreddingUtils { const std::shared_ptr& logical_schema, const std::map& field_to_num_columns); + /// Builds the physical Arrow type for one shredding MAP column with physical_col_ids. + /// @param value_type The value type of the original MAP. + /// @param physical_col_ids The set of physical column ids to include. + /// @param value_nullable Whether the MAP's value field is nullable. + /// @param include_overflow Whether to include __overflow column. + static std::shared_ptr BuildSpecificPhysicalStructType( + const std::shared_ptr& value_type, + const std::set& physical_col_ids, bool value_nullable, bool include_overflow); + /// Builds field_to_num_columns map from DetectShreddingColumns result and CoreOptions. /// @param shredding_field_names Field names returned by DetectShreddingColumns. /// @param options CoreOptions containing per-column max-columns config. @@ -110,6 +127,10 @@ class MapSharedShreddingUtils { /// Checks whether a KeyValueMetadata contains shredding MAP metadata. static bool HasShreddingMetadata(const std::shared_ptr& metadata); + /// Checks whether a field in MapSharedShreddingFieldMeta is a overflow field. + static Result IsOverflowField(const MapSharedShreddingFieldMeta& meta, + const std::string& name); + // ---- Writer helpers ---- /// Builds a MetadataFinalizer that serializes shredding metadata into per-field @@ -134,6 +155,15 @@ class MapSharedShreddingUtils { static std::shared_ptr BuildPhysicalStructType( const std::shared_ptr& value_type, int32_t num_columns, bool value_nullable); + + /// Builds the physical Arrow type for one shredding MAP column with sorted_cols. + /// @param value_type The value type of the original MAP. + /// @param sorted_cols The vector of physical column ids to include. + /// @param value_nullable Whether the MAP's value field is nullable. + /// @param include_overflow Whether to include __overflow column. + static std::shared_ptr InnerBuildSpecificPhysicalStructType( + const std::shared_ptr& value_type, const std::vector& sorted_cols, + bool value_nullable, bool include_overflow); }; } // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp index 9eb65879..99894721 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp @@ -19,6 +19,8 @@ #include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include + #include "arrow/type.h" #include "arrow/util/key_value_metadata.h" #include "gtest/gtest.h" @@ -165,6 +167,38 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNoShreddingColumns) { ASSERT_TRUE(physical_schema->Equals(schema)); } +TEST(MapSharedShreddingUtilsTest, BuildSpecificPhysicalStructTypeWithOverflow) { + auto actual = MapSharedShreddingUtils::BuildSpecificPhysicalStructType( + arrow::int64(), /*physical_col_ids=*/{3, 1}, /*value_nullable=*/false, + /*include_overflow=*/true); + + auto expected = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_1", arrow::int64(), false), + arrow::field("__col_3", arrow::int64(), false), + arrow::field("__overflow", + arrow::map(arrow::int32(), arrow::field("value", arrow::int64(), false)), + true), + }); + ASSERT_TRUE(actual->Equals(*expected)) << "Expected:\n" + << expected->ToString() << "\nActual:\n" + << actual->ToString(); +} + +TEST(MapSharedShreddingUtilsTest, BuildSpecificPhysicalStructTypeWithoutOverflow) { + auto actual = MapSharedShreddingUtils::BuildSpecificPhysicalStructType( + arrow::utf8(), /*physical_col_ids=*/{3}, /*value_nullable=*/true, + /*include_overflow=*/false); + + auto expected = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_3", arrow::utf8(), true), + }); + ASSERT_TRUE(actual->Equals(*expected)) << "Expected:\n" + << expected->ToString() << "\nActual:\n" + << actual->ToString(); +} + // ---- BuildColumnToNumColumns ---- TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumns) { @@ -365,4 +399,94 @@ TEST(MapSharedShreddingUtilsTest, PhysicalColumnName) { ASSERT_EQ(MapSharedShreddingDefine::PhysicalColumnName(99), "__col_99"); } +// ---- GetPhysicalColumnIndices ---- + +// Normal: single physical column per field +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesSingleColumn) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"age", 0}, {"name", 1}}; + meta.field_to_columns = {{0, {2}}, {1, {5}}}; + + ASSERT_OK_AND_ASSIGN(auto cols_age, + MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "age")); + ASSERT_EQ(cols_age, (std::vector{2})); + + ASSERT_OK_AND_ASSIGN(auto cols_name, + MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "name")); + ASSERT_EQ(cols_name, (std::vector{5})); +} + +// Normal: multiple physical columns for one field +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesMultipleColumns) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"tags", 0}}; + meta.field_to_columns = {{0, {0, 3, 7}}}; + + ASSERT_OK_AND_ASSIGN(auto cols, + MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "tags")); + ASSERT_EQ(cols, (std::vector{0, 3, 7})); +} + +// Normal: many fields each mapping to different physical columns +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesMultipleFields) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + meta.field_to_columns = {{0, {0, 1}}, {1, {2, 3, 4}}, {2, {5}}}; + + ASSERT_OK_AND_ASSIGN(auto cols_a, MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "a")); + ASSERT_EQ(cols_a, (std::vector{0, 1})); + + ASSERT_OK_AND_ASSIGN(auto cols_b, MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "b")); + ASSERT_EQ(cols_b, (std::vector{2, 3, 4})); + + ASSERT_OK_AND_ASSIGN(auto cols_c, MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "c")); + ASSERT_EQ(cols_c, (std::vector{5})); +} + +// Error: field name not found in name_to_id +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesFieldNotFound) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"age", 0}}; + meta.field_to_columns = {{0, {1}}}; + + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "nonexistent"), + "cannot find field nonexistent in map shared shredding meta"); +} + +// Error: field name not found in empty meta +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesEmptyMeta) { + MapSharedShreddingFieldMeta meta; + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "any"), + "cannot find field any in map shared shredding meta"); +} + +// Error: field id exists in name_to_id but is missing from field_to_columns +TEST(MapSharedShreddingUtilsTest, GetPhysicalColumnIndicesFieldIdMissingInFieldToColumns) { + MapSharedShreddingFieldMeta meta; + // "score" -> field_id 42, but field_to_columns has no entry for 42 + meta.name_to_id = {{"score", 42}}; + meta.field_to_columns = {}; + + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::GetPhysicalColumnIndices(meta, "score"), + "cannot find field id 42 in field_to_columns in map shared shredding meta"); +} + +TEST(MapSharedShreddingUtilsTest, IsOverflowField) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + meta.overflow_field_set = {0, 2}; + + ASSERT_OK_AND_ASSIGN(bool a_overflow, MapSharedShreddingUtils::IsOverflowField(meta, "a")); + ASSERT_TRUE(a_overflow); + + ASSERT_OK_AND_ASSIGN(bool b_overflow, MapSharedShreddingUtils::IsOverflowField(meta, "b")); + ASSERT_FALSE(b_overflow); + + ASSERT_OK_AND_ASSIGN(bool c_overflow, MapSharedShreddingUtils::IsOverflowField(meta, "c")); + ASSERT_TRUE(c_overflow); + + ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::IsOverflowField(meta, "missing"), + "cannot find field missing in map shared shredding meta"); +} + } // namespace paimon::test diff --git a/src/paimon/common/data/shredding/shared_shredding_file_reader.cpp b/src/paimon/common/data/shredding/shared_shredding_file_reader.cpp new file mode 100644 index 00000000..5e80ad27 --- /dev/null +++ b/src/paimon/common/data/shredding/shared_shredding_file_reader.cpp @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/shared_shredding_file_reader.h" + +#include +#include +#include +#include + +#include "arrow/c/bridge.h" +#include "arrow/util/key_value_metadata.h" +#include "fmt/format.h" +#include "paimon/common/reader/reader_utils.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/casting/casting_utils.h" + +namespace paimon { +namespace { +// TODO(lisizhuo.lsz): rm kSelectedKeysMetadataKey after zhanyu pr +constexpr const char* kSelectedKeysMetadataKey = "paimon.map.selected-keys"; + +Result>> GetSelectedKeys( + const std::shared_ptr& field) { + if (!field->HasMetadata() || !field->metadata()) { + return std::optional>(); + } + int32_t index = field->metadata()->FindKey(kSelectedKeysMetadataKey); + if (index < 0) { + return std::optional>(); + } + std::string selected_keys = field->metadata()->value(index); + // paimon will not ignore empty for '' is a valid key + return std::optional>( + StringUtils::Split(selected_keys, ",", /*ignore_empty=*/false)); +} + +} // namespace + +Result> SharedShreddingFileReader::Create( + std::unique_ptr&& reader, const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, reader->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_arrow_schema, + arrow::ImportSchema(file_schema.get())); + + std::map shared_shredding_name_to_meta; + for (const auto& field : file_arrow_schema->fields()) { + std::shared_ptr metadata = + std::const_pointer_cast(field->metadata()); + if (MapSharedShreddingUtils::HasShreddingMetadata(metadata)) { + PAIMON_ASSIGN_OR_RAISE( + MapSharedShreddingFieldMeta meta, + MapSharedShreddingUtils::DeserializeMetadata( + metadata, MapSharedShreddingDefine::kDefaultDictCompression)); + shared_shredding_name_to_meta[field->name()] = std::move(meta); + } + } + return std::unique_ptr( + new SharedShreddingFileReader(std::move(reader), shared_shredding_name_to_meta, pool)); +} + +SharedShreddingFileReader::SharedShreddingFileReader( + std::unique_ptr&& reader, + const std::map& shared_shredding_name_to_meta, + const std::shared_ptr& pool) + : arrow_pool_(GetArrowPool(pool)), + reader_(std::move(reader)), + shared_shredding_name_to_meta_(shared_shredding_name_to_meta) {} + +Result> SharedShreddingFileReader::GetFileSchema() const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> physical_schema, + reader_->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr physical_arrow_schema, + arrow::ImportSchema(physical_schema.get())); + + arrow::FieldVector logical_fields = physical_arrow_schema->fields(); + for (int32_t i = 0; i < physical_arrow_schema->num_fields(); ++i) { + const auto& field = physical_arrow_schema->field(i); + std::shared_ptr metadata = + std::const_pointer_cast(field->metadata()); + if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata)) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(logical_fields[i], ToLogicalMapField(field)); + } + + auto logical_schema = arrow::schema(std::move(logical_fields)); + auto c_logical_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*logical_schema, c_logical_schema.get())); + return c_logical_schema; +} + +Result> SharedShreddingFileReader::ToLogicalMapField( + const std::shared_ptr& physical_field) { + auto physical_type = std::dynamic_pointer_cast(physical_field->type()); + if (!physical_type) { + return Status::Invalid(fmt::format("shared-shredding field {} is not a physical struct", + physical_field->name())); + } + + std::shared_ptr value_type; + bool value_nullable = true; + for (const auto& child : physical_type->fields()) { + if (child->name() == MapSharedShreddingDefine::kFieldMapping || + child->name() == MapSharedShreddingDefine::kOverflow) { + continue; + } + value_type = child->type(); + value_nullable = child->nullable(); + break; + } + if (!value_type) { + return Status::Invalid(fmt::format("cannot infer shared-shredding value type for field {}", + physical_field->name())); + } + return arrow::field( + physical_field->name(), + arrow::map(arrow::utf8(), arrow::field("value", value_type, value_nullable)), + physical_field->nullable()); +} + +Status SharedShreddingFileReader::SetReadSchema( + ::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) { + if (!read_schema) { + return Status::Invalid("invalid read schema in SharedShreddingFileReader, cannot be null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_read_schema, + arrow::ImportSchema(read_schema)); + std::vector shared_shredding_names; + for (const auto& field : logical_read_schema->fields()) { + if (shared_shredding_name_to_meta_.find(field->name()) != + shared_shredding_name_to_meta_.end()) { + shared_shredding_names.push_back(field->name()); + } + } + if (shared_shredding_names.empty()) { + // suppose not fall into SharedShreddingFileReader + return Status::Invalid("do not exist shared shredding columns in read schema"); + } + shared_shredding_name_to_selected_keys_.clear(); + shared_shredding_name_to_map_type_.clear(); + arrow::FieldVector resolved_fields = logical_read_schema->fields(); + for (const auto& name : shared_shredding_names) { + const auto& field = logical_read_schema->GetFieldByName(name); + if (!field) { + return Status::Invalid( + fmt::format("cannot find shared-shredding field in read schema")); + } + auto meta_iter = shared_shredding_name_to_meta_.find(field->name()); + if (meta_iter == shared_shredding_name_to_meta_.end()) { + return Status::Invalid( + fmt::format("cannot find shared-shredding metadata for field {}", field->name())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional> selected_keys_opt, + GetSelectedKeys(field)); + std::vector selected_keys; + if (!selected_keys_opt) { + // select all keys + selected_keys.reserve(meta_iter->second.name_to_id.size()); + for (const auto& [key_name, _] : meta_iter->second.name_to_id) { + selected_keys.push_back(key_name); + } + } else { + std::set seen_keys; + for (const auto& selected_key : selected_keys_opt.value()) { + if (!seen_keys.insert(selected_key).second) { + return Status::Invalid( + fmt::format("duplicate key [{}] in paimon.map.selected-keys for field {}", + selected_key, field->name())); + } + selected_keys.push_back(selected_key); + } + } + shared_shredding_name_to_selected_keys_[field->name()] = selected_keys; + + auto map_type = arrow::internal::checked_pointer_cast(field->type()); + shared_shredding_name_to_map_type_[field->name()] = map_type; + std::set selected_physical_column_ids; + bool include_overflow = false; + for (const auto& selected_key : selected_keys) { + auto name_iter = meta_iter->second.name_to_id.find(selected_key); + if (name_iter == meta_iter->second.name_to_id.end()) { + continue; + } + auto column_iter = meta_iter->second.field_to_columns.find(name_iter->second); + if (column_iter == meta_iter->second.field_to_columns.end()) { + continue; + } + const std::vector& physical_column_ids = column_iter->second; + selected_physical_column_ids.insert(physical_column_ids.begin(), + physical_column_ids.end()); + PAIMON_ASSIGN_OR_RAISE(bool is_overflow_field, MapSharedShreddingUtils::IsOverflowField( + meta_iter->second, selected_key)); + include_overflow = include_overflow || is_overflow_field; + } + std::shared_ptr resolved_type = + MapSharedShreddingUtils::BuildSpecificPhysicalStructType( + map_type->item_type(), selected_physical_column_ids, + map_type->item_field()->nullable(), include_overflow); + resolved_fields[logical_read_schema->GetFieldIndex(name)] = + arrow::field(field->name(), resolved_type, field->nullable()); + } + + auto resolved_schema = arrow::schema(std::move(resolved_fields)); + std::unique_ptr c_resolved_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*resolved_schema, c_resolved_schema.get())); + return reader_->SetReadSchema(c_resolved_schema.get(), predicate, selection_bitmap); +} + +Result SharedShreddingFileReader::NextBatch() { + return Status::Invalid( + "paimon inner reader SharedShreddingFileReader should use NextBatchWithBitmap"); +} + +Result SharedShreddingFileReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, + reader_->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + return batch_with_bitmap; + } + + auto& [batch, bitmap] = batch_with_bitmap; + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + auto struct_array = std::dynamic_pointer_cast(arrow_array); + if (!struct_array) { + return Status::Invalid("cannot cast batch to StructArray in SharedShreddingFileReader"); + } + + arrow::ArrayVector resolved_arrays = struct_array->fields(); + arrow::FieldVector resolved_fields = struct_array->struct_type()->fields(); + for (int32_t field_idx = 0; field_idx < struct_array->num_fields(); ++field_idx) { + const auto& physical_field = struct_array->struct_type()->field(field_idx); + auto iter = shared_shredding_name_to_selected_keys_.find(physical_field->name()); + if (iter == shared_shredding_name_to_selected_keys_.end()) { + continue; + } + auto physical_struct_array = + std::dynamic_pointer_cast(struct_array->field(field_idx)); + if (!physical_struct_array) { + return Status::Invalid(fmt::format( + "cannot cast physical shredding field {} to StructArray", physical_field->name())); + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr logical_map_array, + RebuildLogicalMapArray(physical_field, physical_struct_array)); + resolved_arrays[field_idx] = logical_map_array; + resolved_fields[field_idx] = arrow::field(physical_field->name(), logical_map_array->type(), + physical_field->nullable()); + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr new_struct_array, + arrow::StructArray::Make(resolved_arrays, resolved_fields)); + auto new_c_array = std::make_unique(); + auto new_c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*new_struct_array, new_c_array.get(), new_c_schema.get())); + batch = std::make_pair(std::move(new_c_array), std::move(new_c_schema)); + return batch_with_bitmap; +} + +Result> SharedShreddingFileReader::RebuildLogicalMapArray( + const std::shared_ptr& physical_field, + const std::shared_ptr& physical_struct_array) const { + std::string shredding_field_name = physical_field->name(); + auto meta_iter = shared_shredding_name_to_meta_.find(shredding_field_name); + if (meta_iter == shared_shredding_name_to_meta_.end()) { + return Status::Invalid(fmt::format("cannot find shared-shredding metadata for field {}", + shredding_field_name)); + } + auto selected_iter = shared_shredding_name_to_selected_keys_.find(shredding_field_name); + if (selected_iter == shared_shredding_name_to_selected_keys_.end()) { + return Status::Invalid( + fmt::format("cannot find selected keys for field {}", shredding_field_name)); + } + auto map_type_iter = shared_shredding_name_to_map_type_.find(shredding_field_name); + if (map_type_iter == shared_shredding_name_to_map_type_.end()) { + return Status::Invalid( + fmt::format("cannot find logical map type for field {}", shredding_field_name)); + } + const MapSharedShreddingFieldMeta& meta = meta_iter->second; + const std::vector& selected_keys = selected_iter->second; + const auto& map_type = map_type_iter->second; + + auto field_mapping_array = std::dynamic_pointer_cast( + physical_struct_array->GetFieldByName(MapSharedShreddingDefine::kFieldMapping)); + if (!field_mapping_array) { + return Status::Invalid( + fmt::format("cannot find __field_mapping for field {}", shredding_field_name)); + } + auto field_mapping_values = + std::dynamic_pointer_cast(field_mapping_array->values()); + if (!field_mapping_values) { + return Status::Invalid("__field_mapping values is not an Int32Array"); + } + + auto selected_key_ids = ResolveSelectedKeyIds(meta, selected_keys); + + std::map> physical_column_name_to_array; + std::shared_ptr overflow_array; + CollectPhysicalColumns(physical_struct_array, &physical_column_name_to_array, &overflow_array); + for (auto& [_, physical_column_array] : physical_column_name_to_array) { + if (physical_column_array->type_id() == arrow::Type::DICTIONARY) { + PAIMON_ASSIGN_OR_RAISE( + physical_column_array, + CastingUtils::Cast(physical_column_array, map_type->item_type(), + arrow::compute::CastOptions::Safe(), arrow_pool_.get())); + } + } + + std::shared_ptr overflow_keys; + std::shared_ptr overflow_items; + if (overflow_array) { + overflow_keys = + arrow::internal::checked_pointer_cast(overflow_array->keys()); + overflow_items = overflow_array->items(); + if (!overflow_keys || !overflow_items) { + return Status::Invalid("__overflow map has invalid key or item array"); + } + if (overflow_items->type_id() == arrow::Type::DICTIONARY) { + PAIMON_ASSIGN_OR_RAISE( + overflow_items, + CastingUtils::Cast(overflow_items, map_type->item_type(), + arrow::compute::CastOptions::Safe(), arrow_pool_.get())); + } + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr map_builder_base, + arrow::MakeBuilder(map_type, arrow_pool_.get())); + auto* map_builder = dynamic_cast(map_builder_base.get()); + if (!map_builder) { + return Status::Invalid( + fmt::format("cannot create MapBuilder for field {}", shredding_field_name)); + } + auto* key_builder = dynamic_cast(map_builder->key_builder()); + if (!key_builder) { + return Status::Invalid(fmt::format("map key builder is not a StringBuilder for field {}", + shredding_field_name)); + } + arrow::ArrayBuilder* item_builder = map_builder->item_builder(); + if (!item_builder) { + return Status::Invalid( + fmt::format("map item builder is null for field {}", shredding_field_name)); + } + + int64_t row_count = physical_struct_array->length(); + int64_t max_item_count = row_count * static_cast(selected_key_ids.size()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->Reserve(row_count)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Reserve(max_item_count)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(item_builder->Reserve(max_item_count)); + + for (int64_t row = 0; row < row_count; ++row) { + if (physical_struct_array->IsNull(row)) { + // null struct -> null map + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->AppendNull()); + continue; + } + if (field_mapping_array->IsNull(row)) { + return Status::Invalid(fmt::format( + "__field_mapping cannot be null in non-null shared-shredding row for field {}", + shredding_field_name)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->Append()); + int32_t mapping_offset = field_mapping_array->value_offset(row); + int32_t mapping_length = field_mapping_array->value_length(row); + // follow the sequence in paimon.map.selected-keys + for (const auto& [selected_key, selected_field_id] : selected_key_ids) { + bool found = false; + for (int32_t pos = 0; pos < mapping_length; ++pos) { + int32_t mapping_index = mapping_offset + pos; + if (field_mapping_values->IsNull(mapping_index)) { + return Status::Invalid("__field_mapping element cannot be null"); + } + if (field_mapping_values->Value(mapping_index) != selected_field_id) { + continue; + } + std::string physical_column_name = + MapSharedShreddingDefine::PhysicalColumnName(pos); + auto physical_column_iter = + physical_column_name_to_array.find(physical_column_name); + if (physical_column_iter == physical_column_name_to_array.end()) { + return Status::Invalid( + fmt::format("cannot find selected physical column {} for field {}", + physical_column_name, shredding_field_name)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Append(selected_key)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + item_builder->AppendArraySlice(*physical_column_iter->second->data(), row, 1)); + found = true; + break; + } + if (found || !overflow_array) { + continue; + } + int32_t overflow_offset = overflow_array->value_offset(row); + int32_t overflow_length = overflow_array->value_length(row); + for (int32_t pos = 0; pos < overflow_length; ++pos) { + int32_t overflow_index = overflow_offset + pos; + if (!overflow_keys->IsNull(overflow_index) && + overflow_keys->Value(overflow_index) == selected_field_id) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Append(selected_key)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + item_builder->AppendArraySlice(*overflow_items->data(), overflow_index, 1)); + break; + } + } + } + } + std::shared_ptr map_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->Finish(&map_array)); + return map_array; +} + +std::vector> SharedShreddingFileReader::ResolveSelectedKeyIds( + const MapSharedShreddingFieldMeta& meta, const std::vector& selected_keys) { + std::vector> selected_key_ids; + selected_key_ids.reserve(selected_keys.size()); + for (const auto& selected_key : selected_keys) { + auto id_iter = meta.name_to_id.find(selected_key); + if (id_iter == meta.name_to_id.end()) { + continue; + } + selected_key_ids.emplace_back(selected_key, id_iter->second); + } + return selected_key_ids; +} + +void SharedShreddingFileReader::CollectPhysicalColumns( + const std::shared_ptr& physical_struct_array, + std::map>* physical_column_name_to_array, + std::shared_ptr* overflow_array) { + const auto& struct_type = physical_struct_array->struct_type(); + for (int32_t i = 0; i < struct_type->num_fields(); ++i) { + const auto& sub_field = struct_type->field(i); + if (sub_field->name() == MapSharedShreddingDefine::kFieldMapping) { + continue; + } + if (sub_field->name() == MapSharedShreddingDefine::kOverflow) { + *overflow_array = arrow::internal::checked_pointer_cast( + physical_struct_array->field(i)); + continue; + } + (*physical_column_name_to_array)[sub_field->name()] = physical_struct_array->field(i); + } +} + +std::shared_ptr SharedShreddingFileReader::GetReaderMetrics() const { + return reader_->GetReaderMetrics(); +} + +void SharedShreddingFileReader::Close() { + reader_->Close(); +} + +Result SharedShreddingFileReader::GetPreviousBatchFirstRowNumber() const { + return reader_->GetPreviousBatchFirstRowNumber(); +} + +Result SharedShreddingFileReader::GetNumberOfRows() const { + return reader_->GetNumberOfRows(); +} + +bool SharedShreddingFileReader::SupportPreciseBitmapSelection() const { + return reader_->SupportPreciseBitmapSelection(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shared_shredding_file_reader.h b/src/paimon/common/data/shredding/shared_shredding_file_reader.h new file mode 100644 index 00000000..7826e269 --- /dev/null +++ b/src/paimon/common/data/shredding/shared_shredding_file_reader.h @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/file_batch_reader.h" + +namespace paimon { + +class SharedShreddingFileReader : public FileBatchReader { + public: + static Result> Create( + std::unique_ptr&& reader, const std::shared_ptr& pool); + + Result> GetFileSchema() const override; + + Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) override; + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + std::shared_ptr GetReaderMetrics() const override; + + void Close() override; + + Result GetPreviousBatchFirstRowNumber() const override; + + Result GetNumberOfRows() const override; + + bool SupportPreciseBitmapSelection() const override; + + private: + SharedShreddingFileReader( + std::unique_ptr&& reader, + const std::map& shared_shredding_name_to_meta, + const std::shared_ptr& pool); + + Result> RebuildLogicalMapArray( + const std::shared_ptr& physical_field, + const std::shared_ptr& physical_struct_array) const; + + static std::vector> ResolveSelectedKeyIds( + const MapSharedShreddingFieldMeta& meta, const std::vector& selected_keys); + + static void CollectPhysicalColumns( + const std::shared_ptr& physical_struct_array, + std::map>* physical_column_name_to_array, + std::shared_ptr* overflow_array); + + static Result> ToLogicalMapField( + const std::shared_ptr& physical_field); + + private: + std::shared_ptr arrow_pool_; + std::unique_ptr reader_; + std::map shared_shredding_name_to_meta_; + std::map> shared_shredding_name_to_selected_keys_; + std::map> shared_shredding_name_to_map_type_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp b/src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp new file mode 100644 index 00000000..8851e337 --- /dev/null +++ b/src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp @@ -0,0 +1,572 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/shared_shredding_file_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "arrow/util/key_value_metadata.h" +#include "gtest/gtest.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/common/fs/external_path_provider.h" +#include "paimon/core/append/append_only_writer.h" +#include "paimon/core/compact/noop_compact_manager.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/commit_increment.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/record_batch.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/mock/mock_file_format_factory.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +class SharedShreddingFileReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + } + + static MapSharedShreddingFieldMeta TagsMeta() { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}, {"e", 4}}; + meta.field_to_columns = {{0, {0, 1}}, {1, {1}}, {2, {0}}, {3, {0}}, {4, {1}}}; + meta.overflow_field_set = {0, 2}; + meta.num_columns = 2; + meta.max_row_width = 4; + return meta; + } + + std::shared_ptr PhysicalSchemaWithMetadata() const { + return PhysicalSchemaWithMetadata(TagsMeta()); + } + + std::shared_ptr PhysicalSchemaWithMetadata( + const MapSharedShreddingFieldMeta& meta) const { + std::map field_to_num_columns = {{"tags", 2}}; + EXPECT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema_, field_to_num_columns)); + auto metadata = std::make_shared(); + EXPECT_OK(MapSharedShreddingUtils::SerializeMetadata( + meta, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get())); + + arrow::FieldVector fields = physical_schema->fields(); + fields[1] = fields[1]->WithMetadata(metadata); + return arrow::schema(std::move(fields)); + } + + std::shared_ptr PhysicalArray() const { + std::shared_ptr physical_schema = PhysicalSchemaWithMetadata(); + std::string json = R"([ + [1, [[0, 1], 10, 20, null]], + [2, [[2, 0], 30, 40, null]], + [3, null], + [4, [[3, 4], 60, 70, [[0, 80], [2, null]]]] + ])"; + return arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), + json) + .ValueOrDie(); + } + + std::unique_ptr CreateReader( + std::shared_ptr physical_array = nullptr, + std::shared_ptr physical_schema = nullptr) const { + if (!physical_schema) { + physical_schema = PhysicalSchemaWithMetadata(); + } + if (!physical_array) { + physical_array = PhysicalArray(); + } + auto mock_reader = std::make_unique( + physical_array, arrow::struct_(physical_schema->fields()), /*read_batch_size=*/10); + mock_reader->EnableRandomizeBatchSize(false); + EXPECT_OK_AND_ASSIGN(auto shared_shredding_reader, + SharedShreddingFileReader::Create(std::move(mock_reader), pool_)); + return shared_shredding_reader; + } + + std::shared_ptr ReadSchema( + const std::optional& selected_keys) const { + arrow::FieldVector fields = logical_schema_->fields(); + if (selected_keys) { + auto metadata = std::make_shared(); + metadata->Append("paimon.map.selected-keys", *selected_keys); + fields[1] = fields[1]->WithMetadata(metadata); + } + return arrow::schema(std::move(fields)); + } + + std::unique_ptr ExportSchema(const std::shared_ptr& schema) const { + auto c_schema = std::make_unique(); + EXPECT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + return c_schema; + } + + std::unique_ptr CreateBatch(const std::shared_ptr& schema, + const std::string& json) const { + auto array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(schema->fields()), json) + .ValueOrDie(); + ::ArrowArray arrow_array; + EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + EXPECT_OK_AND_ASSIGN(auto batch, batch_builder.Finish()); + return batch; + } + + void AssertChunkedArrayEquals(const std::shared_ptr& expected, + const std::shared_ptr& actual) const { + ASSERT_TRUE(expected->Equals(actual)) << "Expected:\n" + << expected->ToString() << "\nActual:\n" + << actual->ToString(); + } + + std::shared_ptr CreatePathFactory(const std::string& dir, + const std::string& format, + const CoreOptions& options) const { + auto path_factory = std::make_shared(); + EXPECT_OK(path_factory->Init(dir, format, options.DataFilePrefix(), nullptr)); + return path_factory; + } + + std::unique_ptr OpenFormatReader( + const std::string& file_path, const std::string& format, + const std::map& options = {}) const { + auto fs = std::make_shared(); + EXPECT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(file_path)); + EXPECT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get(format, options)); + EXPECT_OK_AND_ASSIGN(auto reader_builder, + file_format->CreateReaderBuilder(/*batch_size=*/10)); + return reader_builder->Build(input_stream).value(); + } + + private: + std::shared_ptr pool_; + std::shared_ptr logical_schema_ = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + std::map options_ = { + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "mock_format"}, + {Options::MANIFEST_FORMAT, "mock_format"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + {Options::WRITE_ONLY, "true"}, + }; +}; + +TEST_F(SharedShreddingFileReaderTest, TestGetFileSchemaReturnsLogicalMapSchema) { + auto reader = CreateReader(); + + ASSERT_OK_AND_ASSIGN(auto c_schema, reader->GetFileSchema()); + auto schema = arrow::ImportSchema(c_schema.get()).ValueOrDie(); + + ASSERT_TRUE(schema->Equals(logical_schema_, /*check_metadata=*/false)) + << "Expected:\n" + << logical_schema_->ToString() << "\nActual:\n" + << schema->ToString(); + ASSERT_FALSE(schema->field(1)->HasMetadata()); +} + +TEST_F(SharedShreddingFileReaderTest, TestAllExistSelectedKeysWithoutOverflow) { + auto reader = CreateReader(); + auto read_schema = ExportSchema(ReadSchema("b")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["b", 20]]], + [2, []], + [3, null], + [4, []] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(SharedShreddingFileReaderTest, TestAllExistSelectedKeysWithOverflow) { + auto reader = CreateReader(); + auto read_schema = ExportSchema(ReadSchema("a,c")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["a", 10]]], + [2, [["a", 40], ["c", 30]]], + [3, null], + [4, [["a", 80], ["c", null]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(SharedShreddingFileReaderTest, TestPartialExistSelectedKeys) { + auto reader = CreateReader(); + auto read_schema = ExportSchema(ReadSchema("a,c,missing")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["a", 10]]], + [2, [["a", 40], ["c", 30]]], + [3, null], + [4, [["a", 80], ["c", null]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(SharedShreddingFileReaderTest, TestDuplicatedSelectedKeys) { + auto reader = CreateReader(); + auto read_schema = ExportSchema(ReadSchema("a,c,a")); + ASSERT_NOK_WITH_MSG(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + "duplicate key [a] in paimon.map.selected-keys for field tags"); +} + +TEST_F(SharedShreddingFileReaderTest, TestMissingSelectedKeysReadsWholeMap) { + auto reader = CreateReader(); + auto read_schema = ExportSchema(ReadSchema(std::nullopt)); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["a", 40], ["c", 30]]], + [3, null], + [4, [["a", 80], ["c", null], ["d", 60], ["e", 70]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(SharedShreddingFileReaderTest, TestSpecialSelectedKeys) { + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"", 0}, {" ", 1}, {".", 2}, {"a", 3}}; + meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}, {3, {1}}}; + meta.num_columns = 2; + meta.max_row_width = 2; + auto physical_schema = PhysicalSchemaWithMetadata(meta); + std::string json = R"([ + [1, [[0, 1], 10, 20, null]], + [2, [[2, 3], 30, 40, null]], + [3, null] + ])"; + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) + .ValueOrDie(); + + auto assert_read = [&](const std::string& selected_keys, const std::string& expected_json) { + auto reader = CreateReader(physical_array, physical_schema); + auto read_schema = ExportSchema(ReadSchema(selected_keys)); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {expected_json}, &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); + }; + + assert_read(" ", R"([ + [1, [[" ", 20]]], + [2, []], + [3, null] + ])"); + assert_read(".", R"([ + [1, []], + [2, [[".", 30]]], + [3, null] + ])"); + assert_read("a,", R"([ + [1, [["", 10]]], + [2, [["a", 40]]], + [3, null] + ])"); + assert_read("", R"([ + [1, [["", 10]]], + [2, []], + [3, null] + ])"); +} + +TEST_F(SharedShreddingFileReaderTest, TestSpecialSelectedKeysWithDuplicatedEmptyKey) { + for (const auto& selected_keys : {",", ",,"}) { + auto reader = CreateReader(); + auto read_schema = ExportSchema(ReadSchema(selected_keys)); + ASSERT_NOK_WITH_MSG(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + "duplicate key [] in paimon.map.selected-keys for field tags"); + } +} + +TEST_F(SharedShreddingFileReaderTest, TestUnknownSelectedKeyReturnsEmptyMap) { + auto reader = CreateReader(); + auto read_schema = ExportSchema(ReadSchema("missing")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, []], + [2, []], + [3, null], + [4, []] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(SharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) { + auto physical_schema = PhysicalSchemaWithMetadata(); + std::string json = R"([ + [1, [null, 10, null, null]] + ])"; + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) + .ValueOrDie(); + auto reader = CreateReader(physical_array, physical_schema); + auto read_schema = ExportSchema(ReadSchema("a")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(ReadResultCollector::CollectResult(reader.get()), + "__field_mapping cannot be null"); +} + +TEST_F(SharedShreddingFileReaderTest, TestInvalidNullFieldMappingFieldElement) { + auto physical_schema = PhysicalSchemaWithMetadata(); + std::string json = R"([ + [1, [[0, null], 10, null, null]] + ])"; + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) + .ValueOrDie(); + auto reader = CreateReader(physical_array, physical_schema); + auto read_schema = ExportSchema(ReadSchema("b")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(ReadResultCollector::CollectResult(reader.get()), + "__field_mapping element cannot be null"); +} + +TEST_F(SharedShreddingFileReaderTest, TestListValue) { + std::shared_ptr logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::list(arrow::int32()))), + }); + MapSharedShreddingFieldMeta meta; + meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + meta.field_to_columns = {{0, {0, 1}}, {1, {0}}, {2, {0}}}; + meta.overflow_field_set = {2}; + meta.num_columns = 2; + meta.max_row_width = 3; + + std::map field_to_num_columns = {{"tags", 2}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); + auto metadata = std::make_shared(); + ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata( + meta, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get())); + arrow::FieldVector physical_fields = physical_schema->fields(); + physical_fields[1] = physical_fields[1]->WithMetadata(metadata); + physical_schema = arrow::schema(std::move(physical_fields)); + + auto physical_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), R"([ + [1, [[0, 1], [1, null, 2], [3], null]], + [2, [[2, 0], [5, 6], [7], null]], + [3, null], + [4, [[1, 0], [8], [9, 10], [[2, [null]]]]] + ])") + .ValueOrDie(); + auto reader = CreateReader(physical_array, physical_schema); + + auto read_metadata = std::make_shared(); + read_metadata->Append("paimon.map.selected-keys", "a,c"); + arrow::FieldVector read_fields = logical_schema->fields(); + read_fields[1] = read_fields[1]->WithMetadata(read_metadata); + auto read_schema = ExportSchema(arrow::schema(std::move(read_fields))); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema->fields()), {R"([ + [1, [["a", [1, null, 2]]]], + [2, [["a", [7]], ["c", [5, 6]]]], + [3, null], + [4, [["a", [9, 10]], ["c", [null]]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(SharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) { + std::shared_ptr logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + }); + auto options = options_; + std::string format = "orc"; + options[Options::FILE_FORMAT] = format; + options["orc.dictionary-key-size-threshold"] = "1"; + ASSERT_OK_AND_ASSIGN(auto table_schema, + TableSchema::Create(TableSchema::FIRST_SCHEMA_ID, logical_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options)); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + auto path_factory = CreatePathFactory(dir->Str(), format, core_options); + auto compact_manager = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto writer, + AppendOnlyWriter::Create(core_options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, pool_)); + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", "red"], ["b", "blue"]]], + [2, [["c", "green"], ["a", "red"], ["b", "blue"]]], + [3, null], + [4, [["d", "yellow"], ["e", "blue"], ["c", null], ["a", "red"]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(auto inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + std::map reader_options = {{"orc.read.enable-lazy-decoding", "true"}}; + ASSERT_OK_AND_ASSIGN(auto reader, + SharedShreddingFileReader::Create( + OpenFormatReader(data_file_path, format, reader_options), pool_)); + + auto read_metadata = std::make_shared(); + read_metadata->Append("paimon.map.selected-keys", "a,c"); + arrow::FieldVector read_fields = logical_schema->fields(); + read_fields[1] = read_fields[1]->WithMetadata(read_metadata); + auto read_schema = ExportSchema(arrow::schema(std::move(read_fields))); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema->fields()), {R"([ + [1, [["a", "red"]]], + [2, [["a", "red"], ["c", "green"]]], + [3, null], + [4, [["a", "red"], ["c", null]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +TEST_F(SharedShreddingFileReaderTest, TestReadsRealFormatFile) { + // TODO(lisizhuo.lsz): support other format + auto options = options_; + std::string format = "orc"; + options[Options::FILE_FORMAT] = format; + ASSERT_OK_AND_ASSIGN(auto table_schema, + TableSchema::Create(TableSchema::FIRST_SCHEMA_ID, logical_schema_, + /*partition_keys=*/{}, /*primary_keys=*/{}, options)); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); + auto path_factory = CreatePathFactory(dir->Str(), format, core_options); + auto compact_manager = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + auto writer, + AppendOnlyWriter::Create(core_options, /*schema_id=*/0, logical_schema_, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager, pool_)); + auto batch = CreateBatch(logical_schema_, R"([ + [1, [["a", 1], ["b", 2]]], + [2, [["c", 3], ["a", 4], ["b", 5]]], + [3, null], + [4, [["d", 6], ["e", 7], ["c", null], ["a", 9]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(auto inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + ASSERT_OK_AND_ASSIGN(auto reader, SharedShreddingFileReader::Create( + OpenFormatReader(data_file_path, format), pool_)); + + auto read_schema = ExportSchema(ReadSchema("a,c")); + ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(reader.get())); + + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(logical_schema_->fields()), {R"([ + [1, [["a", 1]]], + [2, [["a", 4], ["c", 3]]], + [3, null], + [4, [["a", 9], ["c", null]]] + ])"}, + &expected) + .ok()); + AssertChunkedArrayEquals(expected, actual); +} + +} // namespace paimon::test From 49f3fb5815ecd6de7987d172942f325373177691 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Tue, 23 Jun 2026 14:16:58 +0800 Subject: [PATCH 069/138] fix: address avro blob and executor review issues --- src/paimon/common/data/blob_defs.h | 4 +- src/paimon/format/avro/avro_direct_decoder.h | 3 + .../format/avro/avro_direct_encoder.cpp | 4 +- src/paimon/format/avro/avro_direct_encoder.h | 3 + src/paimon/format/avro/avro_file_format.cpp | 2 +- src/paimon/format/avro/avro_reader_builder.h | 5 +- .../format/blob/blob_file_batch_reader.cpp | 20 ++--- src/paimon/format/blob/blob_format_writer.cpp | 14 ++-- src/paimon/format/blob/blob_format_writer.h | 4 +- .../format/blob/blob_stats_extractor.cpp | 2 +- .../format/blob/blob_stats_extractor_test.cpp | 2 +- test/inte/write_inte_test.cpp | 83 ++++++++++++++++--- 12 files changed, 104 insertions(+), 42 deletions(-) diff --git a/src/paimon/common/data/blob_defs.h b/src/paimon/common/data/blob_defs.h index 6a478e70..35b52e2a 100644 --- a/src/paimon/common/data/blob_defs.h +++ b/src/paimon/common/data/blob_defs.h @@ -56,8 +56,8 @@ class BlobDefs { static constexpr int32_t kContentStartOffset = 4; /// Total metadata length per bin: magic(4) + bin_length(8) + crc32(4) = 16. static constexpr int32_t kTotalMetaLength = 16; - /// Blob file header length: index_len(4) + version(1) = 5. - static constexpr uint32_t kBlobFileHeaderLength = 5; + /// Blob file footer length: index_len(4) + version(1) = 5. + static constexpr uint32_t kBlobFileFooterLength = 5; }; } // namespace paimon diff --git a/src/paimon/format/avro/avro_direct_decoder.h b/src/paimon/format/avro/avro_direct_decoder.h index 41e54f45..c507091a 100644 --- a/src/paimon/format/avro/avro_direct_decoder.h +++ b/src/paimon/format/avro/avro_direct_decoder.h @@ -33,6 +33,9 @@ namespace paimon::avro { class AvroDirectDecoder { public: + AvroDirectDecoder() = delete; + ~AvroDirectDecoder() = delete; + /// Context for reusing scratch buffers during Avro decoding /// /// Avoids frequent small allocations by reusing temporary buffers across multiple decode diff --git a/src/paimon/format/avro/avro_direct_encoder.cpp b/src/paimon/format/avro/avro_direct_encoder.cpp index 90ff7199..b371c27f 100644 --- a/src/paimon/format/avro/avro_direct_encoder.cpp +++ b/src/paimon/format/avro/avro_direct_encoder.cpp @@ -354,11 +354,11 @@ Status AvroDirectEncoder::EncodeArrowToAvro(const ::avro::NodePtr& avro_node, return Status::Invalid(fmt::format("AVRO_MAP keys must be StringArray, got {}", keys->type()->ToString())); } + const auto& string_array = + arrow::internal::checked_cast(*keys); for (int64_t i = start; i < end; ++i) { encoder->startItem(); - const auto& string_array = - arrow::internal::checked_cast(*keys); std::string_view key_value = string_array.GetView(i); encoder->encodeString(std::string(key_value)); diff --git a/src/paimon/format/avro/avro_direct_encoder.h b/src/paimon/format/avro/avro_direct_encoder.h index 4b5a998d..646e19e5 100644 --- a/src/paimon/format/avro/avro_direct_encoder.h +++ b/src/paimon/format/avro/avro_direct_encoder.h @@ -33,6 +33,9 @@ namespace paimon::avro { class AvroDirectEncoder { public: + AvroDirectEncoder() = delete; + ~AvroDirectEncoder() = delete; + /// Context for reusing scratch buffers during Avro encoding /// /// Avoids frequent small allocations by reusing temporary buffers across multiple encode diff --git a/src/paimon/format/avro/avro_file_format.cpp b/src/paimon/format/avro/avro_file_format.cpp index 1cbef060..e69b8e03 100644 --- a/src/paimon/format/avro/avro_file_format.cpp +++ b/src/paimon/format/avro/avro_file_format.cpp @@ -41,7 +41,7 @@ AvroFileFormat::AvroFileFormat(const std::map& options Result> AvroFileFormat::CreateReaderBuilder( int32_t batch_size) const { - return std::make_unique(options_, batch_size); + return std::make_unique(batch_size); } Result> AvroFileFormat::CreateWriterBuilder( diff --git a/src/paimon/format/avro/avro_reader_builder.h b/src/paimon/format/avro/avro_reader_builder.h index bd1da1a2..eb87917b 100644 --- a/src/paimon/format/avro/avro_reader_builder.h +++ b/src/paimon/format/avro/avro_reader_builder.h @@ -34,8 +34,8 @@ namespace paimon::avro { class AvroReaderBuilder : public ReaderBuilder { public: - AvroReaderBuilder(const std::map& options, int32_t batch_size) - : batch_size_(batch_size), pool_(GetDefaultPool()), options_(options) {} + explicit AvroReaderBuilder(int32_t batch_size) + : batch_size_(batch_size), pool_(GetDefaultPool()) {} ReaderBuilder* WithMemoryPool(const std::shared_ptr& pool) override { pool_ = pool; @@ -50,7 +50,6 @@ class AvroReaderBuilder : public ReaderBuilder { private: const int32_t batch_size_; std::shared_ptr pool_; - const std::map options_; }; } // namespace paimon::avro diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index 4cddf625..f74386e0 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -19,7 +19,6 @@ #include "paimon/format/blob/blob_file_batch_reader.h" #include -#include #include #include "arrow/api.h" @@ -29,7 +28,6 @@ #include "arrow/util/bit_util.h" #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" -#include "paimon/common/executor/future.h" #include "paimon/common/io/offset_input_stream.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/mem_utils.h" @@ -54,24 +52,24 @@ Result> BlobFileBatchReader::Create( PAIMON_ASSIGN_OR_RAISE(int64_t file_size, input_stream->Length()); PAIMON_RETURN_NOT_OK( - input_stream->Seek(file_size - BlobDefs::kBlobFileHeaderLength, FS_SEEK_SET)); - int8_t header[BlobDefs::kBlobFileHeaderLength]; + input_stream->Seek(file_size - BlobDefs::kBlobFileFooterLength, FS_SEEK_SET)); + int8_t footer[BlobDefs::kBlobFileFooterLength]; PAIMON_ASSIGN_OR_RAISE( int64_t actual_size, - input_stream->Read(reinterpret_cast(header), BlobDefs::kBlobFileHeaderLength)); - if (actual_size != BlobDefs::kBlobFileHeaderLength) { + input_stream->Read(reinterpret_cast(footer), BlobDefs::kBlobFileFooterLength)); + if (actual_size != BlobDefs::kBlobFileFooterLength) { return Status::Invalid( - fmt::format("actual read size {} not match with expect header length {}", actual_size, - BlobDefs::kBlobFileHeaderLength)); + fmt::format("actual read size {} not match with expect footer length {}", actual_size, + BlobDefs::kBlobFileFooterLength)); } - int8_t version = header[4]; + int8_t version = footer[4]; if (version != BlobDefs::kFileVersion) { return Status::Invalid(fmt::format( "create blob format reader failed. unsupported blob file version: {}", version)); } - int32_t index_length = GetIndexLength(header, 0); + int32_t index_length = GetIndexLength(footer, 0); PAIMON_RETURN_NOT_OK(input_stream->Seek( - file_size - BlobDefs::kBlobFileHeaderLength - index_length, FS_SEEK_SET)); + file_size - BlobDefs::kBlobFileFooterLength - index_length, FS_SEEK_SET)); std::vector index_bytes(index_length, '\0'); PAIMON_ASSIGN_OR_RAISE(actual_size, input_stream->Read(index_bytes.data(), index_length)); if (actual_size != index_length) { diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index 6331f94e..fc4042ee 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -49,6 +49,7 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con write_consumer_(std::move(write_consumer)) { metrics_ = std::make_shared(); tmp_buffer_ = Bytes::AllocateBytes(kTmpBufferSize, pool_.get()); + magic_number_bytes_ = IntegerToLittleEndian(BlobDefs::kMagicNumber, pool_); } Result> BlobFormatWriter::Create( @@ -163,9 +164,7 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { PAIMON_ASSIGN_OR_RAISE(int64_t previous_pos, out_->GetPos()); // write magic number - static PAIMON_UNIQUE_PTR kMagicNumberBytes = - IntegerToLittleEndian(BlobDefs::kMagicNumber, pool_); - PAIMON_RETURN_NOT_OK(WriteWithCrc32(kMagicNumberBytes->data(), kMagicNumberBytes->size())); + PAIMON_RETURN_NOT_OK(WriteWithCrc32(magic_number_bytes_->data(), magic_number_bytes_->size())); // write blob content // Dynamically check whether blob_data is a serialized BlobDescriptor (by magic header) @@ -187,8 +186,9 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { while (read_len > 0) { PAIMON_ASSIGN_OR_RAISE(int64_t actual_read_len, in->Read(tmp_buffer_->data(), read_len)); if (actual_read_len != read_len) { - return Status::Invalid("actual read length {}, not match with expect length {}", - actual_read_len, read_len); + return Status::Invalid( + fmt::format("actual read length {}, not match with expect length {}", + actual_read_len, read_len)); } PAIMON_RETURN_NOT_OK(WriteWithCrc32(tmp_buffer_->data(), actual_read_len)); total_read_length += actual_read_len; @@ -216,8 +216,8 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { Status BlobFormatWriter::WriteBytes(const char* data, int64_t length) { PAIMON_ASSIGN_OR_RAISE(int64_t actual, out_->Write(data, length)); if (actual != length) { - return Status::Invalid("not suppose actual length {} not match with expect {}", actual, - length); + return Status::Invalid( + fmt::format("unexpected actual length {} not match with expect {}", actual, length)); } return Status::OK(); } diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 0e734a44..9a78c2a6 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -92,15 +92,15 @@ class BlobFormatWriter : public FormatWriter { static PAIMON_UNIQUE_PTR IntegerToLittleEndian(T value, const std::shared_ptr& pool); - public: + private: static constexpr uint32_t kTmpBufferSize = 1024 * 1024; - private: uint32_t crc32_ = 0; std::vector bin_lengths_; std::shared_ptr out_; std::string uri_; PAIMON_UNIQUE_PTR tmp_buffer_; + PAIMON_UNIQUE_PTR magic_number_bytes_; std::shared_ptr data_type_; std::shared_ptr fs_; std::shared_ptr pool_; diff --git a/src/paimon/format/blob/blob_stats_extractor.cpp b/src/paimon/format/blob/blob_stats_extractor.cpp index 12a7abdd..4b5845c5 100644 --- a/src/paimon/format/blob/blob_stats_extractor.cpp +++ b/src/paimon/format/blob/blob_stats_extractor.cpp @@ -56,7 +56,7 @@ BlobStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& file_ /*batch_size=*/1024, /*blob_as_descriptor=*/true, pool)); ColumnStatsVector result_stats; result_stats.push_back( - ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, /*null_count=*/0)); + ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt)); PAIMON_ASSIGN_OR_RAISE(uint64_t num_rows, blob_reader->GetNumberOfRows()); return std::make_pair(result_stats, FileInfo(num_rows)); } diff --git a/src/paimon/format/blob/blob_stats_extractor_test.cpp b/src/paimon/format/blob/blob_stats_extractor_test.cpp index e6c8d694..befbfa52 100644 --- a/src/paimon/format/blob/blob_stats_extractor_test.cpp +++ b/src/paimon/format/blob/blob_stats_extractor_test.cpp @@ -70,7 +70,7 @@ TEST_F(BlobStatsExtractorTest, TestDifferentBlobFiles) { ASSERT_EQ(1u, stats_with_info.first.size()); ASSERT_TRUE(stats_with_info.first[0]); ASSERT_EQ(FieldType::STRING, stats_with_info.first[0]->GetFieldType()); - ASSERT_EQ("min null, max null, null count 0", stats_with_info.first[0]->ToString()); + ASSERT_EQ("min null, max null, null count null", stats_with_info.first[0]->ToString()); // Check row count matches expected ASSERT_EQ(expected_rows, stats_with_info.second.GetRowCount()) diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index eef6d6e4..2c7fcf64 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -277,6 +277,55 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface return array; } + SimpleStats GenerateBlobValueStats() const { + return BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, + std::vector({-1}), pool_.get()); + } + + Status CheckReadBlobs(const std::string& table_path, + const std::map& options, + const std::vector>& data_splits, + const std::string& blob_field, + const std::vector>& expected_blobs) const { + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadSchema({blob_field}); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(data_splits)); + PAIMON_ASSIGN_OR_RAISE(auto read_result, + ReadResultCollector::CollectResult(batch_reader.get())); + if (read_result == nullptr) { + return Status::Invalid(fmt::format("No rows read for blob field {}", blob_field)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto combined_array, + arrow::Concatenate(read_result->chunks())); + auto struct_array = std::dynamic_pointer_cast(combined_array); + if (struct_array == nullptr) { + return Status::Invalid( + fmt::format("Read result for {} is not a struct array", blob_field)); + } + auto struct_type = std::dynamic_pointer_cast(struct_array->type()); + int blob_index = struct_type->GetFieldIndex(blob_field); + if (blob_index < 0) { + return Status::Invalid( + fmt::format("Blob field {} was not found in read result", blob_field)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + auto blob_struct_array, + arrow::StructArray::Make({struct_array->field(blob_index)}, + {BlobUtils::ToArrowField(blob_field, false)})); + PAIMON_ASSIGN_OR_RAISE( + auto actual_blobs, + TestHelper::ToBlobs(std::static_pointer_cast(blob_struct_array))); + PAIMON_ASSIGN_OR_RAISE(bool blobs_equal, TestHelper::CheckBlobsEqual( + actual_blobs, expected_blobs, file_system_)); + if (!blobs_equal) { + return Status::Invalid(fmt::format("Read blobs for {} do not match", blob_field)); + } + return Status::OK(); + } + void CheckCreationTime(const std::vector>& commit_messages) { TimezoneGuard guard("Asia/Shanghai"); for (const auto& msg : commit_messages) { @@ -3755,20 +3804,26 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); int64_t commit_identifier = 0; std::vector> blob_descriptors; + std::vector> expected_blobs; std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; ASSERT_OK_AND_ASSIGN(auto blob1, Blob::FromPath(file1)); blob_descriptors.emplace_back(blob1->ToDescriptor(pool_)); + expected_blobs.emplace_back(std::shared_ptr(std::move(blob1))); std::string file2 = paimon::test::GetDataDir() + "/xxhash.data"; ASSERT_OK_AND_ASSIGN(auto blob2, Blob::FromPath(file2, /*offset=*/0, /*length=*/91)); blob_descriptors.emplace_back(blob2->ToDescriptor(pool_)); + expected_blobs.emplace_back(std::shared_ptr(std::move(blob2))); ASSERT_OK_AND_ASSIGN(auto blob3, Blob::FromPath(file2, /*offset=*/92, /*length=*/85)); blob_descriptors.emplace_back(blob3->ToDescriptor(pool_)); + expected_blobs.emplace_back(std::shared_ptr(std::move(blob3))); ASSERT_OK_AND_ASSIGN(auto blob4, Blob::FromPath(file2, /*offset=*/300, /*length=*/3000)); blob_descriptors.emplace_back(blob4->ToDescriptor(pool_)); + expected_blobs.emplace_back(std::shared_ptr(std::move(blob4))); std::vector>> blob_fields; blob_fields.emplace_back(std::move(blob_descriptors)); @@ -3794,9 +3849,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { auto file_meta2 = std::make_shared( "data-xxx.blob", /*file_size=*/764, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), - /*key_stats=*/SimpleStats::EmptyStats(), - BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, std::vector({0}), - pool_.get()), + /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1724090888706ll, 0), @@ -3806,9 +3859,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { auto file_meta3 = std::make_shared( "data-xxx.blob", /*file_size=*/3023, /*row_count=*/1, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), - /*key_stats=*/SimpleStats::EmptyStats(), - BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, std::vector({0}), - pool_.get()), + /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(1724090888706ll, 0), @@ -3840,6 +3891,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { for (size_t i = 0; i < expected_meta.size(); i++) { ASSERT_TRUE(data_split->DataFiles()[i]->TEST_Equal(*expected_meta[i])); } + ASSERT_OK(CheckReadBlobs(table_path, options, data_splits, "blob", expected_blobs)); } TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) { @@ -4570,29 +4622,38 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { ASSERT_OK_AND_ASSIGN( auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, /*primary_keys=*/{}, options, /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); int64_t commit_identifier = 0; // Prepare blob descriptors for both blob fields std::vector> blob1_descriptors; std::vector> blob2_descriptors; + std::vector> expected_blob1s; + std::vector> expected_blob2s; std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; ASSERT_OK_AND_ASSIGN(auto blob1_a, Blob::FromPath(file1)); blob1_descriptors.emplace_back(blob1_a->ToDescriptor(pool_)); + expected_blob1s.emplace_back(std::shared_ptr(std::move(blob1_a))); std::string file2 = paimon::test::GetDataDir() + "/xxhash.data"; ASSERT_OK_AND_ASSIGN(auto blob1_b, Blob::FromPath(file2, /*offset=*/0, /*length=*/91)); blob1_descriptors.emplace_back(blob1_b->ToDescriptor(pool_)); + expected_blob1s.emplace_back(std::shared_ptr(std::move(blob1_b))); ASSERT_OK_AND_ASSIGN(auto blob1_c, Blob::FromPath(file2, /*offset=*/92, /*length=*/85)); blob1_descriptors.emplace_back(blob1_c->ToDescriptor(pool_)); + expected_blob1s.emplace_back(std::shared_ptr(std::move(blob1_c))); // blob2 field uses different data slices ASSERT_OK_AND_ASSIGN(auto blob2_a, Blob::FromPath(file2, /*offset=*/300, /*length=*/3000)); blob2_descriptors.emplace_back(blob2_a->ToDescriptor(pool_)); + expected_blob2s.emplace_back(std::shared_ptr(std::move(blob2_a))); ASSERT_OK_AND_ASSIGN(auto blob2_b, Blob::FromPath(file2, /*offset=*/0, /*length=*/91)); blob2_descriptors.emplace_back(blob2_b->ToDescriptor(pool_)); + expected_blob2s.emplace_back(std::shared_ptr(std::move(blob2_b))); ASSERT_OK_AND_ASSIGN(auto blob2_c, Blob::FromPath(file1)); blob2_descriptors.emplace_back(blob2_c->ToDescriptor(pool_)); + expected_blob2s.emplace_back(std::shared_ptr(std::move(blob2_c))); std::vector>> blob_fields; blob_fields.emplace_back(std::move(blob1_descriptors)); @@ -4623,9 +4684,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { auto expected_blob1 = std::make_shared( "data-xxx.blob", /*file_size=*/0, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), - /*key_stats=*/SimpleStats::EmptyStats(), - BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, std::vector({0}), - pool_.get()), + /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), @@ -4637,9 +4696,7 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { auto expected_blob2 = std::make_shared( "data-xxx.blob", /*file_size=*/0, /*row_count=*/3, /*min_key=*/BinaryRow::EmptyRow(), /*max_key=*/BinaryRow::EmptyRow(), - /*key_stats=*/SimpleStats::EmptyStats(), - BinaryRowGenerator::GenerateStats({NullType()}, {NullType()}, std::vector({0}), - pool_.get()), + /*key_stats=*/SimpleStats::EmptyStats(), GenerateBlobValueStats(), /*min_sequence_number=*/1, /*max_sequence_number=*/1, /*schema_id=*/0, /*level=*/0, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), @@ -4682,6 +4739,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { } } } + ASSERT_OK(CheckReadBlobs(table_path, options, data_splits, "blob1", expected_blob1s)); + ASSERT_OK(CheckReadBlobs(table_path, options, data_splits, "blob2", expected_blob2s)); } TEST_P(WriteInteTest, TestRowTrackingPartitionGroupOnCommit) { From 2453996a1f62520d56589706fd785710b266abfa Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Wed, 24 Jun 2026 12:56:14 +0800 Subject: [PATCH 070/138] feat(shredding): improve shared-shredding adaptive width restore & refactor write --- src/paimon/CMakeLists.txt | 7 + .../map_shared_shredding_context.cpp | 26 +- .../shredding/map_shared_shredding_context.h | 9 +- .../map_shared_shredding_context_test.cpp | 74 +++- .../shredding/map_shared_shredding_utils.cpp | 6 +- .../map_shared_shredding_utils_test.cpp | 16 + .../shared_shredding_file_reader_test.cpp | 26 +- src/paimon/core/append/append_only_writer.cpp | 140 ++----- src/paimon/core/append/append_only_writer.h | 41 +- .../core/append/append_only_writer_test.cpp | 205 +++++++--- .../io/append_data_file_writer_factory.cpp | 61 +++ .../core/io/append_data_file_writer_factory.h | 70 ++++ .../core/io/blob_data_file_writer_factory.cpp | 81 ++++ .../core/io/blob_data_file_writer_factory.h | 74 ++++ .../core/io/data_file_writer_factory.cpp | 61 +++ src/paimon/core/io/data_file_writer_factory.h | 60 +++ .../core/io/external_storage_blob_writer.cpp | 59 +-- .../io/key_value_data_file_writer_factory.cpp | 69 ++++ .../io/key_value_data_file_writer_factory.h | 67 ++++ .../io/map_shared_shredding_core_utils.cpp | 141 +++++++ .../core/io/map_shared_shredding_core_utils.h | 51 +++ .../core/io/rolling_blob_file_writer.cpp | 8 +- src/paimon/core/io/rolling_blob_file_writer.h | 3 +- src/paimon/core/io/rolling_file_writer.h | 14 +- ...edding_append_data_file_writer_factory.cpp | 81 ++++ ...hredding_append_data_file_writer_factory.h | 54 +++ ...ing_key_value_data_file_writer_factory.cpp | 84 +++++ ...dding_key_value_data_file_writer_factory.h | 51 +++ .../core/io/single_file_writer_factory.h | 37 ++ .../manifest/manifest_entry_writer_factory.h | 85 +++++ src/paimon/core/manifest/manifest_file.cpp | 14 +- .../compact/changelog_merge_tree_rewriter.cpp | 5 +- ...ookup_merge_tree_compact_rewriter_test.cpp | 2 +- .../compact/merge_tree_compact_rewriter.cpp | 100 ++--- .../compact/merge_tree_compact_rewriter.h | 5 +- .../remote_lookup_file_manager_test.cpp | 3 +- .../core/mergetree/lookup_levels_test.cpp | 3 +- .../core/mergetree/merge_tree_writer.cpp | 81 +--- src/paimon/core/mergetree/merge_tree_writer.h | 1 + .../core/mergetree/merge_tree_writer_test.cpp | 34 +- .../append_only_file_store_write.cpp | 94 ++--- .../operation/append_only_file_store_write.h | 14 +- .../append_only_file_store_write_test.cpp | 354 ++++++++++++++++++ .../operation/key_value_file_store_write.cpp | 16 +- .../key_value_file_store_write_test.cpp | 140 +++++++ .../postpone_bucket_file_store_write.h | 10 +- .../core/postpone/postpone_bucket_writer.cpp | 70 +--- .../core/postpone/postpone_bucket_writer.h | 1 + .../postpone/postpone_bucket_writer_test.cpp | 52 +-- src/paimon/core/table/system/system_table.cpp | 6 +- src/paimon/format/orc/orc_adapter.cpp | 6 +- 51 files changed, 2185 insertions(+), 587 deletions(-) create mode 100644 src/paimon/core/io/append_data_file_writer_factory.cpp create mode 100644 src/paimon/core/io/append_data_file_writer_factory.h create mode 100644 src/paimon/core/io/blob_data_file_writer_factory.cpp create mode 100644 src/paimon/core/io/blob_data_file_writer_factory.h create mode 100644 src/paimon/core/io/data_file_writer_factory.cpp create mode 100644 src/paimon/core/io/data_file_writer_factory.h create mode 100644 src/paimon/core/io/key_value_data_file_writer_factory.cpp create mode 100644 src/paimon/core/io/key_value_data_file_writer_factory.h create mode 100644 src/paimon/core/io/map_shared_shredding_core_utils.cpp create mode 100644 src/paimon/core/io/map_shared_shredding_core_utils.h create mode 100644 src/paimon/core/io/shredding_append_data_file_writer_factory.cpp create mode 100644 src/paimon/core/io/shredding_append_data_file_writer_factory.h create mode 100644 src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp create mode 100644 src/paimon/core/io/shredding_key_value_data_file_writer_factory.h create mode 100644 src/paimon/core/io/single_file_writer_factory.h create mode 100644 src/paimon/core/manifest/manifest_entry_writer_factory.h diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 175a766e..a7f2b972 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -222,17 +222,24 @@ set(PAIMON_CORE_SRCS core/io/data_file_meta.cpp core/io/data_file_meta_serializer.cpp core/io/data_file_path_factory.cpp + core/io/append_data_file_writer_factory.cpp + core/io/blob_data_file_writer_factory.cpp + core/io/data_file_writer_factory.cpp core/io/data_file_writer.cpp core/io/field_mapping_reader.cpp core/io/complete_row_tracking_fields_reader.cpp core/io/file_index_evaluator.cpp core/io/key_value_data_file_record_reader.cpp + core/io/key_value_data_file_writer_factory.cpp core/io/key_value_data_file_writer.cpp core/io/key_value_in_memory_record_reader.cpp core/io/merged_key_value_record_reader.cpp core/io/key_value_meta_projection_consumer.cpp core/io/key_value_projection_consumer.cpp core/io/key_value_projection_reader.cpp + core/io/map_shared_shredding_core_utils.cpp + core/io/shredding_append_data_file_writer_factory.cpp + core/io/shredding_key_value_data_file_writer_factory.cpp core/io/external_storage_blob_writer.cpp core/io/multiple_blob_file_writer.cpp core/io/rolling_blob_file_writer.cpp diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context.cpp b/src/paimon/common/data/shredding/map_shared_shredding_context.cpp index 9d5356f5..e5174284 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_context.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_context.cpp @@ -20,6 +20,7 @@ #include "paimon/common/data/shredding/map_shared_shredding_context.h" #include +#include namespace paimon { @@ -35,8 +36,8 @@ std::map MapSharedShreddingContext::ComputeNextK() const { // First file — no history, use K_max. result[field_name] = k_max; } else { - int32_t window_max = ComputeWindowMax(it->second); - result[field_name] = std::max(1, std::min(window_max, k_max)); + int32_t adaptive_width = ComputeAdaptiveWidth(it->second); + result[field_name] = std::max(1, std::min(adaptive_width, k_max)); } } return result; @@ -60,12 +61,27 @@ std::vector MapSharedShreddingContext::GetShreddingColumnNames() co return names; } -int32_t MapSharedShreddingContext::ComputeWindowMax(const std::vector& values) { +int32_t MapSharedShreddingContext::ComputeAdaptiveWidth(const std::vector& values) { if (values.empty()) { return 0; } - // TODO(xinyu.lxy): support P99 - return *std::max_element(values.begin(), values.end()); + + std::vector sorted_values(values.begin(), values.end()); + std::sort(sorted_values.begin(), sorted_values.end()); + + int32_t max_width = sorted_values.back(); + auto percentile_rank = static_cast(std::ceil(kPercentileRatio * sorted_values.size())); + percentile_rank = std::clamp(percentile_rank, 1, sorted_values.size()); + int32_t percentile_width = sorted_values[percentile_rank - 1]; + + // Use P90 to ignore far outliers, but keep max when it is close enough to normal rows. + auto relative_close_threshold = static_cast( + std::ceil(static_cast(percentile_width) * kMaxCloseRelativeRatio)); + if (max_width - percentile_width <= kMaxCloseAbsoluteSlack || + max_width <= relative_close_threshold) { + return max_width; + } + return percentile_width; } } // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context.h b/src/paimon/common/data/shredding/map_shared_shredding_context.h index 54466941..c3497556 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_context.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_context.h @@ -33,7 +33,7 @@ namespace paimon { /// values to support adaptive K sizing across files. /// /// - First file: K = K_max (no history). -/// - Subsequent files: K = min(max(recent_max_row_widths), K_max). +/// - Subsequent files: K = min(adaptive_width(recent_max_row_widths), K_max). class MapSharedShreddingContext { public: /// @param column_to_k_max Map from field name to its K_max (from options). @@ -53,9 +53,12 @@ class MapSharedShreddingContext { std::vector GetShreddingColumnNames() const; private: - static constexpr int32_t kWindowSize = 100; + static constexpr int32_t kWindowSize = 20; + static constexpr double kPercentileRatio = 0.90; + static constexpr int32_t kMaxCloseAbsoluteSlack = 4; + static constexpr double kMaxCloseRelativeRatio = 1.25; - static int32_t ComputeWindowMax(const std::vector& values); + static int32_t ComputeAdaptiveWidth(const std::vector& values); /// K_max per shared-shredding field, from options. std::map column_to_k_max_; diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp index 655f8de6..8f3ea276 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_context_test.cpp @@ -40,7 +40,7 @@ TEST(MapSharedShreddingContextTest, FirstFileUsesKMax) { TEST(MapSharedShreddingContextTest, AdaptKAfterOneFile) { // After reporting stats from one file, K should adapt to - // min(max_row_width, K_max). + // min(adaptive width, K_max). std::map field_to_k_max = {{"m", 10}}; MapSharedShreddingContext context(field_to_k_max); @@ -67,8 +67,8 @@ TEST(MapSharedShreddingContextTest, AdaptKCappedByKMax) { ASSERT_EQ(5, next_k.at("m")); } -TEST(MapSharedShreddingContextTest, WindowMaxTracksLargest) { - // K should use the max of all recent max_row_widths within the window. +TEST(MapSharedShreddingContextTest, WindowP90UsesMaxWhenSamplesAreClose) { + // Small sample windows still use max because max and P90 are close. std::map field_to_k_max = {{"m", 20}}; MapSharedShreddingContext context(field_to_k_max); @@ -76,11 +76,62 @@ TEST(MapSharedShreddingContextTest, WindowMaxTracksLargest) { context.ReportFileStats("m", 7); context.ReportFileStats("m", 5); - // max of {3, 7, 5} = 7, capped by K_max=20 → K=7. auto next_k = context.ComputeNextK(); ASSERT_EQ(7, next_k.at("m")); } +TEST(MapSharedShreddingContextTest, WindowP90IgnoresSingleFarOutlier) { + std::map field_to_k_max = {{"m", 2000}}; + MapSharedShreddingContext context(field_to_k_max); + + for (int32_t i = 0; i < 19; ++i) { + context.ReportFileStats("m", 3); + } + context.ReportFileStats("m", 1000); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(3, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, WindowP90UsesMaxWithinAbsoluteSlack) { + std::map field_to_k_max = {{"m", 20}}; + MapSharedShreddingContext context(field_to_k_max); + + for (int32_t i = 0; i < 19; ++i) { + context.ReportFileStats("m", 3); + } + context.ReportFileStats("m", 7); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(7, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, WindowP90UsesMaxWithinRelativeSlack) { + std::map field_to_k_max = {{"m", 200}}; + MapSharedShreddingContext context(field_to_k_max); + + for (int32_t i = 0; i < 19; ++i) { + context.ReportFileStats("m", 100); + } + context.ReportFileStats("m", 125); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(125, next_k.at("m")); +} + +TEST(MapSharedShreddingContextTest, WindowP90IgnoresMaxBeyondBothSlacks) { + std::map field_to_k_max = {{"m", 200}}; + MapSharedShreddingContext context(field_to_k_max); + + for (int32_t i = 0; i < 19; ++i) { + context.ReportFileStats("m", 100); + } + context.ReportFileStats("m", 130); + + auto next_k = context.ComputeNextK(); + ASSERT_EQ(100, next_k.at("m")); +} + TEST(MapSharedShreddingContextTest, MultipleColumnsIndependent) { // Each field adapts independently. std::map field_to_k_max = {{"tags", 10}, {"attrs", 6}}; @@ -104,8 +155,7 @@ TEST(MapSharedShreddingContextTest, MultipleColumnsIndependent) { context.ReportFileStats("attrs", 6); auto k3 = context.ComputeNextK(); - // tags: max(4,8)=8, capped by 10 → 8 - // attrs: max(2,6)=6, capped by 6 → 6 + // tags and attrs are close sample windows, so adaptive width keeps max. ASSERT_EQ(8, k3.at("tags")); ASSERT_EQ(6, k3.at("attrs")); } @@ -119,7 +169,7 @@ TEST(MapSharedShreddingContextTest, GetShreddingColumnNames) { } TEST(MapSharedShreddingContextTest, SlidingWindowEvictsOldEntries) { - // The window size is 100. After filling 100 entries, adding one more + // The window size is 20. After filling 20 entries, adding one more // should evict the oldest. Verify that the evicted value no longer // affects ComputeNextK. std::map field_to_k_max = {{"m", 256}}; @@ -128,19 +178,19 @@ TEST(MapSharedShreddingContextTest, SlidingWindowEvictsOldEntries) { // Insert a large value as the first entry. context.ReportFileStats("m", 200); - // Fill the remaining 99 slots with small values. - for (int i = 0; i < 99; ++i) { + // Fill the remaining 19 slots with small values. + for (int32_t i = 0; i < 19; ++i) { context.ReportFileStats("m", 3); } - // Window = [200, 3, 3, ..., 3] (100 entries). Max = 200. + // Window = [200, 3, 3, ..., 3] (20 entries). P90 = 3, max is a far outlier. auto k_before = context.ComputeNextK(); - ASSERT_EQ(200, k_before.at("m")); + ASSERT_EQ(3, k_before.at("m")); // Push one more — evicts the 200. context.ReportFileStats("m", 5); - // Window = [3, 3, ..., 3, 5] (100 entries). Max = 5. + // Window = [3, 3, ..., 3, 5] (20 entries). Max is within the absolute slack. auto k_after = context.ComputeNextK(); ASSERT_EQ(5, k_after.at("m")); } diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp index 8b376a9a..518ed32d 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp @@ -142,7 +142,7 @@ Result> MapSharedShreddingUtils::LogicalToPhysica auto value_type = map_type->item_type(); bool value_nullable = map_type->item_field()->nullable(); auto physical_type = BuildPhysicalStructType(value_type, it->second, value_nullable); - auto physical_field = arrow::field(field->name(), physical_type, field->nullable()); + auto physical_field = field->WithType(physical_type); physical_fields.push_back(physical_field); } else { physical_fields.push_back(field); @@ -454,6 +454,10 @@ MapSharedShreddingUtils::BuildMetadataFinalizer( arrow::FieldVector updated_fields = physical_schema->fields(); for (const std::string& field_name : shredding_field_names) { int32_t col_index = physical_schema->GetFieldIndex(field_name); + if (col_index < 0) { + return Status::Invalid(fmt::format( + "Shared-shredding field '{}' not found in physical schema.", field_name)); + } const auto& field = physical_schema->field(col_index); auto metadata = field->metadata() ? field->metadata()->Copy() : std::make_shared(); diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp index 99894721..79a21fdf 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp @@ -141,6 +141,7 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNullable) { ASSERT_OK_AND_ASSIGN( auto physical, MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_nullable, col_map)); auto struct_type = physical->field(0)->type(); + ASSERT_TRUE(struct_type->field(0)->nullable()); ASSERT_TRUE(struct_type->field(1)->nullable()); ASSERT_TRUE(struct_type->field(2)->nullable()); @@ -155,6 +156,21 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNullable) { ASSERT_FALSE(struct_type2->field(2)->nullable()); } +TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaPreservesFieldMetadata) { + auto metadata = std::make_shared(); + metadata->Append("paimon.field.id", "7"); + metadata->Append("description", "original map field"); + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + auto schema = arrow::schema({arrow::field("m", map_type, false, metadata)}); + std::map col_map = {{"m", 2}}; + + ASSERT_OK_AND_ASSIGN(auto physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, col_map)); + + ASSERT_FALSE(physical_schema->field(0)->nullable()); + ASSERT_TRUE(physical_schema->field(0)->metadata()->Equals(*metadata)); +} + TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNoShreddingColumns) { auto schema = arrow::schema({ arrow::field("id", arrow::int32()), diff --git a/src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp b/src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp index 8851e337..5f8e274b 100644 --- a/src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp +++ b/src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp @@ -169,6 +169,20 @@ class SharedShreddingFileReaderTest : public ::testing::Test { return reader_builder->Build(input_stream).value(); } + Result> CreateAppendOnlyWriter( + const CoreOptions& core_options, int64_t schema_id, + const std::shared_ptr& logical_schema, + const std::optional>& write_cols, int64_t max_sequence_number, + const std::shared_ptr& path_factory, + const std::shared_ptr& compact_manager) const { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(logical_schema, core_options)); + return std::make_unique(core_options, schema_id, logical_schema, + write_cols, max_sequence_number, path_factory, + compact_manager, shredding_context, pool_); + } + private: std::shared_ptr pool_; std::shared_ptr logical_schema_ = arrow::schema({ @@ -476,9 +490,9 @@ TEST_F(SharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) { auto compact_manager = std::make_shared(); ASSERT_OK_AND_ASSIGN( auto writer, - AppendOnlyWriter::Create(core_options, /*schema_id=*/0, logical_schema, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, pool_)); + CreateAppendOnlyWriter(core_options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager)); auto batch = CreateBatch(logical_schema, R"([ [1, [["a", "red"], ["b", "blue"]]], [2, [["c", "green"], ["a", "red"], ["b", "blue"]]], @@ -533,9 +547,9 @@ TEST_F(SharedShreddingFileReaderTest, TestReadsRealFormatFile) { auto compact_manager = std::make_shared(); ASSERT_OK_AND_ASSIGN( auto writer, - AppendOnlyWriter::Create(core_options, /*schema_id=*/0, logical_schema_, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, compact_manager, pool_)); + CreateAppendOnlyWriter(core_options, /*schema_id=*/0, logical_schema_, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager)); auto batch = CreateBatch(logical_schema_, R"([ [1, [["a", 1], ["b", 2]]], [2, [["c", 3], ["a", 4], ["b", 5]]], diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index c5d81093..48b3d321 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -28,54 +28,31 @@ #include "arrow/c/helpers.h" #include "arrow/type.h" #include "arrow/util/key_value_metadata.h" -#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" -#include "paimon/common/data/shredding/map_shared_shredding_context.h" -#include "paimon/common/data/shredding/map_shared_shredding_utils.h" -#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/types/row_kind.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/long_counter.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/io/append_data_file_writer_factory.h" +#include "paimon/core/io/blob_data_file_writer_factory.h" #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_path_factory.h" -#include "paimon/core/io/data_file_writer.h" #include "paimon/core/io/data_increment.h" #include "paimon/core/io/external_storage_blob_writer.h" #include "paimon/core/io/multiple_blob_file_writer.h" #include "paimon/core/io/rolling_blob_file_writer.h" #include "paimon/core/io/rolling_file_writer.h" +#include "paimon/core/io/shredding_append_data_file_writer_factory.h" #include "paimon/core/io/single_file_writer.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/operation/blob_file_context.h" #include "paimon/core/utils/commit_increment.h" -#include "paimon/format/file_format.h" -#include "paimon/format/file_format_factory.h" -#include "paimon/format/writer_builder.h" #include "paimon/macros.h" #include "paimon/metrics.h" #include "paimon/record_batch.h" namespace paimon { -class MemoryPool; -class FormatStatsExtractor; - -Result> AppendOnlyWriter::Create( - const CoreOptions& options, int64_t schema_id, - const std::shared_ptr& write_schema, - const std::optional>& write_cols, int64_t max_sequence_number, - const std::shared_ptr& path_factory, - const std::shared_ptr& compact_manager, - const std::shared_ptr& memory_pool) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context, - MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); - - return std::unique_ptr( - new AppendOnlyWriter(options, schema_id, write_schema, write_cols, max_sequence_number, - path_factory, compact_manager, shredding_context, memory_pool)); -} - AppendOnlyWriter::AppendOnlyWriter( const CoreOptions& options, int64_t schema_id, const std::shared_ptr& write_schema, @@ -246,88 +223,37 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWrit // No BLOB fields at all -> plain rolling writer return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterCreator(write_schema_, main_write_cols)); + GetDataFileWriterFactory(write_schema_, main_write_cols)); } else { // All BLOB fields are inline, no .blob files needed -> plain rolling writer // The main data file contains all fields including inline descriptors/views. return std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterCreator(write_schema_, main_write_cols)); + GetDataFileWriterFactory(write_schema_, main_write_cols)); } } -AppendOnlyWriter::SingleFileWriterCreator AppendOnlyWriter::GetDataFileWriterCreator( +AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetDataFileWriterFactory( const std::shared_ptr& schema, const std::optional>& write_cols) const { - return - [this, schema, write_cols]() - -> Result< - std::unique_ptr>>> { - // Determine the schema to use for file writing. - // When shared-shredding map is active, compute per-file K and build a physical schema. - PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, - MapSharedShreddingBatchConverter::CreateConverter( - schema, shredding_context_, memory_pool_)); - std::shared_ptr file_schema = - bundle.physical_schema ? bundle.physical_schema : schema; - - ::ArrowSchema arrow_schema; - ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); - auto format = options_.GetFileFormat(); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr writer_builder, - format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); - writer_builder->WithMemoryPool(memory_pool_); - - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, - format->CreateStatsExtractor(&arrow_schema)); - // Build the converter that transforms logical batches to physical batches. - // When shredding is active, it performs MAP→STRUCT conversion first. - std::function batch_converter; - if (bundle.converter) { - auto converter = bundle.converter; - batch_converter = [converter](ArrowArray* input, ArrowArray* output) -> Status { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, - converter->Convert(input)); - ArrowArrayMove(physical.get(), output); - return Status::OK(); - }; - } - - auto writer = std::make_unique( - options_.GetFileCompression(), batch_converter, schema_id_, seq_num_counter_, - FileSource::Append(), stats_extractor, path_factory_->IsExternalPath(), write_cols, - memory_pool_); - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder)); - - if (bundle.converter) { - writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( - bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, - shredding_context_, file_schema)); - } - return writer; - }; + if (shredding_context_) { + return std::make_shared( + options_, schema_id_, schema, write_cols, seq_num_counter_, FileSource::Append(), + path_factory_, shredding_context_, memory_pool_); + } + return std::make_shared(options_, schema_id_, schema, write_cols, + seq_num_counter_, FileSource::Append(), + path_factory_, memory_pool_); } -AppendOnlyWriter::SingleFileWriterCreator AppendOnlyWriter::GetBlobFileWriterCreator( - const std::shared_ptr& writer_builder, - const std::shared_ptr& stats_extractor, +AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetBlobFileWriterFactory( + const std::shared_ptr& single_field_schema, const std::optional>& write_cols) const { - return - [this, writer_builder, stats_extractor, write_cols]() - -> Result< - std::unique_ptr>>> { - auto writer = std::make_unique( - /*compression=*/"none", std::function(), - schema_id_, seq_num_counter_, FileSource::Append(), stats_extractor, - path_factory_->IsExternalPath(), write_cols, memory_pool_); - PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), - path_factory_->NewBlobPath(), writer_builder)); - return writer; - }; + std::shared_ptr path_factory = path_factory_; + return std::make_shared( + options_, schema_id_, single_field_schema, write_cols, seq_num_counter_, path_factory, + [path_factory]() { return path_factory->NewBlobPath(); }, + blob::BlobFormatWriter::WriteConsumer(), memory_pool_); } AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWriter( @@ -335,7 +261,8 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri // Multiple blob fields are supported. Each blob field gets its own rolling file writer // via MultipleBlobFileWriter. auto blob_schema = schemas.blob_schema; - auto blob_writer_creator = [this, blob_schema](const std::string& blob_field_name) + MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator = + [this, blob_schema](const std::string& blob_field_name) -> Result< std::unique_ptr>>> { // Create a single-field schema for this blob field @@ -345,29 +272,16 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri fmt::format("Blob field '{}' not found in blob schema", blob_field_name)); } auto single_field_schema = arrow::schema({field}); - ::ArrowSchema arrow_schema; - ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*single_field_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, - FileFormatFactory::Get("blob", options_.ToMap())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr writer_builder, - format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); - writer_builder->WithMemoryPool(memory_pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*single_field_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, - format->CreateStatsExtractor(&arrow_schema)); - std::vector write_cols = {blob_field_name}; - auto single_blob_file_writer_creator = - GetBlobFileWriterCreator(writer_builder, stats_extractor, write_cols); + auto single_blob_file_writer_factory = + GetBlobFileWriterFactory(single_field_schema, write_cols); return std::make_unique>>( - options_.GetBlobTargetFileSize(), single_blob_file_writer_creator); + options_.GetBlobTargetFileSize(), single_blob_file_writer_factory); }; return std::make_unique( options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterCreator(schemas.main_schema, schemas.main_schema->field_names()), + GetDataFileWriterFactory(schemas.main_schema, schemas.main_schema->field_names()), blob_schema, blob_writer_creator, arrow::struct_(write_schema_->fields()), inline_fields); } diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index 1a78d16f..c95ff807 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -19,7 +19,6 @@ #pragma once #include -#include #include #include #include @@ -31,7 +30,6 @@ #include "paimon/core/compact/compact_manager.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" -#include "paimon/core/io/single_file_writer.h" #include "paimon/core/utils/batch_writer.h" #include "paimon/result.h" #include "paimon/status.h" @@ -51,22 +49,23 @@ class MapSharedShreddingContext; class RecordBatch; template class RollingFileWriter; +template +class SingleFileWriterFactory; class LongCounter; class DataFilePathFactory; class MemoryPool; class Metrics; -class FormatStatsExtractor; -class WriterBuilder; class AppendOnlyWriter : public BatchWriter { public: - static Result> Create( - const CoreOptions& options, int64_t schema_id, - const std::shared_ptr& write_schema, - const std::optional>& write_cols, int64_t max_sequence_number, - const std::shared_ptr& path_factory, - const std::shared_ptr& compact_manager, - const std::shared_ptr& memory_pool); + AppendOnlyWriter(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, + int64_t max_sequence_number, + const std::shared_ptr& path_factory, + const std::shared_ptr& compact_manager, + const std::shared_ptr& shredding_context, + const std::shared_ptr& memory_pool); ~AppendOnlyWriter() override; @@ -95,20 +94,11 @@ class AppendOnlyWriter : public BatchWriter { } private: - using SingleFileWriterCreator = std::function< - Result>>>()>; + using WriterFactory = + std::shared_ptr>>; using RollingFileWriterResult = Result>>>; - AppendOnlyWriter(const CoreOptions& options, int64_t schema_id, - const std::shared_ptr& write_schema, - const std::optional>& write_cols, - int64_t max_sequence_number, - const std::shared_ptr& path_factory, - const std::shared_ptr& compact_manager, - const std::shared_ptr& shredding_context, - const std::shared_ptr& memory_pool); - RollingFileWriterResult CreateRollingRowWriter(); RollingFileWriterResult CreateRollingBlobWriter( const BlobUtils::SeparatedSchemas& schemas, @@ -117,13 +107,12 @@ class AppendOnlyWriter : public BatchWriter { Result DrainIncrement(); Status Flush(bool wait_for_latest_compaction, bool forced_full_compaction); - SingleFileWriterCreator GetDataFileWriterCreator( + WriterFactory GetDataFileWriterFactory( const std::shared_ptr& schema, const std::optional>& write_cols) const; - SingleFileWriterCreator GetBlobFileWriterCreator( - const std::shared_ptr& writer_builder, - const std::shared_ptr& stats_extractor, + WriterFactory GetBlobFileWriterFactory( + const std::shared_ptr& single_field_schema, const std::optional>& write_cols) const; Status TrySyncLatestCompaction(bool blocking); diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp index f6c87dea..1e574e5b 100644 --- a/src/paimon/core/append/append_only_writer_test.cpp +++ b/src/paimon/core/append/append_only_writer_test.cpp @@ -40,6 +40,7 @@ #include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/fs/external_path_provider.h" @@ -311,6 +312,21 @@ class AppendOnlyWriterTest : public testing::Test { ASSERT_EQ(expected_meta, deserialized_meta); } + Result> CreateAppendOnlyWriter( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, int64_t max_sequence_number, + const std::shared_ptr& path_factory, + const std::shared_ptr& compact_manager, + const std::shared_ptr& memory_pool) const { + PAIMON_ASSIGN_OR_RAISE( + auto shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); + return std::make_unique(options, schema_id, write_schema, write_cols, + max_sequence_number, path_factory, + compact_manager, shredding_context, memory_pool); + } + protected: std::shared_ptr memory_pool_; std::shared_ptr compact_manager_; @@ -336,7 +352,7 @@ TEST_F(AppendOnlyWriterTest, TestEmptyCommits) { ASSERT_OK(path_factory->Init(dir->Str(), "mock_format", options.DataFilePrefix(), nullptr)); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); for (int32_t i = 0; i < 3; i++) { @@ -366,7 +382,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndPrepareCommit) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), "mock_format", options.DataFilePrefix(), nullptr)); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/2, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); arrow::StringBuilder builder; @@ -409,7 +425,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteAndClose) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); auto struct_type = arrow::struct_(fields); @@ -454,7 +470,7 @@ TEST_F(AppendOnlyWriterTest, TestInvalidRowKind) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), "orc", options.DataFilePrefix(), nullptr)); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/1, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); auto struct_type = arrow::struct_(fields); @@ -493,7 +509,7 @@ TEST_F(AppendOnlyWriterTest, TestPrepareCommitWaitCompactionUsesBlockingGetResul arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); @@ -516,7 +532,7 @@ TEST_F(AppendOnlyWriterTest, TestPrepareCommitForceCompactUsesBlockingGetResult) arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); @@ -560,7 +576,7 @@ TEST_F(AppendOnlyWriterTest, arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); @@ -604,7 +620,7 @@ TEST_F(AppendOnlyWriterTest, TestCloseDeletesCompactAfterFiles) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); @@ -637,7 +653,7 @@ TEST_F(AppendOnlyWriterTest, TestCloseCleansDeletionFile) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); @@ -660,7 +676,7 @@ TEST_F(AppendOnlyWriterTest, TestCompactNotCompletedTriggersCompaction) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); @@ -680,7 +696,7 @@ TEST_F(AppendOnlyWriterTest, TestCompactPassesFullCompactionFlag) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager, memory_pool_)); @@ -702,7 +718,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { auto schema = arrow::schema({int_field, blob_field}); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); @@ -738,7 +754,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { arrow::schema({arrow::field("id", arrow::int32()), BlobUtils::ToArrowField("blob1", false), BlobUtils::ToArrowField("blob2", false)}); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); @@ -777,7 +793,7 @@ TEST_F(AppendOnlyWriterTest, TestMultiplePrepareCommitSequenceContinuity) { arrow::FieldVector fields = {arrow::field("f0", arrow::utf8())}; auto schema = arrow::schema(fields); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); @@ -806,7 +822,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteValidBlobViewField) { auto schema = arrow::schema({arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); @@ -843,7 +859,7 @@ TEST_F(AppendOnlyWriterTest, TestWriteInvalidBlobViewFieldRejected) { auto schema = arrow::schema({arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("view", true)}); ASSERT_OK_AND_ASSIGN( - auto writer, AppendOnlyWriter::Create( + auto writer, CreateAppendOnlyWriter( options, /*schema_id=*/0, schema, /*write_cols=*/std::nullopt, /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_)); @@ -874,6 +890,39 @@ class AppendOnlyWriterShreddingTest : public AppendOnlyWriterTest, INSTANTIATE_TEST_SUITE_P(FileFormats, AppendOnlyWriterShreddingTest, ::testing::Values("parquet", "orc")); +TEST_F(AppendOnlyWriterTest, TestSharedShreddingMapRejectsAvroFormatOnCommit) { + auto options = CreateOptions({ + {Options::FILE_FORMAT, "avro"}, + {Options::MANIFEST_FORMAT, "avro"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), "avro", options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_NOK_WITH_MSG(writer->PrepareCommit(/*wait_compaction=*/true), + "AddMetadata is not supported by avro format writer."); + ASSERT_OK(writer->Close()); +} + TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) { std::string format = GetFormat(); // Configure with shared-shredding map on "tags" field, K=3. @@ -896,10 +945,10 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) { auto path_factory = CreatePathFactory(dir->Str(), format, options); ASSERT_OK_AND_ASSIGN(auto writer, - AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, - compact_manager_, memory_pool_)); + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); // Write a batch with MAP data using the logical schema. // Row0: id=1, tags={a:10, b:20} → fits K=3 @@ -967,10 +1016,10 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllEmptyFirstFile) { auto path_factory = CreatePathFactory(dir->Str(), format, options); ASSERT_OK_AND_ASSIGN(auto writer, - AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, - compact_manager_, memory_pool_)); + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); auto batch = CreateBatch(logical_schema, R"([ [1, []], @@ -1024,10 +1073,10 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF auto path_factory = CreatePathFactory(dir->Str(), format, options); ASSERT_OK_AND_ASSIGN(auto writer, - AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, - compact_manager_, memory_pool_)); + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); auto null_batch = CreateBatch(logical_schema, R"([ [1, null], @@ -1143,10 +1192,10 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithOverflow) { auto path_factory = CreatePathFactory(dir->Str(), format, options); ASSERT_OK_AND_ASSIGN(auto writer, - AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, - compact_manager_, memory_pool_)); + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); // Row0: {a:1, b:2} → fits K=2 // Row1: {c:3, a:4, b:5} → 3 keys, K=2: c→col0, a→col1, b→overflow @@ -1214,10 +1263,10 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil auto path_factory = CreatePathFactory(dir->Str(), format, options); ASSERT_OK_AND_ASSIGN(auto writer, - AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, - compact_manager_, memory_pool_)); + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); // --- File 1: max_row_width = 3, K = K_max = 10 (first file, no history) --- auto batch1 = CreateBatch(logical_schema, R"([ @@ -1327,6 +1376,66 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil ASSERT_OK(writer->Close()); } +TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapUsesInitialContextForFirstFile) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "10"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + auto initial_context = + std::make_shared(std::map{{"tags", 10}}); + initial_context->ReportFileStats("tags", 2); + auto writer = std::make_unique( + options, /*schema_id=*/0, logical_schema, /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, compact_manager_, initial_context, memory_pool_); + + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + + std::string file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map column_to_k = {{"tags", 2}}; + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k)); + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 2; + expected_meta.max_row_width = 3; + CheckShreddingFileSchema(file_path, format, physical_schema, /*field_index=*/1, expected_meta, + options.GetFileCompression()); + + auto physical_type = arrow::struct_(physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1], 10, 20, [[2, 30]]]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(file_path, format, expected_array); + + ASSERT_OK(writer->Close()); +} + TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKAdaptation) { std::string format = GetFormat(); // Two shared-shredding MAP fields with different initial K: tags(K=8), attrs(K=4). @@ -1351,10 +1460,10 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA auto path_factory = CreatePathFactory(dir->Str(), format, options); ASSERT_OK_AND_ASSIGN(auto writer, - AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, - compact_manager_, memory_pool_)); + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); std::string compression = options.GetFileCompression(); // --- File 1: first file, tags K=8, attrs K=4 --- @@ -1481,10 +1590,10 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapDataFileMetaInfo) { auto path_factory = CreatePathFactory(dir->Str(), format, options); ASSERT_OK_AND_ASSIGN(auto writer, - AppendOnlyWriter::Create(options, /*schema_id=*/5, logical_schema, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/9, path_factory, - compact_manager_, memory_pool_)); + CreateAppendOnlyWriter(options, /*schema_id=*/5, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/9, path_factory, + compact_manager_, memory_pool_)); // Write 3 rows. auto batch = CreateBatch(logical_schema, R"([ @@ -1561,10 +1670,10 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapWithBlobSeparation) auto path_factory = CreatePathFactory(dir->Str(), format, options); ASSERT_OK_AND_ASSIGN(auto writer, - AppendOnlyWriter::Create(options, /*schema_id=*/0, logical_schema, - /*write_cols=*/std::nullopt, - /*max_sequence_number=*/-1, path_factory, - compact_manager_, memory_pool_)); + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); // Write rows with id, blob_data, and tags. // Row0: id=1, blob="hello", tags={a:10, b:20} diff --git a/src/paimon/core/io/append_data_file_writer_factory.cpp b/src/paimon/core/io/append_data_file_writer_factory.cpp new file mode 100644 index 00000000..370c99e2 --- /dev/null +++ b/src/paimon/core/io/append_data_file_writer_factory.cpp @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/append_data_file_writer_factory.h" + +#include + +#include "arrow/c/abi.h" +#include "arrow/c/helpers.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/fs/file_system.h" + +namespace paimon { + +AppendDataFileWriterFactory::AppendDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, FileSource file_source, + const std::shared_ptr& path_factory, + const std::shared_ptr& pool) + : DataFileWriterFactory(options, schema_id, pool), + write_schema_(write_schema), + write_cols_(write_cols), + seq_num_counter_(seq_num_counter), + file_source_(file_source), + path_factory_(path_factory) {} + +Result>>> +AppendDataFileWriterFactory::CreateWriter() const { + PAIMON_ASSIGN_OR_RAISE(WriterResources resources, + CreateWriterResources(*options_.GetFileFormat(), write_schema_, + /*create_stats_extractor=*/true)); + auto writer = std::make_unique( + options_.GetFileCompression(), std::function(), + schema_id_, seq_num_counter_, file_source_, resources.stats_extractor, + path_factory_->IsExternalPath(), write_cols_, pool_); + PAIMON_RETURN_NOT_OK( + writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + return std::unique_ptr>>( + std::move(writer)); +} + +} // namespace paimon diff --git a/src/paimon/core/io/append_data_file_writer_factory.h b/src/paimon/core/io/append_data_file_writer_factory.h new file mode 100644 index 00000000..be944fc9 --- /dev/null +++ b/src/paimon/core/io/append_data_file_writer_factory.h @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/c/abi.h" +#include "paimon/core/io/data_file_writer.h" +#include "paimon/core/io/data_file_writer_factory.h" +#include "paimon/core/io/single_file_writer_factory.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class DataFilePathFactory; +class LongCounter; +class MemoryPool; + +class AppendDataFileWriterFactory + : public DataFileWriterFactory, + public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> { + public: + AppendDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, + FileSource file_source, + const std::shared_ptr& path_factory, + const std::shared_ptr& pool); + + Result>>> + CreateWriter() const override; + + protected: + std::shared_ptr write_schema_; + std::optional> write_cols_; + std::shared_ptr seq_num_counter_; + FileSource file_source_; + std::shared_ptr path_factory_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/blob_data_file_writer_factory.cpp b/src/paimon/core/io/blob_data_file_writer_factory.cpp new file mode 100644 index 00000000..d5f4fe5f --- /dev/null +++ b/src/paimon/core/io/blob_data_file_writer_factory.cpp @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/blob_data_file_writer_factory.h" + +#include +#include + +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/format/blob/blob_writer_builder.h" +#include "paimon/format/file_format.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/fs/file_system.h" + +namespace paimon { + +BlobDataFileWriterFactory::BlobDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& file_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, + const std::shared_ptr& path_factory, PathCreator path_creator, + blob::BlobFormatWriter::WriteConsumer write_consumer, const std::shared_ptr& pool) + : DataFileWriterFactory(options, schema_id, pool), + file_schema_(file_schema), + write_cols_(write_cols), + seq_num_counter_(seq_num_counter), + path_factory_(path_factory), + path_creator_(std::move(path_creator)), + write_consumer_(std::move(write_consumer)) {} + +Result>>> +BlobDataFileWriterFactory::CreateWriter() const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, + FileFormatFactory::Get("blob", options_.ToMap())); + PAIMON_ASSIGN_OR_RAISE(WriterResources resources, + CreateWriterResources(*format, file_schema_, + /*create_stats_extractor=*/true)); + if (write_consumer_) { + auto blob_writer_builder = + std::dynamic_pointer_cast(resources.writer_builder); + if (!blob_writer_builder) { + return Status::Invalid( + "writer_builder cannot be casted to BlobWriterBuilder " + "in BlobDataFileWriterFactory"); + } + blob_writer_builder->WithWriteConsumer(write_consumer_); + } + + auto writer = std::make_unique( + /*compression=*/"none", std::function(), schema_id_, + seq_num_counter_, FileSource::Append(), resources.stats_extractor, + path_factory_->IsExternalPath(), write_cols_, pool_); + if (!path_creator_) { + return Status::Invalid("BlobDataFileWriterFactory path creator is empty."); + } + PAIMON_RETURN_NOT_OK( + writer->Init(options_.GetFileSystem(), path_creator_(), resources.writer_builder)); + return std::unique_ptr>>( + std::move(writer)); +} + +} // namespace paimon diff --git a/src/paimon/core/io/blob_data_file_writer_factory.h b/src/paimon/core/io/blob_data_file_writer_factory.h new file mode 100644 index 00000000..fd66b263 --- /dev/null +++ b/src/paimon/core/io/blob_data_file_writer_factory.h @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/c/abi.h" +#include "paimon/core/io/data_file_writer.h" +#include "paimon/core/io/data_file_writer_factory.h" +#include "paimon/core/io/single_file_writer_factory.h" +#include "paimon/format/blob/blob_format_writer.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class DataFilePathFactory; +class LongCounter; +class MemoryPool; + +class BlobDataFileWriterFactory + : public DataFileWriterFactory, + public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> { + public: + using PathCreator = std::function; + + BlobDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& file_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, + const std::shared_ptr& path_factory, + PathCreator path_creator, + blob::BlobFormatWriter::WriteConsumer write_consumer, + const std::shared_ptr& pool); + + Result>>> + CreateWriter() const override; + + private: + std::shared_ptr file_schema_; + std::optional> write_cols_; + std::shared_ptr seq_num_counter_; + std::shared_ptr path_factory_; + PathCreator path_creator_; + blob::BlobFormatWriter::WriteConsumer write_consumer_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/data_file_writer_factory.cpp b/src/paimon/core/io/data_file_writer_factory.cpp new file mode 100644 index 00000000..b929dde8 --- /dev/null +++ b/src/paimon/core/io/data_file_writer_factory.cpp @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/data_file_writer_factory.h" + +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" +#include "paimon/format/file_format.h" +#include "paimon/format/writer_builder.h" + +namespace paimon { + +DataFileWriterFactory::DataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& pool) + : options_(options), schema_id_(schema_id), pool_(pool) {} + +Result DataFileWriterFactory::CreateWriterResources( + const FileFormat& format, const std::shared_ptr& file_schema, + bool create_stats_extractor) const { + WriterResources resources; + { + ::ArrowSchema arrow_schema; + ArrowSchemaMarkReleased(&arrow_schema); + ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); + PAIMON_ASSIGN_OR_RAISE( + resources.writer_builder, + format.CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); + resources.writer_builder->WithMemoryPool(pool_); + } + if (create_stats_extractor) { + ::ArrowSchema arrow_schema; + ArrowSchemaMarkReleased(&arrow_schema); + ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); + PAIMON_ASSIGN_OR_RAISE(resources.stats_extractor, + format.CreateStatsExtractor(&arrow_schema)); + } + return resources; +} + +} // namespace paimon diff --git a/src/paimon/core/io/data_file_writer_factory.h b/src/paimon/core/io/data_file_writer_factory.h new file mode 100644 index 00000000..c727b47d --- /dev/null +++ b/src/paimon/core/io/data_file_writer_factory.h @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/core_options.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class FileFormat; +class FormatStatsExtractor; +class MemoryPool; +class WriterBuilder; + +class DataFileWriterFactory { + public: + DataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& pool); + virtual ~DataFileWriterFactory() = default; + + protected: + struct WriterResources { + std::shared_ptr writer_builder; + std::shared_ptr stats_extractor; + }; + + Result CreateWriterResources(const FileFormat& format, + const std::shared_ptr& file_schema, + bool create_stats_extractor) const; + + CoreOptions options_; + int64_t schema_id_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/external_storage_blob_writer.cpp b/src/paimon/core/io/external_storage_blob_writer.cpp index 05b69e74..1ea730ce 100644 --- a/src/paimon/core/io/external_storage_blob_writer.cpp +++ b/src/paimon/core/io/external_storage_blob_writer.cpp @@ -29,12 +29,8 @@ #include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/io/blob_data_file_writer_factory.h" #include "paimon/core/io/data_file_path_factory.h" -#include "paimon/core/io/data_file_writer.h" -#include "paimon/format/blob/blob_writer_builder.h" -#include "paimon/format/file_format.h" -#include "paimon/format/file_format_factory.h" #include "paimon/fs/file_system.h" #include "paimon/memory/memory_pool.h" @@ -64,47 +60,22 @@ ExternalStorageBlobWriter::CreateFieldRollingWriter(FieldWriter* field_writer) { } auto single_field_schema = arrow::schema({field}); - ::ArrowSchema arrow_schema; - ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*single_field_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, - FileFormatFactory::Get("blob", options_.ToMap())); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr writer_builder, - format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); - writer_builder->WithMemoryPool(memory_pool_); - - // Inject WriteConsumer to capture BlobDescriptors during writes - auto blob_writer_builder = std::dynamic_pointer_cast(writer_builder); - if (!blob_writer_builder) { - return Status::Invalid( - "writer_builder cannot be casted to BlobWriterBuilder in ExternalStorageBlobWriter"); - } - blob_writer_builder->WithWriteConsumer( - [field_writer](std::unique_ptr descriptor) -> bool { - field_writer->captured_descriptors.push_back(std::move(descriptor)); - return true; // Always flush for single row. - }); - - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*single_field_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, - format->CreateStatsExtractor(&arrow_schema)); - - std::vector write_cols = {field_writer->field_name}; - auto single_blob_file_writer_creator = [this, writer_builder, stats_extractor, write_cols]() - -> Result>>> { - auto writer = std::make_unique( - /*compression=*/"none", std::function(), schema_id_, - seq_num_counter_, FileSource::Append(), stats_extractor, - path_factory_->IsExternalPath(), write_cols, memory_pool_); - PAIMON_RETURN_NOT_OK(writer->Init( - options_.GetFileSystem(), - path_factory_->NewExternalStorageBlobPath(external_storage_path_), writer_builder)); - return writer; + auto write_consumer = [field_writer](std::unique_ptr descriptor) -> bool { + field_writer->captured_descriptors.push_back(std::move(descriptor)); + return true; // Always flush for single row. }; - return std::make_unique(options_.GetBlobTargetFileSize(), - single_blob_file_writer_creator); + std::vector write_cols = {field_writer->field_name}; + std::shared_ptr path_factory = path_factory_; + std::string external_storage_path = external_storage_path_; + auto writer_factory = std::make_shared( + options_, schema_id_, single_field_schema, write_cols, seq_num_counter_, path_factory, + [path_factory, external_storage_path]() { + return path_factory->NewExternalStorageBlobPath(external_storage_path); + }, + write_consumer, memory_pool_); + + return std::make_unique(options_.GetBlobTargetFileSize(), writer_factory); } Status ExternalStorageBlobWriter::InitializeFieldWritersIfNeeded() { diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.cpp b/src/paimon/core/io/key_value_data_file_writer_factory.cpp new file mode 100644 index 00000000..07d50b98 --- /dev/null +++ b/src/paimon/core/io/key_value_data_file_writer_factory.cpp @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/key_value_data_file_writer_factory.h" + +#include +#include + +#include "arrow/c/helpers.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/key_value_data_file_writer.h" +#include "paimon/format/file_format.h" +#include "paimon/fs/file_system.h" + +namespace paimon { + +KeyValueDataFileWriterFactory::KeyValueDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, int32_t level, FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, bool create_stats_extractor, + const std::shared_ptr& pool) + : DataFileWriterFactory(options, schema_id, pool), + write_schema_(write_schema), + level_(level), + file_source_(file_source), + primary_keys_(primary_keys), + path_factory_(path_factory), + create_stats_extractor_(create_stats_extractor) {} + +Result>>> +KeyValueDataFileWriterFactory::CreateWriter() const { + std::function converter = + [](KeyValueBatch key_value_batch, ::ArrowArray* array) -> Status { + ArrowArrayMove(key_value_batch.batch.get(), array); + return Status::OK(); + }; + + auto format = options_.GetWriteFileFormat(level_); + PAIMON_ASSIGN_OR_RAISE(WriterResources resources, + CreateWriterResources(*format, write_schema_, create_stats_extractor_)); + auto writer = std::make_unique( + options_.GetWriteFileCompression(level_), std::move(converter), schema_id_, level_, + file_source_, primary_keys_, resources.stats_extractor, write_schema_, + path_factory_->IsExternalPath(), pool_); + PAIMON_RETURN_NOT_OK( + writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + return std::unique_ptr>>( + std::move(writer)); +} + +} // namespace paimon diff --git a/src/paimon/core/io/key_value_data_file_writer_factory.h b/src/paimon/core/io/key_value_data_file_writer_factory.h new file mode 100644 index 00000000..6ac50aba --- /dev/null +++ b/src/paimon/core/io/key_value_data_file_writer_factory.h @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/io/data_file_writer_factory.h" +#include "paimon/core/io/single_file_writer_factory.h" +#include "paimon/core/key_value.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class DataFilePathFactory; +class MemoryPool; + +class KeyValueDataFileWriterFactory + : public DataFileWriterFactory, + public SingleFileWriterFactory> { + public: + KeyValueDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, int32_t level, + FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, + bool create_stats_extractor, + const std::shared_ptr& pool); + + Result>>> + CreateWriter() const override; + + protected: + std::shared_ptr write_schema_; + int32_t level_; + FileSource file_source_; + std::vector primary_keys_; + std::shared_ptr path_factory_; + bool create_stats_extractor_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/map_shared_shredding_core_utils.cpp b/src/paimon/core/io/map_shared_shredding_core_utils.cpp new file mode 100644 index 00000000..38c9518b --- /dev/null +++ b/src/paimon/core/io/map_shared_shredding_core_utils.cpp @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/map_shared_shredding_core_utils.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/format/file_format.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/format/reader_builder.h" +#include "paimon/fs/file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/file_batch_reader.h" + +namespace paimon { +namespace { + +bool ContainsWriteColumn(const std::vector& write_cols, const std::string& field) { + return std::find(write_cols.begin(), write_cols.end(), field) != write_cols.end(); +} + +Result> ReadFileSchema( + const std::shared_ptr& file, + const std::shared_ptr& path_factory, const CoreOptions& options, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::string format_str, file->FileFormat()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, + FileFormatFactory::Get(format_str, options.ToMap())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, + format->CreateReaderBuilder(options.GetReadBatchSize())); + reader_builder->WithMemoryPool(pool); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, + options.GetFileSystem()->Open(path_factory->ToPath(file))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + reader_builder->Build(input_stream)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, + arrow::ImportSchema(c_file_schema.get())); + return file_schema; +} + +// Restores each shared-shredding field from the newest data file that carries its file metadata. +// In data evolution mode, the newest file may only contain a subset of write_cols, so restoring +// from files.back() alone can miss other shared-shredding fields. Use write_cols to avoid opening +// unrelated files, and fall back to Kmax for fields whose metadata cannot be found. +Status RestoreContextFromRecentFiles(const std::vector>& files, + const std::shared_ptr& path_factory, + const CoreOptions& options, + const std::shared_ptr& pool, + MapSharedShreddingContext* context) { + if (!context || files.empty()) { + return Status::OK(); + } + + std::vector shredding_fields = context->GetShreddingColumnNames(); + std::set pending_fields(shredding_fields.begin(), shredding_fields.end()); + + for (auto file_it = files.rbegin(); file_it != files.rend() && !pending_fields.empty(); + ++file_it) { + const auto& file = *file_it; + std::vector candidate_fields; + if (!file->write_cols) { + candidate_fields.assign(pending_fields.begin(), pending_fields.end()); + } else { + for (const auto& field : pending_fields) { + if (ContainsWriteColumn(file->write_cols.value(), field)) { + candidate_fields.push_back(field); + } + } + } + if (candidate_fields.empty()) { + continue; + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, + ReadFileSchema(file, path_factory, options, pool)); + for (const auto& field_name : candidate_fields) { + std::shared_ptr field = file_schema->GetFieldByName(field_name); + if (!field) { + continue; + } + const auto& metadata = field->metadata(); + if (!metadata) { + continue; + } + auto metadata_copy = metadata->Copy(); + if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata_copy)) { + continue; + } + PAIMON_ASSIGN_OR_RAISE( + MapSharedShreddingFieldMeta field_meta, + MapSharedShreddingUtils::DeserializeMetadata( + metadata_copy, MapSharedShreddingDefine::kDefaultDictCompression)); + context->ReportFileStats(field->name(), field_meta.max_row_width); + pending_fields.erase(field_name); + } + } + return Status::OK(); +} + +} // namespace + +Result> +MapSharedShreddingCoreUtils::CreateAndRestoreContext( + const std::shared_ptr& write_schema, + const std::vector>& restore_files, + const std::shared_ptr& path_factory, const CoreOptions& options, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); + PAIMON_RETURN_NOT_OK( + RestoreContextFromRecentFiles(restore_files, path_factory, options, pool, context.get())); + return context; +} + +} // namespace paimon diff --git a/src/paimon/core/io/map_shared_shredding_core_utils.h b/src/paimon/core/io/map_shared_shredding_core_utils.h new file mode 100644 index 00000000..413d6347 --- /dev/null +++ b/src/paimon/core/io/map_shared_shredding_core_utils.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +struct DataFileMeta; +class DataFilePathFactory; +class MapSharedShreddingContext; +class MemoryPool; + +class MapSharedShreddingCoreUtils { + public: + MapSharedShreddingCoreUtils() = delete; + ~MapSharedShreddingCoreUtils() = delete; + + static Result> CreateAndRestoreContext( + const std::shared_ptr& write_schema, + const std::vector>& restore_files, + const std::shared_ptr& path_factory, const CoreOptions& options, + const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/core/io/rolling_blob_file_writer.cpp b/src/paimon/core/io/rolling_blob_file_writer.cpp index 40c362f8..06eb1199 100644 --- a/src/paimon/core/io/rolling_blob_file_writer.cpp +++ b/src/paimon/core/io/rolling_blob_file_writer.cpp @@ -42,13 +42,12 @@ class DataType; namespace paimon { RollingBlobFileWriter::RollingBlobFileWriter( - int64_t target_file_size, - std::function>()> create_file_writer, + int64_t target_file_size, const std::shared_ptr& writer_factory, const std::shared_ptr& blob_schema, MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator, const std::shared_ptr& data_type, const std::set& inline_fields) : RollingFileWriter<::ArrowArray*, std::shared_ptr>(target_file_size, - create_file_writer), + writer_factory), blob_schema_(blob_schema), blob_writer_creator_(std::move(blob_writer_creator)), data_type_(data_type), @@ -124,7 +123,8 @@ Result>> RollingBlobFileWriter::GetRes Result> RollingBlobFileWriter::CloseMainWriter() { PAIMON_RETURN_NOT_OK(current_writer_->Close()); - PAIMON_ASSIGN_OR_RAISE(auto abort_executor, current_writer_->GetAbortExecutor()); + PAIMON_ASSIGN_OR_RAISE(MainWriter::AbortExecutor abort_executor, + current_writer_->GetAbortExecutor()); closed_writers_.push_back(abort_executor); return current_writer_->GetResult(); } diff --git a/src/paimon/core/io/rolling_blob_file_writer.h b/src/paimon/core/io/rolling_blob_file_writer.h index a3f10383..70094581 100644 --- a/src/paimon/core/io/rolling_blob_file_writer.h +++ b/src/paimon/core/io/rolling_blob_file_writer.h @@ -60,9 +60,10 @@ class RollingBlobFileWriter : public RollingFileWriter<::ArrowArray*, std::shared_ptr> { public: using MainWriter = SingleFileWriter<::ArrowArray*, std::shared_ptr>; + using MainWriterFactory = SingleFileWriterFactory<::ArrowArray*, std::shared_ptr>; RollingBlobFileWriter(int64_t target_file_size, - std::function>()> create_file_writer, + const std::shared_ptr& writer_factory, const std::shared_ptr& blob_schema, MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator, const std::shared_ptr& data_type, diff --git a/src/paimon/core/io/rolling_file_writer.h b/src/paimon/core/io/rolling_file_writer.h index fa8a51df..71d04cc3 100644 --- a/src/paimon/core/io/rolling_file_writer.h +++ b/src/paimon/core/io/rolling_file_writer.h @@ -19,13 +19,14 @@ #pragma once #include -#include +#include #include #include "arrow/c/bridge.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/core/io/file_writer.h" #include "paimon/core/io/single_file_writer.h" +#include "paimon/core/io/single_file_writer_factory.h" #include "paimon/core/key_value.h" #include "paimon/metrics.h" #include "paimon/record_batch.h" @@ -36,11 +37,10 @@ namespace paimon { template class RollingFileWriter : public FileWriter> { public: - RollingFileWriter( - int64_t target_file_size, - std::function>>()> create_file_writer) + RollingFileWriter(int64_t target_file_size, + const std::shared_ptr>& writer_factory) : target_file_size_(target_file_size), - create_file_writer(create_file_writer), + writer_factory_(writer_factory), metrics_(std::make_shared()), logger_(Logger::GetLogger("RollingFileWriter")) {} @@ -72,7 +72,7 @@ class RollingFileWriter : public FileWriter> { Status OpenCurrentWriter(); int64_t target_file_size_ = 0; - std::function>>()> create_file_writer; + std::shared_ptr> writer_factory_; std::shared_ptr metrics_; int64_t record_count_ = 0; @@ -139,7 +139,7 @@ Result> RollingFileWriter::GetResult() { template Result>> RollingFileWriter::NewWriter() { - return create_file_writer(); + return writer_factory_->CreateWriter(); } template diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp new file mode 100644 index 00000000..d319e08f --- /dev/null +++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/shredding_append_data_file_writer_factory.h" + +#include + +#include "arrow/c/helpers.h" +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/data_file_writer.h" +#include "paimon/fs/file_system.h" + +namespace paimon { + +ShreddingAppendDataFileWriterFactory::ShreddingAppendDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, FileSource file_source, + const std::shared_ptr& path_factory, + const std::shared_ptr& shredding_context, + const std::shared_ptr& pool) + : AppendDataFileWriterFactory(options, schema_id, write_schema, write_cols, seq_num_counter, + file_source, path_factory, pool), + shredding_context_(shredding_context) {} + +Result>>> +ShreddingAppendDataFileWriterFactory::CreateWriter() const { + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, + MapSharedShreddingBatchConverter::CreateConverter( + write_schema_, shredding_context_, pool_)); + if (!bundle.converter || !bundle.physical_schema) { + return Status::Invalid( + "Shared-shredding append writer requires a converter and physical schema."); + } + std::shared_ptr file_schema = bundle.physical_schema; + auto converter = bundle.converter; + std::function batch_converter = + [converter](::ArrowArray* input, ::ArrowArray* output) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowArray> physical, converter->Convert(input)); + ArrowArrayMove(physical.get(), output); + return Status::OK(); + }; + PAIMON_ASSIGN_OR_RAISE(WriterResources resources, + CreateWriterResources(*options_.GetFileFormat(), file_schema, + /*create_stats_extractor=*/true)); + auto writer = std::make_unique( + options_.GetFileCompression(), std::move(batch_converter), schema_id_, seq_num_counter_, + file_source_, resources.stats_extractor, path_factory_->IsExternalPath(), write_cols_, + pool_); + PAIMON_RETURN_NOT_OK( + writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( + bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, shredding_context_, + file_schema)); + return std::unique_ptr>>( + std::move(writer)); +} + +} // namespace paimon diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.h b/src/paimon/core/io/shredding_append_data_file_writer_factory.h new file mode 100644 index 00000000..3c46a0be --- /dev/null +++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.h @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/io/append_data_file_writer_factory.h" + +namespace paimon { + +class DataFilePathFactory; +class LongCounter; +class MapSharedShreddingContext; +class MemoryPool; + +class ShreddingAppendDataFileWriterFactory : public AppendDataFileWriterFactory { + public: + ShreddingAppendDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, + const std::optional>& write_cols, + const std::shared_ptr& seq_num_counter, FileSource file_source, + const std::shared_ptr& path_factory, + const std::shared_ptr& shredding_context, + const std::shared_ptr& pool); + + Result>>> + CreateWriter() const override; + + private: + std::shared_ptr shredding_context_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp new file mode 100644 index 00000000..412f3b4c --- /dev/null +++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" + +#include +#include + +#include "arrow/c/helpers.h" +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/key_value_data_file_writer.h" +#include "paimon/format/file_format.h" +#include "paimon/fs/file_system.h" + +namespace paimon { + +ShreddingKeyValueDataFileWriterFactory::ShreddingKeyValueDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, int32_t level, FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, bool create_stats_extractor, + const std::shared_ptr& shredding_context, + const std::shared_ptr& pool) + : KeyValueDataFileWriterFactory(options, schema_id, write_schema, level, file_source, + primary_keys, path_factory, create_stats_extractor, pool), + shredding_context_(shredding_context) {} + +Result>>> +ShreddingKeyValueDataFileWriterFactory::CreateWriter() const { + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, + MapSharedShreddingBatchConverter::CreateConverter( + write_schema_, shredding_context_, pool_)); + if (!bundle.converter || !bundle.physical_schema) { + return Status::Invalid( + "Shared-shredding key-value writer requires a converter and physical schema."); + } + std::shared_ptr file_schema = bundle.physical_schema; + auto converter = bundle.converter; + std::function batch_converter = + [converter](KeyValueBatch key_value_batch, ::ArrowArray* array) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowArray> physical, + converter->Convert(key_value_batch.batch.get())); + ArrowArrayMove(physical.get(), array); + return Status::OK(); + }; + + auto format = options_.GetWriteFileFormat(level_); + PAIMON_ASSIGN_OR_RAISE(WriterResources resources, + CreateWriterResources(*format, file_schema, create_stats_extractor_)); + auto writer = std::make_unique( + options_.GetWriteFileCompression(level_), std::move(batch_converter), schema_id_, level_, + file_source_, primary_keys_, resources.stats_extractor, file_schema, + path_factory_->IsExternalPath(), pool_); + PAIMON_RETURN_NOT_OK( + writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); + writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( + bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, shredding_context_, + file_schema)); + return std::unique_ptr>>( + std::move(writer)); +} + +} // namespace paimon diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h new file mode 100644 index 00000000..2e2d654f --- /dev/null +++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/core/io/key_value_data_file_writer_factory.h" + +namespace paimon { + +class DataFilePathFactory; +class MapSharedShreddingContext; +class MemoryPool; + +class ShreddingKeyValueDataFileWriterFactory : public KeyValueDataFileWriterFactory { + public: + ShreddingKeyValueDataFileWriterFactory( + const CoreOptions& options, int64_t schema_id, + const std::shared_ptr& write_schema, int32_t level, FileSource file_source, + const std::vector& primary_keys, + const std::shared_ptr& path_factory, bool create_stats_extractor, + const std::shared_ptr& shredding_context, + const std::shared_ptr& pool); + + Result>>> + CreateWriter() const override; + + private: + std::shared_ptr shredding_context_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/single_file_writer_factory.h b/src/paimon/core/io/single_file_writer_factory.h new file mode 100644 index 00000000..8ec8ecf0 --- /dev/null +++ b/src/paimon/core/io/single_file_writer_factory.h @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/core/io/single_file_writer.h" +#include "paimon/result.h" + +namespace paimon { + +template +class SingleFileWriterFactory { + public: + virtual ~SingleFileWriterFactory() = default; + + virtual Result>> CreateWriter() const = 0; +}; + +} // namespace paimon diff --git a/src/paimon/core/manifest/manifest_entry_writer_factory.h b/src/paimon/core/manifest/manifest_entry_writer_factory.h new file mode 100644 index 00000000..49620225 --- /dev/null +++ b/src/paimon/core/manifest/manifest_entry_writer_factory.h @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "arrow/c/abi.h" +#include "paimon/core/io/single_file_writer_factory.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/manifest/manifest_entry_writer.h" +#include "paimon/core/manifest/manifest_file_meta.h" +#include "paimon/core/utils/path_factory.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class FileSystem; +class ManifestEntryWriter; +class MemoryPool; +class PathFactory; +class WriterBuilder; + +class ManifestEntryWriterFactory + : public SingleFileWriterFactory { + public: + ManifestEntryWriterFactory(const std::string& compression, + std::function converter, + const std::shared_ptr& pool, + const std::shared_ptr& partition_type, + const std::shared_ptr& file_system, + const std::shared_ptr& path_factory, + const std::shared_ptr& writer_builder) + : compression_(compression), + converter_(std::move(converter)), + pool_(pool), + partition_type_(partition_type), + file_system_(file_system), + path_factory_(path_factory), + writer_builder_(writer_builder) {} + + Result>> CreateWriter() + const override { + auto writer = + std::make_unique(compression_, converter_, pool_, partition_type_); + PAIMON_RETURN_NOT_OK(writer->Init(file_system_, path_factory_->NewPath(), writer_builder_)); + return std::unique_ptr>( + std::move(writer)); + } + + private: + std::string compression_; + std::function converter_; + std::shared_ptr pool_; + std::shared_ptr partition_type_; + std::shared_ptr file_system_; + std::shared_ptr path_factory_; + std::shared_ptr writer_builder_; +}; + +} // namespace paimon diff --git a/src/paimon/core/manifest/manifest_file.cpp b/src/paimon/core/manifest/manifest_file.cpp index acdd2de4..a952888c 100644 --- a/src/paimon/core/manifest/manifest_file.cpp +++ b/src/paimon/core/manifest/manifest_file.cpp @@ -27,7 +27,7 @@ #include "paimon/core/io/rolling_file_writer.h" #include "paimon/core/manifest/manifest_entry.h" #include "paimon/core/manifest/manifest_entry_serializer.h" -#include "paimon/core/manifest/manifest_entry_writer.h" +#include "paimon/core/manifest/manifest_entry_writer_factory.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/object_serializer.h" @@ -102,16 +102,12 @@ Result> ManifestFile::Write( return Status::OK(); }; - auto create_file_writer = [&]() -> Result> { - auto writer = std::make_unique(options_.GetManifestCompression(), - converter, pool_, partition_type_); - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder_)); - return writer; - }; + auto writer_factory = std::make_shared( + options_.GetManifestCompression(), converter, pool_, partition_type_, + options_.GetFileSystem(), path_factory_, writer_builder_); std::unique_ptr> writer = std::make_unique>( - target_file_size_, create_file_writer); + target_file_size_, writer_factory); for (const auto& entry : entries) { auto s = writer->Write(entry); if (!s.ok()) { diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp index bb415bb0..f6b71311 100644 --- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp @@ -82,9 +82,11 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( GenerateKeyValueConsumer()); std::vector> reader_holders; + auto before = ExtractFilesFromSections(sections); std::unique_ptr compact_file_writer; if (rewrite_compact_file) { - compact_file_writer = CreateRollingRowWriter(output_level); + PAIMON_RETURN_NOT_OK(RestoreShreddingContextFromFiles(before)); + PAIMON_ASSIGN_OR_RAISE(compact_file_writer, CreateRollingRowWriter(output_level)); } // TODO(xinyu.lxy): produce changelog ScopeGuard write_guard([&]() -> void { @@ -104,7 +106,6 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog( if (compact_file_writer) { PAIMON_RETURN_NOT_OK(compact_file_writer->Close()); } - auto before = ExtractFilesFromSections(sections); std::vector> after; if (compact_file_writer) { PAIMON_ASSIGN_OR_RAISE(after, compact_file_writer->GetResult()); diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index 662426ed..48e880b8 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -104,7 +104,7 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParamId(), arrow_schema_, options, std::make_shared(), /*io_manager=*/nullptr, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, /*shredding_context=*/nullptr, pool_)); // write data ArrowArray c_src_array; diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index 6c16b0f1..f3f59b82 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -22,21 +22,18 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" -#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" -#include "paimon/common/data/shredding/map_shared_shredding_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" -#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/scope_guard.h" -#include "paimon/core/io/key_value_data_file_writer.h" +#include "paimon/core/io/key_value_data_file_writer_factory.h" #include "paimon/core/io/key_value_meta_projection_consumer.h" #include "paimon/core/io/key_value_record_reader.h" +#include "paimon/core/io/map_shared_shredding_core_utils.h" #include "paimon/core/io/row_to_arrow_array_converter.h" -#include "paimon/core/io/single_file_writer.h" +#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/operation/internal_read_context.h" #include "paimon/format/file_format.h" -#include "paimon/format/writer_builder.h" #include "paimon/read_context.h" namespace paimon { MergeTreeCompactRewriter::MergeTreeCompactRewriter( @@ -141,67 +138,34 @@ std::vector> MergeTreeCompactRewriter::ExtractFile return files; } -std::unique_ptr -MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) { - auto create_file_writer = [this, level]() - -> Result>>> { - // Determine file-level schema. When shredding is active, compute per-file K - // and build a physical schema with MAP columns replaced by STRUCT. - PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, - MapSharedShreddingBatchConverter::CreateConverter( - write_schema_, shredding_context_, pool_)); - std::shared_ptr file_schema = - bundle.physical_schema ? bundle.physical_schema : write_schema_; - - ::ArrowSchema arrow_schema{}; - ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); - auto format = options_.GetWriteFileFormat(level); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr writer_builder, - format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); - writer_builder->WithMemoryPool(pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, - format->CreateStatsExtractor(&arrow_schema)); - // Build the converter that transforms KeyValueBatch to ArrowArray. - // When shredding is active, it performs MAP→STRUCT conversion on the batch data. - std::function kv_converter; - if (bundle.converter) { - auto shredding_converter = bundle.converter; - kv_converter = [shredding_converter](KeyValueBatch key_value_batch, - ArrowArray* array) -> Status { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, - shredding_converter->Convert(key_value_batch.batch.get())); - ArrowArrayMove(physical.get(), array); - return Status::OK(); - }; - } else { - kv_converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { - ArrowArrayMove(key_value_batch.batch.get(), array); - return Status::OK(); - }; - } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, - CreateDataFilePathFactory(format->Identifier())); - - auto writer = std::make_unique( - options_.GetWriteFileCompression(level), kv_converter, schema_id_, level, - FileSource::Compact(), trimmed_primary_keys_, stats_extractor, file_schema, - data_file_path_factory->IsExternalPath(), pool_); - PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), - data_file_path_factory->NewPath(), writer_builder)); - - if (bundle.converter) { - writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( - bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, - shredding_context_, file_schema)); - } +Status MergeTreeCompactRewriter::RestoreShreddingContextFromFiles( + const std::vector>& files) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, + CreateDataFilePathFactory(options_.GetFileFormat()->Identifier())); + PAIMON_ASSIGN_OR_RAISE(shredding_context_, + MapSharedShreddingCoreUtils::CreateAndRestoreContext( + write_schema_, files, data_file_path_factory, options_, pool_)); + return Status::OK(); +} - return writer; - }; +Result> +MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) { + auto format = options_.GetWriteFileFormat(level); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, + CreateDataFilePathFactory(format->Identifier())); + std::shared_ptr>> factory; + if (shredding_context_) { + factory = std::make_shared( + options_, schema_id_, write_schema_, level, FileSource::Compact(), + trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, + shredding_context_, pool_); + } else { + factory = std::make_shared( + options_, schema_id_, write_schema_, level, FileSource::Compact(), + trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, pool_); + } return std::make_unique( - options_.GetTargetFileSize(/*has_primary_key=*/true), create_file_writer); + options_.GetTargetFileSize(/*has_primary_key=*/true), factory); } Result @@ -299,9 +263,12 @@ Result MergeTreeCompactRewriter::RewriteCompaction( int32_t output_level, bool drop_delete, const std::vector>& sections) { PAIMON_ASSIGN_OR_RAISE(MergeTreeCompactRewriter::KeyValueConsumerCreator create_consumer, GenerateKeyValueConsumer()); + auto before = ExtractFilesFromSections(sections); + PAIMON_RETURN_NOT_OK(RestoreShreddingContextFromFiles(before)); std::vector> reader_holders; - auto rolling_writer = CreateRollingRowWriter(output_level); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr rolling_writer, + CreateRollingRowWriter(output_level)); ScopeGuard write_guard([&]() -> void { rolling_writer->Abort(); @@ -318,7 +285,6 @@ Result MergeTreeCompactRewriter::RewriteCompaction( PAIMON_RETURN_NOT_OK(rolling_writer->Close()); - auto before = ExtractFilesFromSections(sections); NotifyRewriteCompactBefore(before); PAIMON_ASSIGN_OR_RAISE(std::vector> after, rolling_writer->GetResult()); diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h index a42e26e9..a6af11d7 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h @@ -73,6 +73,9 @@ class MergeTreeCompactRewriter : public CompactRewriter { static std::vector> ExtractFilesFromSections( const std::vector>& sections); + Status RestoreShreddingContextFromFiles( + const std::vector>& files); + MergeTreeCompactRewriter(const BinaryRow& partition, int32_t bucket, int64_t schema_id, const std::vector& trimmed_primary_keys, const CoreOptions& options, @@ -92,7 +95,7 @@ class MergeTreeCompactRewriter : public CompactRewriter { using KeyValueConsumerCreator = AsyncKeyValueProducerAndConsumer::ConsumerCreator; - std::unique_ptr CreateRollingRowWriter(int32_t level); + Result> CreateRollingRowWriter(int32_t level); Result GenerateKeyValueConsumer() const; diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp index cd8f3e18..c28ea61d 100644 --- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp +++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp @@ -84,7 +84,8 @@ class RemoteLookupFileManagerTest : public testing::Test { std::vector({"key"}), data_path_factory, key_comparator, /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper, /*schema_id=*/0, arrow_schema_, options, noop_compact_manager_, - /*io_manager=*/nullptr, /*enable_multi_thread_spill=*/false, pool_)); + /*io_manager=*/nullptr, /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_)); ArrowArray c_src_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*src_array, &c_src_array)); diff --git a/src/paimon/core/mergetree/lookup_levels_test.cpp b/src/paimon/core/mergetree/lookup_levels_test.cpp index a136b625..ee1d0b4e 100644 --- a/src/paimon/core/mergetree/lookup_levels_test.cpp +++ b/src/paimon/core/mergetree/lookup_levels_test.cpp @@ -88,7 +88,8 @@ class LookupLevelsTest : public testing::Test { std::vector({"key"}), data_path_factory, key_comparator, /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper, /*schema_id=*/0, arrow_schema_, options, noop_compact_manager_, - /*io_manager=*/nullptr, /*enable_multi_thread_spill=*/false, pool_)); + /*io_manager=*/nullptr, /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_)); // write data ArrowArray c_src_array; diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp index 459a0055..f6a3b424 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer.cpp @@ -27,10 +27,6 @@ #include "arrow/api.h" #include "arrow/c/abi.h" #include "arrow/c/helpers.h" -#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" -#include "paimon/common/data/shredding/map_shared_shredding_context.h" -#include "paimon/common/data/shredding/map_shared_shredding_utils.h" -#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -40,20 +36,17 @@ #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/data_increment.h" -#include "paimon/core/io/key_value_data_file_writer.h" +#include "paimon/core/io/key_value_data_file_writer_factory.h" #include "paimon/core/io/key_value_meta_projection_consumer.h" #include "paimon/core/io/key_value_record_reader.h" #include "paimon/core/io/row_to_arrow_array_converter.h" -#include "paimon/core/io/single_file_writer.h" +#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/mergetree/compact/sort_merge_reader_with_loser_tree.h" #include "paimon/core/mergetree/write_buffer.h" #include "paimon/core/utils/commit_increment.h" -#include "paimon/format/file_format.h" -#include "paimon/format/writer_builder.h" namespace paimon { -class FormatStatsExtractor; Result> MergeTreeWriter::Create( int64_t last_sequence_number, const std::vector& trimmed_primary_keys, @@ -64,12 +57,10 @@ Result> MergeTreeWriter::Create( int64_t schema_id, const std::shared_ptr& value_schema, const CoreOptions& options, const std::shared_ptr& compact_manager, const std::shared_ptr& io_manager, bool enable_multi_thread_spill, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool) { auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context, - MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); - PAIMON_ASSIGN_OR_RAISE( std::unique_ptr write_buffer, WriteBuffer::Create(last_sequence_number, value_schema, trimmed_primary_keys, @@ -331,61 +322,19 @@ Result MergeTreeWriter::DrainIncrement() { std::unique_ptr>> MergeTreeWriter::CreateRollingRowWriter() const { - auto create_file_writer = [this]() - -> Result>>> { - // Determine file-level schema. When shredding is active, compute per-file K - // and build a physical schema with MAP columns replaced by STRUCT. - PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, - MapSharedShreddingBatchConverter::CreateConverter( - write_schema_, shredding_context_, pool_)); - std::shared_ptr file_schema = - bundle.physical_schema ? bundle.physical_schema : write_schema_; - - ::ArrowSchema arrow_schema; - ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); - auto format = options_.GetWriteFileFormat(/*level=*/0); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr writer_builder, - format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); - writer_builder->WithMemoryPool(pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, - format->CreateStatsExtractor(&arrow_schema)); - // Build the converter that transforms KeyValueBatch to ArrowArray. - // When shredding is active, it performs MAP→STRUCT conversion on the batch data. - std::function kv_converter; - if (bundle.converter) { - auto converter = bundle.converter; - kv_converter = [converter](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, - converter->Convert(key_value_batch.batch.get())); - ArrowArrayMove(physical.get(), array); - return Status::OK(); - }; - } else { - kv_converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { - ArrowArrayMove(key_value_batch.batch.get(), array); - return Status::OK(); - }; - } - auto writer = std::make_unique( - options_.GetWriteFileCompression(0), kv_converter, schema_id_, /*level=*/0, - FileSource::Append(), trimmed_primary_keys_, stats_extractor, file_schema, - path_factory_->IsExternalPath(), pool_); - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder)); - - if (bundle.converter) { - writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( - bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, - shredding_context_, file_schema)); - } - - return writer; - }; + std::shared_ptr>> factory; + if (shredding_context_) { + factory = std::make_shared( + options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, + shredding_context_, pool_); + } else { + factory = std::make_shared( + options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, pool_); + } return std::make_unique>>( - options_.GetTargetFileSize(/*has_primary_key=*/true), create_file_writer); + options_.GetTargetFileSize(/*has_primary_key=*/true), factory); } } // namespace paimon diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h index b437e6ad..d522f679 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.h +++ b/src/paimon/core/mergetree/merge_tree_writer.h @@ -66,6 +66,7 @@ class MergeTreeWriter : public BatchWriter { int64_t schema_id, const std::shared_ptr& value_schema, const CoreOptions& options, const std::shared_ptr& compact_manager, const std::shared_ptr& io_manager, bool enable_multi_thread_spill, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool); Status Write(std::unique_ptr&& batch) override; diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp index cce77330..685e6569 100644 --- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp @@ -210,7 +210,8 @@ class MergeTreeWriterTest : public ::testing::TestWithParam { return MergeTreeWriter::Create( last_sequence_number, primary_keys_, path_factory, key_comparator_, user_defined_seq_comparator, merge_function_wrapper_, schema_id, value_schema_, options, - writer_compact_manager, io_manager, /*enable_multi_thread_spill=*/false, pool_); + writer_compact_manager, io_manager, /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_); } private: @@ -399,9 +400,12 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) { }; auto value_schema = DataField::ConvertDataFieldsToArrowSchema(value_fields); auto value_type = DataField::ConvertDataFieldsToArrowStructType(value_fields); + auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema); ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator, FieldsComparator::Create({value_fields[0]}, /*is_ascending_order=*/true)); + ASSERT_OK_AND_ASSIGN(auto shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); ASSERT_OK_AND_ASSIGN( auto merge_writer, @@ -411,7 +415,7 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) { /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/5, value_schema, options, noop_compact_manager_, GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, shredding_context, pool_)); // Each batch contains duplicated primary keys. DeduplicateMergeFunction should keep the // latest sequence number for each key across and within batches. @@ -444,7 +448,6 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) { ASSERT_OK_AND_ASSIGN(std::unique_ptr data_file_status, options.GetFileSystem()->GetFileStatus(expected_data_file_path)); - auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema); std::map column_to_k = {{"tags", 3}}; ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( write_schema, column_to_k)); @@ -511,6 +514,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator, FieldsComparator::Create({value_fields[0]}, /*is_ascending_order=*/true)); + ASSERT_OK_AND_ASSIGN(auto shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); ASSERT_OK_AND_ASSIGN( auto merge_writer, @@ -520,7 +525,7 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0, value_schema, options, noop_compact_manager_, GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, shredding_context, pool_)); auto array1 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([ [1, [["a", 10], ["b", 20]], [["x", "v1"]]], @@ -1360,7 +1365,8 @@ TEST_F(MergeTreeWriterTest, TestSpillWithSameKeyDeduplicate) { key_comparator_, /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0, value_schema_, options, noop_compact_manager_, io_manager, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_)); std::shared_ptr batch1 = arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([ @@ -1428,7 +1434,8 @@ TEST_F(MergeTreeWriterTest, TestIntermediateMergeSpillFileBound) { key_comparator_, /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0, value_schema_, options, noop_compact_manager_, io_manager, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_)); std::shared_ptr batch1 = arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([ @@ -1494,7 +1501,8 @@ TEST_F(MergeTreeWriterTest, TestDiskQuotaExhaustedFallsBackToFlushWriteBuffer) { key_comparator_, /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0, value_schema_, options, noop_compact_manager_, io_manager, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_)); // Phase 1: Manual FlushMemory path — disk quota exhausted causes fallback. std::shared_ptr array1 = @@ -1572,7 +1580,8 @@ TEST_F(MergeTreeWriterTest, TestFlushMemoryQuotaExhaustedFallsBackToFlushWriteBu key_comparator_, /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0, value_schema_, options, noop_compact_manager_, io_manager, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([ @@ -1616,7 +1625,8 @@ TEST_F(MergeTreeWriterTest, TestCloseDeletesSpillTempFiles) { key_comparator_, /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0, value_schema_, options, noop_compact_manager_, io_manager, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_)); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([ @@ -1649,7 +1659,8 @@ TEST_F(MergeTreeWriterTest, TestMultiplePrepareCommitWithSpill) { key_comparator_, /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0, value_schema_, options, noop_compact_manager_, io_manager, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_)); std::shared_ptr array1 = arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([ @@ -1727,7 +1738,8 @@ TEST_F(MergeTreeWriterTest, TestSpillWithIOException) { key_comparator_, /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0, value_schema_, options, noop_compact_manager_, io_manager, - /*enable_multi_thread_spill=*/false, pool_)); + /*enable_multi_thread_spill=*/false, + /*shredding_context=*/nullptr, pool_)); ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index c7b6355a..ed3401ce 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -24,19 +24,18 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "paimon/common/data/binary_row.h" -#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" -#include "paimon/common/data/shredding/map_shared_shredding_utils.h" -#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/core/append/append_only_writer.h" #include "paimon/core/append/bucketed_append_compact_manager.h" #include "paimon/core/compact/noop_compact_manager.h" #include "paimon/core/core_options.h" +#include "paimon/core/io/append_data_file_writer_factory.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/io/data_file_path_factory.h" -#include "paimon/core/io/data_file_writer.h" +#include "paimon/core/io/map_shared_shredding_core_utils.h" #include "paimon/core/io/rolling_file_writer.h" +#include "paimon/core/io/shredding_append_data_file_writer_factory.h" #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_list.h" #include "paimon/core/operation/append_only_file_store_scan.h" @@ -120,13 +119,15 @@ Result>> AppendOnlyFileStoreWrite::Com PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, CreateFilesReader(partition, bucket, dv_factory, to_compact)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr shredding_context, - MapSharedShreddingUtils::CreateShreddingContext(write_schema_, options_)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, + file_store_path_factory_->CreateDataFilePathFactory(partition, bucket)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context, + MapSharedShreddingCoreUtils::CreateAndRestoreContext( + write_schema_, to_compact, data_file_path_factory, options_, pool_)); auto rewriter = std::make_unique>>( options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterCreator(partition, bucket, write_schema_, write_cols_, to_compact, + GetDataFileWriterFactory(data_file_path_factory, write_schema_, write_cols_, to_compact, shredding_context)); ScopeGuard reader_guard([&]() { @@ -212,71 +213,30 @@ Result> AppendOnlyFileStoreWrite::CreateWriter( } PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr writer, - AppendOnlyWriter::Create(options_, table_schema_->Id(), write_schema_, write_cols_, - restore_max_seq_number, data_file_path_factory, compact_manager, - pool_)); + std::shared_ptr shredding_context, + MapSharedShreddingCoreUtils::CreateAndRestoreContext( + write_schema_, restore_data_files, data_file_path_factory, options_, pool_)); + auto writer = std::make_unique( + options_, table_schema_->Id(), write_schema_, write_cols_, restore_max_seq_number, + data_file_path_factory, compact_manager, shredding_context, pool_); return std::shared_ptr(std::move(writer)); } -AppendOnlyFileStoreWrite::SingleFileWriterCreator -AppendOnlyFileStoreWrite::GetDataFileWriterCreator( - const BinaryRow& partition, int32_t bucket, const std::shared_ptr& schema, +AppendOnlyFileStoreWrite::WriterFactory AppendOnlyFileStoreWrite::GetDataFileWriterFactory( + const std::shared_ptr& data_file_path_factory, + const std::shared_ptr& schema, const std::optional>& write_cols, const std::vector>& to_compact, const std::shared_ptr& shredding_context) const { - return - [this, partition, bucket, schema, write_cols, to_compact, shredding_context]() - -> Result< - std::unique_ptr>>> { - PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, - MapSharedShreddingBatchConverter::CreateConverter( - schema, shredding_context, pool_)); - std::shared_ptr file_schema = - bundle.physical_schema ? bundle.physical_schema : schema; - - ::ArrowSchema arrow_schema; - ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); - auto format = options_.GetFileFormat(); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr writer_builder, - format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); - writer_builder->WithMemoryPool(pool_); - - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr stats_extractor, - format->CreateStatsExtractor(&arrow_schema)); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr data_file_path_factory, - file_store_path_factory_->CreateDataFilePathFactory(partition, bucket)); - - std::function batch_converter; - if (bundle.converter) { - auto converter = bundle.converter; - batch_converter = [converter](ArrowArray* input, ArrowArray* output) -> Status { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, - converter->Convert(input)); - ArrowArrayMove(physical.get(), output); - return Status::OK(); - }; - } - - auto writer = std::make_unique( - options_.GetFileCompression(), batch_converter, table_schema_->Id(), - std::make_shared(to_compact[0]->min_sequence_number), - FileSource::Compact(), stats_extractor, data_file_path_factory->IsExternalPath(), - write_cols, pool_); - PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), - data_file_path_factory->NewPath(), writer_builder)); - - if (bundle.converter) { - writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( - bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, - shredding_context, file_schema)); - } - return writer; - }; + auto seq_num_counter = std::make_shared(to_compact[0]->min_sequence_number); + if (shredding_context) { + return std::make_shared( + options_, table_schema_->Id(), schema, write_cols, seq_num_counter, + FileSource::Compact(), data_file_path_factory, shredding_context, pool_); + } + return std::make_shared( + options_, table_schema_->Id(), schema, write_cols, seq_num_counter, FileSource::Compact(), + data_file_path_factory, pool_); } Result> AppendOnlyFileStoreWrite::CreateFilesReader( diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h index 7fb75fe4..e79bd9b9 100644 --- a/src/paimon/core/operation/append_only_file_store_write.h +++ b/src/paimon/core/operation/append_only_file_store_write.h @@ -31,7 +31,6 @@ #include "paimon/core/compact/cancellation_controller.h" #include "paimon/core/core_options.h" #include "paimon/core/deletionvectors/deletion_vector.h" -#include "paimon/core/io/single_file_writer.h" #include "paimon/core/operation/abstract_file_store_write.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/file_store_write.h" @@ -41,6 +40,7 @@ #include "paimon/type_fwd.h" struct ArrowSchema; +struct ArrowArray; namespace arrow { class Schema; @@ -51,10 +51,13 @@ namespace paimon { struct DataFileMeta; class BatchWriter; class BucketedDvMaintainer; +class DataFilePathFactory; class FileStorePathFactory; class FileStoreScan; class SnapshotManager; class ScanFilter; +template +class SingleFileWriterFactory; class MetricsImpl; class BinaryRow; class CoreOptions; @@ -96,8 +99,8 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { const std::shared_ptr& cancellation_controller); private: - using SingleFileWriterCreator = std::function< - Result>>>()>; + using WriterFactory = + std::shared_ptr>>; Result> CreateWriter( const BinaryRow& partition, int32_t bucket, @@ -108,8 +111,9 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite { Result> CreateFileStoreScan( const std::shared_ptr& filter) const override; - SingleFileWriterCreator GetDataFileWriterCreator( - const BinaryRow& partition, int32_t bucket, const std::shared_ptr& schema, + WriterFactory GetDataFileWriterFactory( + const std::shared_ptr& data_file_path_factory, + const std::shared_ptr& schema, const std::optional>& write_cols, const std::vector>& to_compact, const std::shared_ptr& shredding_context) const; diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index 09b9e28a..daf1fba2 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -20,6 +20,8 @@ #include #include +#include +#include #include #include "arrow/array/array_base.h" @@ -27,20 +29,31 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/ipc/json_simple.h" #include "arrow/status.h" #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/operation/restore_files.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/utils/snapshot_manager.h" +#include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/format/file_format.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/format/reader_builder.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/reader/file_batch_reader.h" #include "paimon/record_batch.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" @@ -68,6 +81,103 @@ class AppendOnlyFileStoreWriteTest : public testing::Test { commit_user_ = "test_commit_user"; } + void CreateTable(const std::string& warehouse, const std::shared_ptr& schema, + const std::map& options) const { + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &c_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + } + + std::unique_ptr MakeBatch(const std::shared_ptr& schema, + const std::string& json) const { + auto struct_type = arrow::struct_(schema->fields()); + auto array = arrow::ipc::internal::json::ArrayFromJSON(struct_type, json).ValueOrDie(); + ::ArrowArray arrow_array; + EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + return batch_builder.SetBucket(0).Finish().value(); + } + + std::vector> WriteAndPrepare( + const std::string& table_path, const std::shared_ptr& schema, + const std::map& options, const std::string& json, + int64_t commit_identifier) const { + WriteContextBuilder builder(table_path, commit_user_); + builder.SetOptions(options); + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); + EXPECT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context))); + EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); + EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( + /*wait_compaction=*/false, commit_identifier)); + EXPECT_OK(file_store_write->Close()); + return commit_msgs; + } + + std::vector> WriteAndPrepareWithWriteSchema( + const std::string& table_path, const std::shared_ptr& schema, + const std::map& options, + const std::vector& write_schema, const std::string& json, + int64_t commit_identifier) const { + WriteContextBuilder builder(table_path, commit_user_); + builder.SetOptions(options).WithWriteSchema(write_schema); + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); + EXPECT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context))); + EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); + EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( + /*wait_compaction=*/false, commit_identifier)); + EXPECT_OK(file_store_write->Close()); + return commit_msgs; + } + + void Commit(const std::string& table_path, const std::map& options, + const std::vector>& commit_msgs) const { + CommitContextBuilder builder(table_path, commit_user_); + builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(file_store_commit->Commit(commit_msgs)); + } + + std::shared_ptr OnlyNewFile( + const std::vector>& commit_msgs) const { + EXPECT_EQ(1, commit_msgs.size()); + auto msg = std::dynamic_pointer_cast(commit_msgs[0]); + EXPECT_NE(nullptr, msg); + EXPECT_EQ(1, msg->GetNewFilesIncrement().NewFiles().size()); + return msg->GetNewFilesIncrement().NewFiles()[0]; + } + + std::shared_ptr ReadDataFileSchema( + const std::string& table_path, const std::shared_ptr& file, + const std::map& options) const { + std::string file_path = + PathUtil::JoinPath(PathUtil::JoinPath(table_path, "bucket-0"), file->file_name); + auto fs = std::make_shared(); + EXPECT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(file_path)); + EXPECT_OK_AND_ASSIGN(auto format_str, file->FileFormat()); + EXPECT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get(format_str, options)); + EXPECT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(10)); + EXPECT_OK_AND_ASSIGN(auto reader, reader_builder->Build(input_stream)); + EXPECT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema()); + return arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + } + + MapSharedShreddingFieldMeta ShreddingMeta(const std::shared_ptr& file_schema, + int32_t field_index) const { + auto metadata = file_schema->field(field_index)->metadata(); + EXPECT_NE(nullptr, metadata); + return MapSharedShreddingUtils::DeserializeMetadata( + metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression) + .value(); + } + private: arrow::FieldVector fields_; std::string commit_user_; @@ -174,4 +284,248 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestGetMaxSequenceNumberFromMultiPartition) } } +TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNextWriter) { + std::map options = { + {"file.format", "parquet"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"write-only", "true"}, + {"bucket", "1"}, + {"bucket-key", "id"}, + }; + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + CreateTable(dir->Str(), logical_schema, options); + + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + auto first_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([ + [1, [["a", 1], ["b", 2]]] + ])", + /*commit_identifier=*/0); + auto first_file_schema = + ReadDataFileSchema(table_path, OnlyNewFile(first_commit_msgs), options); + auto first_meta = ShreddingMeta(first_file_schema, /*field_index=*/1); + ASSERT_EQ(10, first_meta.num_columns); + ASSERT_EQ(2, first_meta.max_row_width); + Commit(table_path, options, first_commit_msgs); + + auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([ + [2, [["c", 3], ["d", 4], ["e", 5]]] + ])", + /*commit_identifier=*/1); + auto second_file_schema = + ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), options); + auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); + + ASSERT_OK_AND_ASSIGN( + auto expected_second_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 2}})); + ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false)); + ASSERT_EQ(2, second_meta.num_columns); + ASSERT_EQ(3, second_meta.max_row_width); +} + +TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreIgnoresAvroFileWithoutMetadata) { + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + std::map avro_options = { + {"file.format", "avro"}, + {"write-only", "true"}, + {"bucket", "1"}, + {"bucket-key", "id"}, + }; + std::map shredding_options = avro_options; + shredding_options["file.format"] = "parquet"; + shredding_options["fields.tags.map.storage-layout"] = "shared-shredding"; + shredding_options["fields.tags.map.shared-shredding.max-columns"] = "10"; + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + CreateTable(dir->Str(), logical_schema, avro_options); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + auto avro_commit_msgs = WriteAndPrepare(table_path, logical_schema, avro_options, R"([ + [1, [["a", 1], ["b", 2]]] + ])", + /*commit_identifier=*/0); + Commit(table_path, avro_options, avro_commit_msgs); + + auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, shredding_options, R"([ + [2, [["c", 3], ["d", 4], ["e", 5]]] + ])", + /*commit_identifier=*/1); + auto second_file_schema = + ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options); + auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); + + ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 10}})); + ASSERT_TRUE(second_file_schema->Equals(*expected_schema, /*check_metadata=*/false)); + ASSERT_EQ(10, second_meta.num_columns); + ASSERT_EQ(3, second_meta.max_row_width); +} + +TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreMultipleMapColumns) { + std::map options = { + {"file.format", "parquet"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.attrs.map.storage-layout", "shared-shredding"}, + {"fields.attrs.map.shared-shredding.max-columns", "10"}, + {"write-only", "true"}, + {"bucket", "1"}, + {"bucket-key", "id"}, + }; + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto tags_schema = arrow::schema({ + logical_schema->field(0), + logical_schema->field(1), + }); + auto attrs_schema = arrow::schema({ + logical_schema->field(0), + logical_schema->field(2), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + CreateTable(dir->Str(), logical_schema, options); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + auto tags_commit_msgs = + WriteAndPrepareWithWriteSchema(table_path, tags_schema, options, {"id", "tags"}, R"([ + [1, [["a", 1], ["b", 2]]] + ])", + /*commit_identifier=*/0); + Commit(table_path, options, tags_commit_msgs); + + auto attrs_commit_msgs = + WriteAndPrepareWithWriteSchema(table_path, attrs_schema, options, {"id", "attrs"}, R"([ + [2, [["c", 3], ["d", 4], ["e", 5], ["f", 6]]] + ])", + /*commit_identifier=*/1); + Commit(table_path, options, attrs_commit_msgs); + + auto full_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([ + [3, [["g", 7], ["h", 8], ["i", 9]], [["j", 10], ["k", 11], ["l", 12], ["m", 13], ["n", 14]]] + ])", + /*commit_identifier=*/2); + auto full_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(full_commit_msgs), options); + auto tags_meta = ShreddingMeta(full_file_schema, /*field_index=*/1); + auto attrs_meta = ShreddingMeta(full_file_schema, /*field_index=*/2); + + ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 2}, {"attrs", 4}})); + ASSERT_TRUE(full_file_schema->Equals(*expected_schema, /*check_metadata=*/false)); + ASSERT_EQ(2, tags_meta.num_columns); + ASSERT_EQ(3, tags_meta.max_row_width); + ASSERT_EQ(4, attrs_meta.num_columns); + ASSERT_EQ(5, attrs_meta.max_row_width); +} + +TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreUsesDefaultForMissingMap) { + std::map options = { + {"file.format", "parquet"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.attrs.map.storage-layout", "shared-shredding"}, + {"fields.attrs.map.shared-shredding.max-columns", "10"}, + {"write-only", "true"}, + {"bucket", "1"}, + {"bucket-key", "id"}, + }; + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto tags_schema = arrow::schema({ + logical_schema->field(0), + logical_schema->field(1), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + CreateTable(dir->Str(), logical_schema, options); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + auto tags_commit_msgs = + WriteAndPrepareWithWriteSchema(table_path, tags_schema, options, {"id", "tags"}, R"([ + [1, [["a", 1], ["b", 2]]] + ])", + /*commit_identifier=*/0); + Commit(table_path, options, tags_commit_msgs); + + auto full_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([ + [2, [["c", 3], ["d", 4], ["e", 5]], [["f", 6], ["g", 7], ["h", 8], ["i", 9]]] + ])", + /*commit_identifier=*/1); + auto full_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(full_commit_msgs), options); + auto tags_meta = ShreddingMeta(full_file_schema, /*field_index=*/1); + auto attrs_meta = ShreddingMeta(full_file_schema, /*field_index=*/2); + + ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 2}, {"attrs", 10}})); + ASSERT_TRUE(full_file_schema->Equals(*expected_schema, /*check_metadata=*/false)); + ASSERT_EQ(2, tags_meta.num_columns); + ASSERT_EQ(3, tags_meta.max_row_width); + ASSERT_EQ(10, attrs_meta.num_columns); + ASSERT_EQ(4, attrs_meta.max_row_width); +} + +TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingPartialWriteSkipsMissingMapColumn) { + std::map options = { + {"file.format", "parquet"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"write-only", "true"}, + {"bucket", "1"}, + {"bucket-key", "id"}, + }; + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto id_only_schema = arrow::schema({ + logical_schema->field(0), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + CreateTable(dir->Str(), logical_schema, options); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + auto first_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([ + [1, [["a", 1], ["b", 2]]] + ])", + /*commit_identifier=*/0); + Commit(table_path, options, first_commit_msgs); + + auto id_only_commit_msgs = + WriteAndPrepareWithWriteSchema(table_path, id_only_schema, options, {"id"}, R"([ + [2] + ])", + /*commit_identifier=*/1); + auto id_only_file_schema = + ReadDataFileSchema(table_path, OnlyNewFile(id_only_commit_msgs), options); + + ASSERT_TRUE(id_only_file_schema->Equals(*id_only_schema, /*check_metadata=*/false)); + for (const auto& field : id_only_file_schema->fields()) { + auto metadata = field->metadata() ? field->metadata()->Copy() : nullptr; + ASSERT_FALSE(MapSharedShreddingUtils::HasShreddingMetadata(metadata)); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp index 08c5ea0c..075d1a96 100644 --- a/src/paimon/core/operation/key_value_file_store_write.cpp +++ b/src/paimon/core/operation/key_value_file_store_write.cpp @@ -21,8 +21,10 @@ #include #include "paimon/common/data/binary_row.h" +#include "paimon/common/table/special_fields.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/map_shared_shredding_core_utils.h" #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_list.h" #include "paimon/core/mergetree/levels.h" @@ -106,6 +108,11 @@ Result> KeyValueFileStoreWrite::CreateWriter( file_store_path_factory_->CreateDataFilePathFactory(partition, bucket)); PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, table_schema_->TrimmedPrimaryKeys()); + auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(schema_); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr shredding_context, + MapSharedShreddingCoreUtils::CreateAndRestoreContext( + write_schema, restore_data_files, data_file_path_factory, options_, pool_)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr levels, Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels())); @@ -117,10 +124,11 @@ Result> KeyValueFileStoreWrite::CreateWriter( PAIMON_ASSIGN_OR_RAISE( std::shared_ptr writer, - MergeTreeWriter::Create( - restore_max_seq_number, trimmed_primary_keys, data_file_path_factory, key_comparator_, - user_defined_seq_comparator_, merge_function_wrapper_, table_schema_->Id(), schema_, - options_, compact_manager, io_manager_, enable_multi_thread_spill_, pool_)); + MergeTreeWriter::Create(restore_max_seq_number, trimmed_primary_keys, + data_file_path_factory, key_comparator_, + user_defined_seq_comparator_, merge_function_wrapper_, + table_schema_->Id(), schema_, options_, compact_manager, + io_manager_, enable_multi_thread_spill_, shredding_context, pool_)); return writer; } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index be4c7153..e37a8289 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -30,13 +31,25 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/ipc/json_simple.h" #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/common/table/special_fields.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/io/data_file_meta.h" #include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/format/file_format.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/format/reader_builder.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/reader/file_batch_reader.h" #include "paimon/record_batch.h" #include "paimon/status.h" #include "paimon/testing/utils/test_helper.h" @@ -99,6 +112,86 @@ class KeyValueFileStoreWriteTest : public ::testing::Test { } return write_status; } + + void CreateTable(const std::string& warehouse, const std::shared_ptr& schema, + const std::map& options) const { + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &c_schema, + /*partition_keys=*/{}, /*primary_keys=*/{"id"}, options, + /*ignore_if_exists=*/false)); + } + + std::unique_ptr MakeBatch(const std::shared_ptr& schema, + const std::string& json) const { + auto struct_type = arrow::struct_(schema->fields()); + auto array = arrow::ipc::internal::json::ArrayFromJSON(struct_type, json).ValueOrDie(); + ::ArrowArray arrow_array; + EXPECT_TRUE(arrow::ExportArray(*array, &arrow_array).ok()); + RecordBatchBuilder batch_builder(&arrow_array); + return batch_builder.SetBucket(0).Finish().value(); + } + + std::vector> WriteAndPrepare( + const std::string& table_path, const std::shared_ptr& schema, + const std::map& options, const std::string& json, + int64_t commit_identifier) const { + WriteContextBuilder builder(table_path, "test"); + builder.SetOptions(options); + EXPECT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish()); + EXPECT_OK_AND_ASSIGN(auto file_store_write, + FileStoreWrite::Create(std::move(write_context))); + EXPECT_OK(file_store_write->Write(MakeBatch(schema, json))); + EXPECT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit( + /*wait_compaction=*/false, commit_identifier)); + EXPECT_OK(file_store_write->Close()); + return commit_msgs; + } + + std::shared_ptr OnlyNewFile( + const std::vector>& commit_msgs) const { + EXPECT_EQ(1, commit_msgs.size()); + auto msg = std::dynamic_pointer_cast(commit_msgs[0]); + EXPECT_NE(nullptr, msg); + EXPECT_EQ(1, msg->GetNewFilesIncrement().NewFiles().size()); + return msg->GetNewFilesIncrement().NewFiles()[0]; + } + + void Commit(const std::string& table_path, const std::map& options, + const std::vector>& commit_msgs) const { + CommitContextBuilder builder(table_path, "test"); + builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(file_store_commit->Commit(commit_msgs)); + } + + std::shared_ptr ReadDataFileSchema( + const std::string& table_path, const std::shared_ptr& file, + const std::map& options) const { + std::string file_path = + PathUtil::JoinPath(PathUtil::JoinPath(table_path, "bucket-0"), file->file_name); + auto fs = std::make_shared(); + EXPECT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs->Open(file_path)); + EXPECT_OK_AND_ASSIGN(auto format_str, file->FileFormat()); + EXPECT_OK_AND_ASSIGN(auto file_format, FileFormatFactory::Get(format_str, options)); + EXPECT_OK_AND_ASSIGN(auto reader_builder, file_format->CreateReaderBuilder(10)); + EXPECT_OK_AND_ASSIGN(auto reader, reader_builder->Build(input_stream)); + EXPECT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema()); + return arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + } + + MapSharedShreddingFieldMeta ShreddingMeta(const std::shared_ptr& file_schema, + int32_t field_index) const { + auto metadata = file_schema->field(field_index)->metadata(); + EXPECT_NE(nullptr, metadata); + return MapSharedShreddingUtils::DeserializeMetadata( + metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression) + .value(); + } }; TEST_F(KeyValueFileStoreWriteTest, TestWriteWithInvalidBatch) { @@ -221,6 +314,53 @@ TEST_F(KeyValueFileStoreWriteTest, ASSERT_EQ(commit_messages.size(), 1); } +TEST_F(KeyValueFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNextWriter) { + std::map options = { + {"file.format", "parquet"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"write-only", "true"}, + {"bucket", "1"}, + {"enable-pk-commit-in-inte-test", ""}, + }; + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(logical_schema); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + CreateTable(dir->Str(), logical_schema, options); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + auto first_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([ + [1, [["a", 1], ["b", 2]]] + ])", + /*commit_identifier=*/0); + auto first_file_schema = + ReadDataFileSchema(table_path, OnlyNewFile(first_commit_msgs), options); + auto first_meta = ShreddingMeta(first_file_schema, /*field_index=*/3); + ASSERT_EQ(10, first_meta.num_columns); + ASSERT_EQ(2, first_meta.max_row_width); + Commit(table_path, options, first_commit_msgs); + + auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([ + [2, [["c", 3], ["d", 4], ["e", 5]]] + ])", + /*commit_identifier=*/1); + auto second_file_schema = + ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), options); + auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/3); + + ASSERT_OK_AND_ASSIGN( + auto expected_second_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 2}})); + ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false)); + ASSERT_EQ(2, second_meta.num_columns); + ASSERT_EQ(3, second_meta.max_row_width); +} + TEST_F(KeyValueFileStoreWriteTest, TestSpillSimple) { auto fields = {arrow::field("f0", arrow::utf8(), /*nullable=*/false)}; arrow::Schema typed_schema(fields); diff --git a/src/paimon/core/postpone/postpone_bucket_file_store_write.h b/src/paimon/core/postpone/postpone_bucket_file_store_write.h index 1ca65d28..567b0267 100644 --- a/src/paimon/core/postpone/postpone_bucket_file_store_write.h +++ b/src/paimon/core/postpone/postpone_bucket_file_store_write.h @@ -25,7 +25,9 @@ #include #include +#include "paimon/common/table/special_fields.h" #include "paimon/common/utils/preconditions.h" +#include "paimon/core/io/map_shared_shredding_core_utils.h" #include "paimon/core/operation/abstract_file_store_write.h" #include "paimon/core/operation/file_store_scan.h" #include "paimon/core/postpone/postpone_bucket_writer.h" @@ -127,10 +129,16 @@ class PostponeBucketFileStoreWrite : public AbstractFileStoreWrite { PAIMON_ASSIGN_OR_RAISE( std::shared_ptr data_file_path_factory, file_store_path_factory_->CreateDataFilePathFactory(partition, bucket)); + auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(schema_); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr shredding_context, + MapSharedShreddingCoreUtils::CreateAndRestoreContext( + write_schema, restore_data_files, data_file_path_factory, options_, pool_)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr writer, PostponeBucketWriter::Create(trimmed_primary_keys, data_file_path_factory, - table_schema_->Id(), schema_, options_, pool_)); + table_schema_->Id(), schema_, options_, shredding_context, + pool_)); return writer; } diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp index 9de632e2..7159e03b 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp @@ -20,7 +20,6 @@ #include #include -#include #include "arrow/api.h" #include "arrow/array/array_base.h" @@ -33,9 +32,6 @@ #include "arrow/scalar.h" #include "arrow/util/checked_cast.h" #include "fmt/format.h" -#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" -#include "paimon/common/data/shredding/map_shared_shredding_utils.h" -#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" @@ -46,8 +42,8 @@ #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/data_increment.h" -#include "paimon/core/io/key_value_data_file_writer.h" -#include "paimon/core/io/single_file_writer.h" +#include "paimon/core/io/key_value_data_file_writer_factory.h" +#include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/utils/commit_increment.h" #include "paimon/format/file_format.h" @@ -77,10 +73,9 @@ Result> PostponeBucketWriter::Create( const std::vector& trimmed_primary_keys, const std::shared_ptr& path_factory, int64_t schema_id, const std::shared_ptr& value_schema, const CoreOptions& options, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool) { auto write_schema = BuildPostponeBucketWriteSchema(value_schema); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context, - MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); return std::unique_ptr( new PostponeBucketWriter(trimmed_primary_keys, path_factory, schema_id, value_schema, write_schema, options, pool, shredding_context)); @@ -267,54 +262,19 @@ PostponeBucketWriter::PrepareMinMaxKey( std::unique_ptr>> PostponeBucketWriter::CreateRollingRowWriter() const { - auto shredding_context = shredding_context_; - auto create_file_writer = [this, shredding_context]() - -> Result>>> { - PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, - MapSharedShreddingBatchConverter::CreateConverter( - write_schema_, shredding_context, pool_)); - std::shared_ptr file_schema = - bundle.physical_schema ? bundle.physical_schema : write_schema_; - - ::ArrowSchema arrow_schema; - ScopeGuard guard([&arrow_schema]() { ArrowSchemaRelease(&arrow_schema); }); - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*file_schema, &arrow_schema)); - auto format = options_.GetWriteFileFormat(/*level=*/0); - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr writer_builder, - format->CreateWriterBuilder(&arrow_schema, options_.GetWriteBatchSize())); - writer_builder->WithMemoryPool(pool_); - std::function converter; - if (bundle.converter) { - auto shredding_converter = bundle.converter; - converter = [shredding_converter](KeyValueBatch key_value_batch, - ArrowArray* array) -> Status { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr physical, - shredding_converter->Convert(key_value_batch.batch.get())); - ArrowArrayMove(physical.get(), array); - return Status::OK(); - }; - } else { - converter = [](KeyValueBatch key_value_batch, ArrowArray* array) -> Status { - ArrowArrayMove(key_value_batch.batch.get(), array); - return Status::OK(); - }; - } - auto writer = std::make_unique( - options_.GetWriteFileCompression(0), converter, schema_id_, /*level=*/0, - FileSource::Append(), trimmed_primary_keys_, /*stats_extractor=*/nullptr, file_schema, - path_factory_->IsExternalPath(), pool_); - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), writer_builder)); - if (bundle.converter) { - writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( - bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, - shredding_context, file_schema)); - } - return writer; - }; + std::shared_ptr>> factory; + if (shredding_context_) { + factory = std::make_shared( + options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, + shredding_context_, pool_); + } else { + factory = std::make_shared( + options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, pool_); + } return std::make_unique>>( - options_.GetTargetFileSize(/*has_primary_key=*/true), create_file_writer); + options_.GetTargetFileSize(/*has_primary_key=*/true), factory); } Status PostponeBucketWriter::Flush() { diff --git a/src/paimon/core/postpone/postpone_bucket_writer.h b/src/paimon/core/postpone/postpone_bucket_writer.h index 78a43ad8..d8c02a2e 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer.h +++ b/src/paimon/core/postpone/postpone_bucket_writer.h @@ -56,6 +56,7 @@ class PostponeBucketWriter : public BatchWriter { const std::vector& trimmed_primary_keys, const std::shared_ptr& path_factory, int64_t schema_id, const std::shared_ptr& value_schema, const CoreOptions& options, + const std::shared_ptr& shredding_context, const std::shared_ptr& pool); ~PostponeBucketWriter() override { diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp index 7679e67e..31814581 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp @@ -175,9 +175,10 @@ TEST_P(PostponeBucketWriterTest, TestSimple) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, - PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, - value_schema_, options, pool_)); + ASSERT_OK_AND_ASSIGN( + auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_, + options, /*shredding_context=*/nullptr, pool_)); // write batch std::shared_ptr array1 = @@ -255,7 +256,8 @@ TEST_P(PostponeBucketWriterTest, TestNestedType) { ASSERT_OK_AND_ASSIGN( auto postpone_bucket_writer, PostponeBucketWriter::Create(std::vector{"key"}, path_factory, /*schema_id=*/1, - arrow::schema(fields), options, pool_)); + arrow::schema(fields), options, + /*shredding_context=*/nullptr, pool_)); // write batch auto array1 = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ @@ -331,11 +333,15 @@ TEST_F(PostponeBucketWriterTest, TestSharedShreddingMap) { auto path_factory = std::make_shared(); ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; + auto value_schema = arrow::schema(fields); + auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema); + ASSERT_OK_AND_ASSIGN(auto shredding_context, + MapSharedShreddingUtils::CreateShreddingContext(write_schema, options)); ASSERT_OK_AND_ASSIGN( auto postpone_bucket_writer, PostponeBucketWriter::Create(std::vector{"key"}, path_factory, /*schema_id=*/1, - arrow::schema(fields), options, pool_)); + value_schema, options, shredding_context, pool_)); auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ ["Lucy", [["a", 1], ["b", 2]]], @@ -394,9 +400,10 @@ TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, - PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, - value_schema_, options, pool_)); + ASSERT_OK_AND_ASSIGN( + auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_, + options, /*shredding_context=*/nullptr, pool_)); // write batch 1, batch size = 3 std::shared_ptr array1 = @@ -492,9 +499,10 @@ TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, - PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, - value_schema_, options, pool_)); + ASSERT_OK_AND_ASSIGN( + auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_, + options, /*shredding_context=*/nullptr, pool_)); // write batch 1, batch size = 3 std::shared_ptr array1 = @@ -622,9 +630,10 @@ TEST_P(PostponeBucketWriterTest, TestPrepareCommitForEmptyData) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, - PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, - value_schema_, options, pool_)); + ASSERT_OK_AND_ASSIGN( + auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_, + options, /*shredding_context=*/nullptr, pool_)); // prepare commit, without write ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment, @@ -663,9 +672,10 @@ TEST_P(PostponeBucketWriterTest, TestCloseBeforePrepareCommit) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, - PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, - value_schema_, options, pool_)); + ASSERT_OK_AND_ASSIGN( + auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_, + options, /*shredding_context=*/nullptr, pool_)); // write batch std::shared_ptr array1 = @@ -696,10 +706,10 @@ TEST_P(PostponeBucketWriterTest, TestIOException) { ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr)); std::string uuid = path_factory->uuid_; - ASSERT_OK_AND_ASSIGN( - auto postpone_bucket_writer, - PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, - value_schema_, options, pool_)); + ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer, + PostponeBucketWriter::Create(primary_keys_, path_factory, + /*schema_id=*/1, value_schema_, options, + /*shredding_context=*/nullptr, pool_)); // write batch std::shared_ptr array = diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index bbbb5b98..95f3eedf 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -135,7 +135,8 @@ const std::vector& SystemTableRegistry() { const std::map& dynamic_options) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); - return std::make_shared(fs, table_path, LoadBranch(options), + auto branch = LoadBranch(options); + return std::make_shared(fs, table_path, std::move(branch), table_schema, std::move(options)); }}, {FilesSystemTable::kName, @@ -144,7 +145,8 @@ const std::vector& SystemTableRegistry() { const std::map& dynamic_options) -> Result> { auto options = MergeOptions(table_schema, dynamic_options); - return std::make_shared(fs, table_path, LoadBranch(options), + auto branch = LoadBranch(options); + return std::make_shared(fs, table_path, std::move(branch), table_schema, std::move(options)); }}, }; diff --git a/src/paimon/format/orc/orc_adapter.cpp b/src/paimon/format/orc/orc_adapter.cpp index 4e58da86..3ca065f7 100644 --- a/src/paimon/format/orc/orc_adapter.cpp +++ b/src/paimon/format/orc/orc_adapter.cpp @@ -357,7 +357,7 @@ class UnPooledStructBuilder : public EmptyBuilder { arrow::FieldVector fields; fields.reserve(children_.size()); for (size_t i = 0; i < children_.size(); i++) { - fields.emplace_back(arrow::field(type_->field(i)->name(), children_[i]->type())); + fields.emplace_back(type_->field(i)->WithType(children_[i]->type())); } return arrow::struct_(fields); } @@ -953,7 +953,7 @@ arrow::Result> NormalizeArray( auto child_length = struct_array->data()->child_data[i]->length; auto child_offset = struct_array->data()->child_data[i]->offset; // field function will change length & offset in child data - std::shared_ptr child = struct_array->field(static_cast(i)); + std::shared_ptr child = struct_array->field(static_cast(i)); const std::shared_ptr child_bitmap = child->null_bitmap(); std::shared_ptr final_child_bitmap; if (child_bitmap == nullptr && bitmap == nullptr) { @@ -1227,7 +1227,7 @@ arrow::Status WriteStructBatch(const arrow::Array& array, for (std::size_t i = 0; i < size; i++) { batch->fields[i]->resize(arrow_length); ARROW_RETURN_NOT_OK( - WriteBatch(*(struct_array->field(static_cast(i))), batch->fields[i])); + WriteBatch(*(struct_array->field(static_cast(i))), batch->fields[i])); } return arrow::Status::OK(); } From f786c9393b144d74e83818b3c0b25f6a3c0aafec Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Wed, 24 Jun 2026 13:49:38 +0800 Subject: [PATCH 071/138] feat: support read for nested type sub column --- include/paimon/read_context.h | 80 +- src/paimon/CMakeLists.txt | 2 + .../common/memory/memory_segment_test.cpp | 5 +- src/paimon/common/types/data_field.cpp | 34 + src/paimon/common/types/data_field.h | 14 + .../global_index/global_index_write_task.cpp | 2 +- src/paimon/core/io/field_mapping_reader.cpp | 190 +- src/paimon/core/io/field_mapping_reader.h | 27 +- .../core/io/field_mapping_reader_test.cpp | 42 +- .../core/operation/abstract_split_read.cpp | 8 +- .../core/operation/internal_read_context.cpp | 219 ++- .../core/operation/internal_read_context.h | 8 + .../operation/internal_read_context_test.cpp | 122 +- .../operation/merge_file_split_read_test.cpp | 26 +- .../operation/raw_file_split_read_test.cpp | 4 +- src/paimon/core/operation/read_context.cpp | 49 +- .../core/operation/read_context_test.cpp | 36 +- src/paimon/core/schema/table_schema.cpp | 3 - .../core/table/source/table_read_test.cpp | 14 +- .../table/system/audit_log_system_table.cpp | 2 +- src/paimon/core/utils/field_mapping.cpp | 50 +- src/paimon/core/utils/field_mapping.h | 2 +- src/paimon/core/utils/field_mapping_test.cpp | 31 + .../core/utils/nested_projection_utils.cpp | 419 +++++ .../core/utils/nested_projection_utils.h | 97 + .../utils/nested_projection_utils_test.cpp | 511 +++++ .../format/avro/avro_file_batch_reader.cpp | 11 +- .../avro/avro_file_batch_reader_test.cpp | 32 + .../format/parquet/file_reader_wrapper.h | 4 + .../parquet/parquet_file_batch_reader.cpp | 118 +- .../parquet/parquet_file_batch_reader.h | 19 + .../parquet_file_batch_reader_test.cpp | 123 ++ test/inte/CMakeLists.txt | 7 + test/inte/blob_table_inte_test.cpp | 2 +- test/inte/data_evolution_table_test.cpp | 2 +- test/inte/global_index_test.cpp | 4 +- test/inte/nested_column_pruning_inte_test.cpp | 1649 +++++++++++++++++ test/inte/read_inte_test.cpp | 34 +- test/inte/scan_and_read_inte_test.cpp | 6 +- test/inte/write_inte_test.cpp | 2 +- 40 files changed, 3796 insertions(+), 214 deletions(-) create mode 100644 src/paimon/core/utils/nested_projection_utils.cpp create mode 100644 src/paimon/core/utils/nested_projection_utils.h create mode 100644 src/paimon/core/utils/nested_projection_utils_test.cpp create mode 100644 test/inte/nested_column_pruning_inte_test.cpp diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 6eeb022d..4597268c 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -25,6 +25,7 @@ #include #include +#include "arrow/c/abi.h" #include "paimon/cache/cache.h" #include "paimon/predicate/predicate.h" #include "paimon/result.h" @@ -46,7 +47,7 @@ class FileSystem; class PAIMON_EXPORT ReadContext { public: ReadContext(const std::string& path, const std::string& branch, - const std::vector& read_schema, + const std::vector& read_field_names, const std::vector& read_field_ids, const std::shared_ptr& predicate, bool enable_predicate_filter, bool enable_prefetch, uint32_t prefetch_batch_count, @@ -77,8 +78,8 @@ class PAIMON_EXPORT ReadContext { return options_; } - const std::vector& GetReadSchema() const { - return read_schema_; + const std::vector& GetReadFieldNames() const { + return read_field_names_; } const std::vector& GetReadFieldIds() const { @@ -132,10 +133,26 @@ class PAIMON_EXPORT ReadContext { return cache_; } + /// Whether a read schema (C ArrowSchema) for nested column pruning was provided. + bool HasReadSchema() const { + return read_schema_ != nullptr && read_schema_->release != nullptr; + } + + /// Get the read schema as a mutable C ArrowSchema pointer. + /// ImportSchema will consume (release) the schema content. + ArrowSchema* GetReadSchema() { + return read_schema_.get(); + } + + /// Set the read schema from a C ArrowSchema unique_ptr and take ownership of + /// schema resources (released via ArrowSchema::release in destructor). + /// Called internally by ReadContextBuilder. + void SetReadSchema(std::unique_ptr schema); + private: std::string path_; std::string branch_; - std::vector read_schema_; + std::vector read_field_names_; std::vector read_field_ids_; std::shared_ptr predicate_; bool enable_predicate_filter_; @@ -153,6 +170,8 @@ class PAIMON_EXPORT ReadContext { PrefetchCacheMode prefetch_cache_mode_; CacheConfig cache_config_; std::shared_ptr cache_; + // Owns schema resources and releases ArrowSchema::release in destructor. + std::unique_ptr read_schema_; }; /// `ReadContextBuilder` used to build a `ReadContext`, has input validation. @@ -175,9 +194,9 @@ class PAIMON_EXPORT ReadContextBuilder { /// /// @param read_field_names Vector of field names to read from the table. /// @return Reference to this builder for method chaining. - /// @note Currently supports top-level field selection. Future versions may support - /// nested field selection using ArrowSchema for more granular projection - ReadContextBuilder& SetReadSchema(const std::vector& read_field_names); + /// @note Currently supports top-level field selection. For nested field selection + /// use SetReadSchema(std::unique_ptr) instead. + ReadContextBuilder& SetReadFieldNames(const std::vector& read_field_names); /// Set the schema fields to read from the table. /// /// If not set, all fields from the table schema will be read. This is useful for @@ -186,12 +205,51 @@ class PAIMON_EXPORT ReadContextBuilder { /// /// @param read_field_ids Vector of field ids to read from the table. /// @return Reference to this builder for method chaining. - /// @note Currently supports top-level field selection. Future versions may support - /// nested field selection using ArrowSchema for more granular projection. - /// @note SetReadFieldIds() and SetReadSchema() are mutually exclusive. - /// Calling both will ignore the read schema set by SetReadSchema(). + /// @note Currently supports top-level field selection. + /// @note SetReadFieldIds() and SetReadFieldNames() are mutually exclusive. + /// Calling both will ignore the read schema set by SetReadFieldNames(). ReadContextBuilder& SetReadFieldIds(const std::vector& read_field_ids); + /// Set the read Arrow Schema for nested column pruning. + /// + /// The read schema is an Arrow C Data Interface schema where STRUCT types + /// may contain only a subset of the original sub-fields, enabling nested column + /// pruning to reduce I/O. Field matching is based on field name: the system + /// looks up each field by name in the table schema and rebuilds the aligned + /// schema using the table schema's type and metadata. Metadata propagation + /// from the user-provided schema is whitelist-based: currently only + /// "paimon.map.selected-keys" is preserved and merged into the final aligned + /// schema. + /// + /// To prune map entries by key, attach metadata "paimon.map.selected-keys" + /// to the target map field in read schema. The value is a comma-separated + /// key list, for example: "k1,k2". Only map fields with string key type + /// (Arrow utf8) are supported. + /// + /// Example: + /// @code{.cpp} + /// auto map_field = arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())); + /// auto map_meta = arrow::KeyValueMetadata::Make( + /// {"paimon.map.selected-keys"}, {"k1,k2"}); + /// auto projected_schema = arrow::schema({ + /// arrow::field("id", arrow::int64()), + /// map_field->WithMetadata(map_meta), + /// }); + /// + /// auto c_schema = std::make_unique(); + /// arrow::ExportSchema(*projected_schema, c_schema.get()); + /// + /// ReadContextBuilder builder("/path/to/table"); + /// builder.SetReadSchema(std::move(c_schema)); + /// @endcode + /// + /// @param read_schema Arrow C Schema. Ownership of schema resources is transferred + /// to the built ReadContext. + /// @return Reference to this builder for method chaining. + /// @note Priority: read_schema > read_field_ids > read_field_names. + /// When set, read_field_ids and read_field_names are ignored. + ReadContextBuilder& SetReadSchema(std::unique_ptr read_schema); + /// Set a configuration options map to set some option entries which are not defined in the /// table schema or whose values you want to overwrite. /// @note The options map will clear the options added by `AddOption()` before. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index a7f2b972..7e075415 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -352,6 +352,7 @@ set(PAIMON_CORE_SRCS core/utils/blob_view_lookup.cpp core/utils/consumer_manager.cpp core/utils/field_mapping.cpp + core/utils/nested_projection_utils.cpp core/utils/file_store_path_factory.cpp core/utils/file_utils.cpp core/utils/manifest_meta_reader.cpp @@ -746,6 +747,7 @@ if(PAIMON_BUILD_TESTS) core/utils/consumer_manager_test.cpp core/utils/file_store_path_factory_cache_test.cpp core/utils/field_mapping_test.cpp + core/utils/nested_projection_utils_test.cpp core/utils/file_store_path_factory_test.cpp core/utils/file_utils_test.cpp core/utils/manifest_meta_reader_test.cpp diff --git a/src/paimon/common/memory/memory_segment_test.cpp b/src/paimon/common/memory/memory_segment_test.cpp index c7c25a7e..ec583044 100644 --- a/src/paimon/common/memory/memory_segment_test.cpp +++ b/src/paimon/common/memory/memory_segment_test.cpp @@ -558,10 +558,7 @@ TEST(MemorySegmentTest, TestDoubleAccess) { delete[] occupied; } -// ------------------------------------------------------------------------ -// Bulk Byte Movements -// ------------------------------------------------------------------------ - +// Bulk Byte Movements TEST(MemorySegmentTest, TestBulkByteAccess) { auto pool = paimon::GetDefaultPool(); // test expected correct behavior with default offset / length diff --git a/src/paimon/common/types/data_field.cpp b/src/paimon/common/types/data_field.cpp index 4aa10924..97a167b0 100644 --- a/src/paimon/common/types/data_field.cpp +++ b/src/paimon/common/types/data_field.cpp @@ -176,4 +176,38 @@ Result> DataField::ProjectFields( return projected_fields; } +std::shared_ptr DataField::MergeFieldMetadataByWhitelist( + const std::shared_ptr& target_field, + const std::shared_ptr& source_field, + const std::vector& metadata_keys_whitelist) { + if (!source_field || !source_field->HasMetadata() || !source_field->metadata()) { + return target_field; + } + + std::unordered_map metadata_map; + for (const auto& key : metadata_keys_whitelist) { + auto metadata_value_result = source_field->metadata()->Get(key); + if (metadata_value_result.ok()) { + metadata_map[key] = metadata_value_result.ValueUnsafe(); + } + } + + if (metadata_map.empty()) { + return target_field; + } + + auto metadata = std::make_shared(metadata_map); + return target_field->WithMergedMetadata(metadata); +} + +DataField DataField::MergeFieldMetadataByWhitelist( + const DataField& target_field, const DataField& source_field, + const std::vector& metadata_keys_whitelist) { + return DataField( + target_field.Id(), + MergeFieldMetadataByWhitelist(target_field.ArrowField(), source_field.ArrowField(), + metadata_keys_whitelist), + target_field.Description()); +} + } // namespace paimon diff --git a/src/paimon/common/types/data_field.h b/src/paimon/common/types/data_field.h index ada3ce90..4ca6d59f 100644 --- a/src/paimon/common/types/data_field.h +++ b/src/paimon/common/types/data_field.h @@ -44,6 +44,9 @@ class DataField : public Jsonizable { static constexpr char FIELD_ID[] = "paimon.id"; static constexpr char DESCRIPTION[] = "paimon.description"; + /// Metadata key for map field selected keys. The value is a comma-separated + /// string of key names, e.g. 'key1,key2'. Only string-keyed maps are supported. + static constexpr char MAP_SELECTED_KEYS[] = "paimon.map.selected-keys"; public: static std::shared_ptr ConvertDataFieldToArrowField(const DataField& field); @@ -66,6 +69,17 @@ class DataField : public Jsonizable { const std::vector& fields, const std::optional>& projected_cols); + /// Merge whitelisted metadata from source_field into target_field. + static std::shared_ptr MergeFieldMetadataByWhitelist( + const std::shared_ptr& target_field, + const std::shared_ptr& source_field, + const std::vector& metadata_keys_whitelist); + + /// Merge whitelisted metadata from source_field into target_field and keep target id/desc. + static DataField MergeFieldMetadataByWhitelist( + const DataField& target_field, const DataField& source_field, + const std::vector& metadata_keys_whitelist); + int32_t Id() const { return id_; } diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index a42f39a1..3208e87e 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -87,7 +87,7 @@ Result> CreateBatchReader( .WithFileSystem(core_options.GetFileSystem()) .EnablePrefetch(true) .WithMemoryPool(pool) - .SetReadSchema({field_name, SpecialFields::RowId().Name()}); + .SetReadFieldNames({field_name, SpecialFields::RowId().Name()}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index 148b62db..4fe5427d 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include "arrow/api.h" @@ -37,48 +38,152 @@ #include "paimon/core/casting/cast_executor.h" #include "paimon/core/casting/casting_utils.h" #include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/memory/bytes.h" #include "paimon/reader/batch_reader.h" namespace paimon { class MemoryPool; -FieldMappingReader::FieldMappingReader(int32_t field_count, - std::unique_ptr&& reader, - const BinaryRow& partition, - std::unique_ptr&& mapping, - const std::shared_ptr& pool) - : field_count_(field_count), - arrow_pool_(GetArrowPool(pool)), - reader_(std::move(reader)), - partition_(partition), - partition_info_(mapping->partition_info), - non_partition_info_(mapping->non_partition_info), - non_exist_field_info_(mapping->non_exist_field_info) { - if (non_exist_field_info_ != std::nullopt || partition_info_ != std::nullopt) { - need_mapping_ = true; +Result FieldMappingReader::HasMapSelectedKeysRecursively( + const std::shared_ptr& read_field) const { + if (!read_field) { + return false; + } + auto type_id = read_field->type()->id(); + if (type_id == arrow::Type::MAP) { + PAIMON_ASSIGN_OR_RAISE(std::vector selected_keys, + NestedProjectionUtils::GetMapSelectedKeys(read_field)); + return !selected_keys.empty(); + } + if (type_id == arrow::Type::STRUCT) { + for (const auto& child : read_field->type()->fields()) { + PAIMON_ASSIGN_OR_RAISE(bool has_selected_keys, HasMapSelectedKeysRecursively(child)); + if (has_selected_keys) { + return true; + } + } + } + return false; +} + +Result> FieldMappingReader::FilterMapSelectedKeysRecursively( + const std::shared_ptr& array, + const std::shared_ptr& read_field) const { + if (!array || !read_field) { + return array; + } + + auto type_id = read_field->type()->id(); + if (type_id == arrow::Type::MAP) { + PAIMON_ASSIGN_OR_RAISE(std::vector selected_keys, + NestedProjectionUtils::GetMapSelectedKeys(read_field)); + if (selected_keys.empty()) { + return array; + } + return NestedProjectionUtils::FilterMapArrayBySelectedKeys(array, selected_keys, + arrow_pool_.get()); + } + + if (type_id == arrow::Type::STRUCT) { + if (array->type_id() != arrow::Type::STRUCT) { + return Status::Invalid( + fmt::format("FilterMapSelectedKeysRecursively requires struct array for read " + "field '{}', got {}", + read_field->name(), array->type()->ToString())); + } + auto struct_array = std::static_pointer_cast(array); + auto read_struct_type = std::static_pointer_cast(read_field->type()); + if (struct_array->num_fields() != read_struct_type->num_fields()) { + return Status::Invalid(fmt::format( + "FilterMapSelectedKeysRecursively struct field count mismatch for '{}': " + "array {} vs read {}", + read_field->name(), struct_array->num_fields(), read_struct_type->num_fields())); + } + + arrow::ArrayVector filtered_children; + std::vector> filtered_child_data; + filtered_children.reserve(struct_array->num_fields()); + filtered_child_data.reserve(struct_array->num_fields()); + for (int32_t i = 0; i < struct_array->num_fields(); ++i) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr filtered_child, + FilterMapSelectedKeysRecursively(struct_array->field(i), + read_struct_type->field(i))); + filtered_child_data.push_back(filtered_child->data()); + filtered_children.push_back(std::move(filtered_child)); + } + + // Preserve parent struct null semantics after filtering children. + auto filtered_struct_data = arrow::ArrayData::Make( + read_struct_type, struct_array->length(), {struct_array->null_bitmap()}, + std::move(filtered_child_data), struct_array->null_count(), struct_array->offset()); + return arrow::MakeArray(std::move(filtered_struct_data)); + } + + return array; +} + +Result> FieldMappingReader::Create( + int32_t field_count, std::unique_ptr&& reader, const BinaryRow& partition, + std::unique_ptr&& mapping, const std::shared_ptr& pool) { + auto mapping_reader = std::unique_ptr(new FieldMappingReader( + field_count, std::move(reader), partition, std::move(mapping), pool)); + + mapping_reader->need_mapping_ = false; + mapping_reader->need_casting_ = false; + + if (mapping_reader->non_exist_field_info_ != std::nullopt || + mapping_reader->partition_info_ != std::nullopt) { + mapping_reader->need_mapping_ = true; } for (int32_t i = 0; - i < static_cast(non_partition_info_.idx_in_target_read_schema.size()); i++) { - if (i != non_partition_info_.idx_in_target_read_schema[i]) { - need_mapping_ = true; + i < + static_cast(mapping_reader->non_partition_info_.idx_in_target_read_schema.size()); + i++) { + if (i != mapping_reader->non_partition_info_.idx_in_target_read_schema[i]) { + mapping_reader->need_mapping_ = true; } - if (non_partition_info_.cast_executors[i] != nullptr) { - need_casting_ = true; + if (mapping_reader->non_partition_info_.cast_executors[i] != nullptr) { + mapping_reader->need_casting_ = true; } // Field name change (RENAME COLUMN) also requires mapping: data schema // carries the file's physical name while read schema carries the // post-rename logical name. If we skipped mapping, the inner reader's // batch would be passed through with the old physical name and the // consumer's name-based lookup against the read schema would fail. - if (non_partition_info_.non_partition_data_schema[i].Name() != - non_partition_info_.non_partition_read_schema[i].Name()) { - need_mapping_ = true; + if (mapping_reader->non_partition_info_.non_partition_data_schema[i].Name() != + mapping_reader->non_partition_info_.non_partition_read_schema[i].Name()) { + mapping_reader->need_mapping_ = true; + } + // Map selected-keys metadata must be validated in Create() (fail-fast). + // Non-empty selected-keys also requires mapping so that + // FilterMapArrayBySelectedKeys can filter out unwanted entries. + PAIMON_ASSIGN_OR_RAISE( + bool has_map_selected_keys, + mapping_reader->HasMapSelectedKeysRecursively( + mapping_reader->non_partition_info_.non_partition_read_schema[i].ArrowField())); + if (has_map_selected_keys) { + mapping_reader->need_mapping_ = true; } } + + return mapping_reader; } +FieldMappingReader::FieldMappingReader(int32_t field_count, + std::unique_ptr&& reader, + const BinaryRow& partition, + std::unique_ptr&& mapping, + const std::shared_ptr& pool) + : field_count_(field_count), + arrow_pool_(GetArrowPool(pool)), + reader_(std::move(reader)), + partition_(partition), + partition_info_(mapping->partition_info), + non_partition_info_(mapping->non_partition_info), + non_exist_field_info_(mapping->non_exist_field_info) {} + Result> FieldMappingReader::CastNonPartitionArrayIfNeed( const std::shared_ptr& src_array) const { if (!need_casting_) { @@ -144,9 +249,9 @@ Result FieldMappingReader::NextBatchWithBitmap // mapping non-partition array PAIMON_ASSIGN_OR_RAISE(std::shared_ptr casted_non_partition_array, CastNonPartitionArrayIfNeed(non_partition_array)); - MappingFields(casted_non_partition_array, non_partition_info_.non_partition_read_schema, - non_partition_info_.idx_in_target_read_schema, &target_array, - &target_field_names); + PAIMON_RETURN_NOT_OK(MappingFields( + casted_non_partition_array, non_partition_info_.non_partition_read_schema, + non_partition_info_.idx_in_target_read_schema, &target_array, &target_field_names)); // mapping partition array if (partition_info_ != std::nullopt) { @@ -155,9 +260,9 @@ Result FieldMappingReader::NextBatchWithBitmap GeneratePartitionArray(non_partition_array->length())); } auto trim_partition_array = partition_array_->Slice(0, non_partition_array->length()); - MappingFields(trim_partition_array, partition_info_.value().partition_read_schema, - partition_info_.value().idx_in_target_read_schema, &target_array, - &target_field_names); + PAIMON_RETURN_NOT_OK(MappingFields( + trim_partition_array, partition_info_.value().partition_read_schema, + partition_info_.value().idx_in_target_read_schema, &target_array, &target_field_names)); } // mapping non-exist array if (non_exist_field_info_ != std::nullopt) { @@ -166,9 +271,10 @@ Result FieldMappingReader::NextBatchWithBitmap GenerateNonExistArray(non_partition_array->length())); } auto trim_non_exist_array = non_exist_array_->Slice(0, non_partition_array->length()); - MappingFields(trim_non_exist_array, non_exist_field_info_.value().non_exist_read_schema, - non_exist_field_info_.value().idx_in_target_read_schema, &target_array, - &target_field_names); + PAIMON_RETURN_NOT_OK(MappingFields(trim_non_exist_array, + non_exist_field_info_.value().non_exist_read_schema, + non_exist_field_info_.value().idx_in_target_read_schema, + &target_array, &target_field_names)); } // construct target array @@ -285,20 +391,26 @@ Result> FieldMappingReader::GenerateNonExistArray( return arrow_array; } -void FieldMappingReader::MappingFields(const std::shared_ptr& data_array, - const std::vector& read_fields_of_data_array, - const std::vector& idx_in_target_schema, - arrow::ArrayVector* target_array, - std::vector* target_field_names) { +Status FieldMappingReader::MappingFields(const std::shared_ptr& data_array, + const std::vector& read_fields_of_data_array, + const std::vector& idx_in_target_schema, + arrow::ArrayVector* target_array, + std::vector* target_field_names) { auto* struct_array = arrow::internal::checked_cast(data_array.get()); assert(struct_array); assert(struct_array->fields().size() == idx_in_target_schema.size()); for (size_t i = 0; i < idx_in_target_schema.size(); i++) { - // target type may be string type, but after adapter transform, type may be dictionary, - // need reconstruct struct type - (*target_array)[idx_in_target_schema[i]] = struct_array->field(i); + std::shared_ptr field_array = struct_array->field(i); + + // Filter map entries by selected keys recursively (supports MAP nested in STRUCT). + PAIMON_ASSIGN_OR_RAISE(field_array, + FilterMapSelectedKeysRecursively( + field_array, read_fields_of_data_array[i].ArrowField())); + + (*target_array)[idx_in_target_schema[i]] = std::move(field_array); (*target_field_names)[idx_in_target_schema[i]] = read_fields_of_data_array[i].Name(); } + return Status::OK(); } } // namespace paimon diff --git a/src/paimon/core/io/field_mapping_reader.h b/src/paimon/core/io/field_mapping_reader.h index a5aad35b..bec1af24 100644 --- a/src/paimon/core/io/field_mapping_reader.h +++ b/src/paimon/core/io/field_mapping_reader.h @@ -48,9 +48,9 @@ struct FieldMapping; class FieldMappingReader : public FileBatchReader { public: - FieldMappingReader(int32_t field_count, std::unique_ptr&& reader, - const BinaryRow& partition, std::unique_ptr&& mapping, - const std::shared_ptr& pool); + static Result> Create( + int32_t field_count, std::unique_ptr&& reader, const BinaryRow& partition, + std::unique_ptr&& mapping, const std::shared_ptr& pool); Result NextBatch() override { return Status::Invalid( @@ -89,6 +89,10 @@ class FieldMappingReader : public FileBatchReader { } private: + FieldMappingReader(int32_t field_count, std::unique_ptr&& reader, + const BinaryRow& partition, std::unique_ptr&& mapping, + const std::shared_ptr& pool); + Result> GenerateSinglePartitionArray(int32_t idx, int32_t batch_size) const; @@ -98,11 +102,18 @@ class FieldMappingReader : public FileBatchReader { Result> CastNonPartitionArrayIfNeed( const std::shared_ptr& src_array) const; - static void MappingFields(const std::shared_ptr& src_array, - const std::vector& read_fields_of_data_array, - const std::vector& idx_in_target_schema, - arrow::ArrayVector* target_array, - std::vector* target_field_names); + Status MappingFields(const std::shared_ptr& src_array, + const std::vector& read_fields_of_data_array, + const std::vector& idx_in_target_schema, + arrow::ArrayVector* target_array, + std::vector* target_field_names); + + Result HasMapSelectedKeysRecursively( + const std::shared_ptr& read_field) const; + + Result> FilterMapSelectedKeysRecursively( + const std::shared_ptr& array, + const std::shared_ptr& read_field) const; private: bool need_mapping_ = false; diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index 9eb6f20a..7fbe7c94 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -47,6 +47,7 @@ #include "paimon/memory/memory_pool.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" @@ -134,9 +135,10 @@ class FieldMappingReaderTest : public ::testing::Test { mapping->non_partition_info.non_partition_filter, /*batch_size=*/1); - auto reader = std::make_shared( - /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), partition_, - std::move(mapping), pool_); + ASSERT_OK_AND_ASSIGN( + auto reader, FieldMappingReader::Create( + /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), + partition_, std::move(mapping), pool_)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); if (expect_array == nullptr && result_array == nullptr) { // expect empty result @@ -173,9 +175,10 @@ class FieldMappingReaderTest : public ::testing::Test { data_path, fs, data_schema.get(), data_array, arrow_schema.get(), /*predicate=*/mapping->non_partition_info.non_partition_filter, /*batch_size=*/1); - auto reader = std::make_shared( - /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), partition, - std::move(mapping), pool_); + ASSERT_OK_AND_ASSIGN( + auto reader, FieldMappingReader::Create( + /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), + partition, std::move(mapping), pool_)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); if (expect_array == nullptr && result_array == nullptr) { // expect empty result @@ -233,8 +236,9 @@ TEST_F(FieldMappingReaderTest, TestGenerateSinglePartitionArray) { {false, static_cast(1), static_cast(2), static_cast(3), static_cast(4), std::string("5"), std::make_shared("6", pool_.get()), 100}, pool_.get()); - auto mapping_reader = std::make_unique( - /*field_count=*/8, /*reader=*/nullptr, partition, std::move(field_mapping), pool_); + ASSERT_OK_AND_ASSIGN(auto mapping_reader, FieldMappingReader::Create( + /*field_count=*/8, /*reader=*/nullptr, partition, + std::move(field_mapping), pool_)); { ASSERT_OK_AND_ASSIGN(auto p7_array, mapping_reader->GenerateSinglePartitionArray( @@ -762,4 +766,26 @@ TEST_F(FieldMappingReaderTest, TestReadWithSchemaEvolutionRenameCombinedCast) { CheckResult(data_schema, data_array, read_schema, /*predicate=*/nullptr, /*partition_keys=*/{}, BinaryRow::EmptyRow(), expected); } + +TEST_F(FieldMappingReaderTest, TestCreateFailFastOnInvalidMapSelectedKeysMetadata) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,b,a"}); + auto map_field = arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), + /*nullable=*/true, metadata); + + NonPartitionInfo non_part_info; + non_part_info.non_partition_data_schema = {DataField(0, map_field)}; + non_part_info.non_partition_read_schema = {DataField(0, map_field)}; + non_part_info.idx_in_target_read_schema = {0}; + non_part_info.cast_executors = {nullptr}; + + auto field_mapping = std::make_unique( + /*partition_info=*/PartitionInfo(), std::move(non_part_info), + /*non_exist_field_info=*/std::nullopt); + + ASSERT_NOK_WITH_MSG(FieldMappingReader::Create( + /*field_count=*/1, + /*reader=*/nullptr, + /*partition=*/BinaryRow::EmptyRow(), std::move(field_mapping), pool_), + "Duplicate selected key 'a'"); +} } // namespace paimon::test diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index 0f19fd80..53e6c5d7 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -203,9 +203,11 @@ Result> AbstractSplitRead::CreateFieldMappingRe return std::unique_ptr(); } - return std::make_unique(field_mapping_builder->GetReadFieldCount(), - std::move(final_reader), partition, - std::move(field_mapping), pool_); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr mapping_reader, + FieldMappingReader::Create(field_mapping_builder->GetReadFieldCount(), + std::move(final_reader), partition, + std::move(field_mapping), pool_)); + return mapping_reader; } Result> AbstractSplitRead::ProjectFieldsForRowTrackingAndDataEvolution( diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 316002da..4d5ad411 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -18,19 +18,149 @@ #include "paimon/core/operation/internal_read_context.h" +#include #include +#include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/status.h" -namespace arrow { -class Schema; -} // namespace arrow - namespace paimon { + +Result> InternalReadContext::AlignReadFieldWithTableFieldIds( + const std::shared_ptr& read_field, + const std::shared_ptr& table_field) { + static const std::vector kReadMetadataWhitelist = {DataField::MAP_SELECTED_KEYS}; + + if (read_field->type()->id() != table_field->type()->id()) { + return Status::Invalid(fmt::format( + "Read schema field '{}' type {} does not match table field type {}", read_field->name(), + read_field->type()->ToString(), table_field->type()->ToString())); + } + + auto type_id = read_field->type()->id(); + if (type_id == arrow::Type::STRUCT) { + auto read_struct = std::static_pointer_cast(read_field->type()); + auto table_struct = std::static_pointer_cast(table_field->type()); + arrow::FieldVector rebased_children; + rebased_children.reserve(read_struct->num_fields()); + for (const auto& read_child : read_struct->fields()) { + auto table_child = + NestedProjectionUtils::FindFieldByName(table_struct->fields(), read_child->name()); + if (!table_child) { + return Status::Invalid(fmt::format( + "Read schema does not support schema evolution inside struct: nested field " + "'{}' does not exist in table field '{}'", + read_child->name(), read_field->name())); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr rebased_child, + AlignReadFieldWithTableFieldIds(read_child, table_child)); + rebased_children.push_back(rebased_child); + } + auto rebased_type = arrow::struct_(rebased_children); + auto aligned_field = table_field->WithType(rebased_type); + return DataField::MergeFieldMetadataByWhitelist(aligned_field, read_field, + kReadMetadataWhitelist); + } + + if (type_id == arrow::Type::LIST) { + auto read_list = std::static_pointer_cast(read_field->type()); + auto table_list = std::static_pointer_cast(table_field->type()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr rebased_value_field, + AlignReadFieldWithTableFieldIds(read_list->value_field(), table_list->value_field())); + auto rebased_type = arrow::list(rebased_value_field); + auto aligned_field = table_field->WithType(rebased_type); + return DataField::MergeFieldMetadataByWhitelist(aligned_field, read_field, + kReadMetadataWhitelist); + } + + if (type_id == arrow::Type::MAP) { + auto read_map = std::static_pointer_cast(read_field->type()); + auto table_map = std::static_pointer_cast(table_field->type()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr rebased_key_field, + AlignReadFieldWithTableFieldIds(read_map->key_field(), table_map->key_field())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr rebased_item_field, + AlignReadFieldWithTableFieldIds(read_map->item_field(), table_map->item_field())); + auto rebased_type = arrow::map(rebased_key_field->type(), rebased_item_field); + auto aligned_field = table_field->WithType(rebased_type); + return DataField::MergeFieldMetadataByWhitelist(aligned_field, read_field, + kReadMetadataWhitelist); + } + + if (!read_field->type()->Equals(table_field->type())) { + return Status::Invalid(fmt::format( + "Read schema field '{}' type {} does not match table field type {}", read_field->name(), + read_field->type()->ToString(), table_field->type()->ToString())); + } + + auto aligned_field = table_field->WithType(read_field->type()); + return DataField::MergeFieldMetadataByWhitelist(aligned_field, read_field, + kReadMetadataWhitelist); +} + +std::optional InternalReadContext::TryResolveSpecialFieldById( + int32_t field_id, const CoreOptions& core_options) { + if (field_id == SpecialFields::ValueKind().Id()) { + return SpecialFields::ValueKind(); + } + if (field_id == SpecialFields::RowId().Id()) { + if (core_options.RowTrackingEnabled()) { + return SpecialFields::RowId(); + } + return std::nullopt; + } + if (field_id == SpecialFields::SequenceNumber().Id()) { + if (core_options.RowTrackingEnabled() || core_options.KeyValueSequenceNumberEnabled()) { + return SpecialFields::SequenceNumber(); + } + return std::nullopt; + } + if (field_id == SpecialFields::IndexScore().Id()) { + if (core_options.DataEvolutionEnabled()) { + return SpecialFields::IndexScore(); + } + return std::nullopt; + } + return std::nullopt; +} + +std::optional InternalReadContext::TryResolveSpecialFieldByName( + const std::string& name, const CoreOptions& core_options) { + if (name == SpecialFields::ValueKind().Name()) { + return SpecialFields::ValueKind(); + } + if (name == SpecialFields::RowId().Name()) { + if (core_options.RowTrackingEnabled()) { + return SpecialFields::RowId(); + } + return std::nullopt; + } + if (name == SpecialFields::SequenceNumber().Name()) { + if (core_options.RowTrackingEnabled() || core_options.KeyValueSequenceNumberEnabled()) { + return SpecialFields::SequenceNumber(); + } + return std::nullopt; + } + if (name == SpecialFields::IndexScore().Name()) { + if (core_options.DataEvolutionEnabled()) { + return SpecialFields::IndexScore(); + } + return std::nullopt; + } + return std::nullopt; +} + Result> InternalReadContext::Create( const std::shared_ptr& context, const std::shared_ptr& table_schema, const std::map& options) { @@ -39,53 +169,46 @@ Result> InternalReadContext::Create( context->GetFileSystemSchemeToIdentifierMap())); core_options.WithCache(context->GetCache()); // prepare read schema + // Priority: projected_arrow_schema > read_field_ids > read_field_names + const bool has_projected_read_schema = context->HasReadSchema(); std::vector read_data_fields; - if (!context->GetReadFieldIds().empty()) { - read_data_fields.reserve(context->GetReadFieldIds().size()); - for (const auto& field_id : context->GetReadFieldIds()) { - // if enable row tracking or data evolution, check special fields - if (core_options.RowTrackingEnabled() && field_id == SpecialFields::RowId().Id()) { - read_data_fields.push_back(SpecialFields::RowId()); - continue; - } - if ((core_options.RowTrackingEnabled() || - core_options.KeyValueSequenceNumberEnabled()) && - field_id == SpecialFields::SequenceNumber().Id()) { - read_data_fields.push_back(SpecialFields::SequenceNumber()); + if (has_projected_read_schema) { + // Nested column pruning path: user provided a read C ArrowSchema + // where STRUCT types may contain only a subset of sub-fields. + // ImportSchema consumes the C schema — that's fine, it's one-shot usage. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema, + arrow::ImportSchema(context->GetReadSchema())); + read_data_fields.reserve(read_schema->num_fields()); + // Align special-field validation with read_field_ids/read_field_names branches. + for (const auto& read_field : read_schema->fields()) { + if (auto resolved_special_field = + TryResolveSpecialFieldByName(read_field->name(), core_options)) { + read_data_fields.push_back(*resolved_special_field); continue; } - if (field_id == SpecialFields::ValueKind().Id()) { - read_data_fields.push_back(SpecialFields::ValueKind()); - continue; - } - if (core_options.DataEvolutionEnabled() && - field_id == SpecialFields::IndexScore().Id()) { - read_data_fields.push_back(SpecialFields::IndexScore()); + PAIMON_ASSIGN_OR_RAISE(DataField table_field, + table_schema->GetField(read_field->name())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr aligned_field, + AlignReadFieldWithTableFieldIds(read_field, table_field.ArrowField())); + read_data_fields.emplace_back(table_field.Id(), aligned_field, + table_field.Description()); + } + } else if (!context->GetReadFieldIds().empty()) { + read_data_fields.reserve(context->GetReadFieldIds().size()); + for (const auto& field_id : context->GetReadFieldIds()) { + if (auto resolved_special_field = TryResolveSpecialFieldById(field_id, core_options)) { + read_data_fields.push_back(*resolved_special_field); continue; } PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(field_id)); read_data_fields.push_back(field); } - } else if (!context->GetReadSchema().empty()) { - read_data_fields.reserve(context->GetReadSchema().size()); - for (const auto& name : context->GetReadSchema()) { - // if enable row tracking or data evolution, check special fields - if (core_options.RowTrackingEnabled() && name == SpecialFields::RowId().Name()) { - read_data_fields.push_back(SpecialFields::RowId()); - continue; - } - if ((core_options.RowTrackingEnabled() || - core_options.KeyValueSequenceNumberEnabled()) && - name == SpecialFields::SequenceNumber().Name()) { - read_data_fields.push_back(SpecialFields::SequenceNumber()); - continue; - } - if (name == SpecialFields::ValueKind().Name()) { - read_data_fields.push_back(SpecialFields::ValueKind()); - continue; - } - if (core_options.DataEvolutionEnabled() && name == SpecialFields::IndexScore().Name()) { - read_data_fields.push_back(SpecialFields::IndexScore()); + } else if (!context->GetReadFieldNames().empty()) { + read_data_fields.reserve(context->GetReadFieldNames().size()); + for (const auto& name : context->GetReadFieldNames()) { + if (auto resolved_special_field = TryResolveSpecialFieldByName(name, core_options)) { + read_data_fields.push_back(*resolved_special_field); continue; } PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(name)); @@ -96,8 +219,14 @@ Result> InternalReadContext::Create( read_data_fields = table_schema->Fields(); } auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_data_fields); - // validate read schema to avoid redundant fields - PAIMON_RETURN_NOT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*read_schema)); + // validate read schema to avoid redundant fields. + // For projected read schema, nested sub-fields may be user-requested fields + // that do not exist in table schema, so they may not have paimon field IDs. + if (has_projected_read_schema) { + PAIMON_RETURN_NOT_OK(ArrowSchemaValidator::ValidateSchema(*read_schema)); + } else { + PAIMON_RETURN_NOT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*read_schema)); + } // validate predicate if (context->GetPredicate()) { PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( diff --git a/src/paimon/core/operation/internal_read_context.h b/src/paimon/core/operation/internal_read_context.h index 0b60d8ce..e0474b76 100644 --- a/src/paimon/core/operation/internal_read_context.h +++ b/src/paimon/core/operation/internal_read_context.h @@ -114,6 +114,14 @@ class InternalReadContext { const std::shared_ptr& read_schema, const CoreOptions& options); + static std::optional TryResolveSpecialFieldById(int32_t field_id, + const CoreOptions& core_options); + static std::optional TryResolveSpecialFieldByName(const std::string& name, + const CoreOptions& core_options); + static Result> AlignReadFieldWithTableFieldIds( + const std::shared_ptr& read_field, + const std::shared_ptr& table_field); + std::shared_ptr read_context_; std::shared_ptr table_schema_; std::shared_ptr read_schema_; diff --git a/src/paimon/core/operation/internal_read_context_test.cpp b/src/paimon/core/operation/internal_read_context_test.cpp index 4d1467b2..30ba77b7 100644 --- a/src/paimon/core/operation/internal_read_context_test.cpp +++ b/src/paimon/core/operation/internal_read_context_test.cpp @@ -52,7 +52,7 @@ TEST(InternalReadContext, TestReadWithUnspecifiedSchema) { TEST(InternalReadContext, TestReadWithSpecifiedSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f0"}); + context_builder.SetReadFieldNames({"f3", "f0"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); @@ -86,7 +86,7 @@ TEST(InternalReadContext, TestReadWithSpecifiedFieldIdAndSchema) { ReadContextBuilder context_builder(path); // read schema is specified, read fields in schema // will use field ids instead of field names. - context_builder.SetReadSchema({"f0"}); + context_builder.SetReadFieldNames({"f0"}); context_builder.SetReadFieldIds({3, 0}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); @@ -105,7 +105,8 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { // test simple std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER", "_INDEX_SCORE"}); + context_builder.SetReadFieldNames( + {"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER", "_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); @@ -126,7 +127,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { // test invalid case: disable row tracking while read row tracking fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER"}); + context_builder.SetReadFieldNames({"f3", "f0", "_ROW_ID", "_SEQUENCE_NUMBER"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); @@ -138,7 +139,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { // test invalid case: disable data evolution while read score fields std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f0", "_INDEX_SCORE"}); + context_builder.SetReadFieldNames({"f3", "f0", "_INDEX_SCORE"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); @@ -151,7 +152,7 @@ TEST(InternalReadContext, TestReadWithRowTrackingAndScoreFields) { TEST(InternalReadContext, TestReadWithValueKindField) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "_VALUE_KIND", "f0"}); + context_builder.SetReadFieldNames({"f3", "_VALUE_KIND", "f0"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); @@ -193,4 +194,113 @@ TEST(InternalReadContext, TestReadWithFieldIdsAndSpecialFields) { } } +TEST(InternalReadContext, TestReadWithProjectedSchemaAndSpecialFields) { + std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + + std::vector projected_fields = { + DataField(0, arrow::field("f0", arrow::utf8())), SpecialFields::RowId(), + SpecialFields::SequenceNumber(), SpecialFields::IndexScore()}; + auto schema_manager = SchemaManager(std::make_shared(), path); + ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); + + // Without options, special fields should be rejected in projected-schema path too. + { + auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + ReadContextBuilder context_builder(path); + context_builder.SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); + std::shared_ptr read_context = std::move(unique_read_context); + ASSERT_NOK_WITH_MSG( + InternalReadContext::Create(read_context, table_schema, table_schema->Options()), + "not exist in table schema"); + } + + // With options enabled, projected-schema path should accept these special fields. + auto enabled_options = table_schema->Options(); + enabled_options[Options::ROW_TRACKING_ENABLED] = "true"; + enabled_options[Options::DATA_EVOLUTION_ENABLED] = "true"; + + { + auto projected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + ReadContextBuilder context_builder(path); + context_builder.SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); + std::shared_ptr read_context = std::move(unique_read_context); + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(read_context, table_schema, enabled_options)); + auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(projected_fields); + ASSERT_TRUE(internal_context->GetReadSchema()->Equals(expected_schema)); + } +} + +TEST(InternalReadContext, TestReadWithProjectedSchemaWithoutFieldIds) { + std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + + auto projected_schema = + arrow::schema({arrow::field("f3", arrow::float64()), arrow::field("f0", arrow::utf8())}); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder context_builder(path); + context_builder.SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); + std::shared_ptr read_context = std::move(unique_read_context); + + SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); + + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + + std::vector expected_fields = { + DataField(3, arrow::field("f3", arrow::float64())), + DataField(0, arrow::field("f0", arrow::utf8())), + }; + auto expected_schema = DataField::ConvertDataFieldsToArrowSchema(expected_fields); + ASSERT_TRUE( + internal_context->GetReadSchema()->Equals(expected_schema, /*check_metadata=*/true)); +} + +TEST(InternalReadContext, TestProjectedSchemaMetadataWhitelist) { + std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; + + auto read_field = + arrow::field("f0", arrow::utf8()) + ->WithMetadata(arrow::KeyValueMetadata::Make( + {DataField::MAP_SELECTED_KEYS, "custom.key"}, {"k1,k2", "should_not_propagate"})); + auto projected_schema = arrow::schema({read_field}); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder context_builder(path); + context_builder.SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto unique_read_context, context_builder.Finish()); + std::shared_ptr read_context = std::move(unique_read_context); + + SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); + ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); + + ASSERT_OK_AND_ASSIGN( + auto internal_context, + InternalReadContext::Create(read_context, table_schema, table_schema->Options())); + + auto aligned_field = internal_context->GetReadSchema()->GetFieldByName("f0"); + ASSERT_TRUE(aligned_field); + ASSERT_TRUE(aligned_field->HasMetadata()); + ASSERT_TRUE(aligned_field->metadata()); + + auto selected_keys_result = aligned_field->metadata()->Get(DataField::MAP_SELECTED_KEYS); + ASSERT_TRUE(selected_keys_result.ok()); + ASSERT_EQ(selected_keys_result.ValueUnsafe(), "k1,k2"); + + auto custom_metadata_result = aligned_field->metadata()->Get("custom.key"); + ASSERT_FALSE(custom_metadata_result.ok()); +} + } // namespace paimon::test diff --git a/src/paimon/core/operation/merge_file_split_read_test.cpp b/src/paimon/core/operation/merge_file_split_read_test.cpp index 054de2f9..911cf796 100644 --- a/src/paimon/core/operation/merge_file_split_read_test.cpp +++ b/src/paimon/core/operation/merge_file_split_read_test.cpp @@ -613,7 +613,7 @@ TEST_P(MergeFileSplitReadTest, TestSimple) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k1", "p1", "s1", "v0", "v1"}); + context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}, {Options::IGNORE_DELETE, "true"}}); @@ -679,7 +679,7 @@ TEST_P(MergeFileSplitReadTest, TestLookUp) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k1", "p1", "s1", "v0", "v1"}); + context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}, {Options::IGNORE_DELETE, "true"}, @@ -753,7 +753,7 @@ TEST_P(MergeFileSplitReadTest, TestDeduplicateMergeEngineWithDeleteMsg) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k0", "k1", "v0", "v1", "v2"}); + context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); context_builder.SetOptions({{Options::MERGE_ENGINE, "deduplicate"}}); AddOptions(&context_builder); ASSERT_OK_AND_ASSIGN(std::shared_ptr read_context, context_builder.Finish()); @@ -794,7 +794,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithPredicate) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k1", "p1", "s1", "s0", "v0", "v1"}); + context_builder.SetReadFieldNames({"k1", "p1", "s1", "s0", "v0", "v1"}); context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}, {Options::IGNORE_DELETE, "true"}}); @@ -859,7 +859,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTable) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k1", "k0", "p0", "p1", "s1", "s0", "v0", "v1", "v2"}); + context_builder.SetReadFieldNames({"k1", "k0", "p0", "p1", "s1", "s0", "v0", "v1", "v2"}); context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}, {Options::IGNORE_DELETE, "true"}}); @@ -908,7 +908,7 @@ TEST_P(MergeFileSplitReadTest, TestReadWithAlterTableWithReverseSequence) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"v2", "p1", "k0", "p0", "s0", "v0"}); + context_builder.SetReadFieldNames({"v2", "p1", "k0", "p0", "s0", "v0"}); context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}, {Options::IGNORE_DELETE, "true"}}); @@ -956,7 +956,7 @@ TEST_P(MergeFileSplitReadTest, TestAggregateMergeEngine) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k1", "p1", "s1", "v0", "v1"}); + context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "aggregation"}, {"fields.v1.aggregate-function", "bool_and"}, @@ -1003,7 +1003,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngine) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k1", "p1", "s1", "v0"}); + context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0"}); context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "partial-update"}, {"fields.v1.sequence-group", "v0"}, @@ -1051,7 +1051,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithIgnoreDelete) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k0", "k1", "v0", "v1", "v2"}); + context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); context_builder.SetOptions( {{Options::MERGE_ENGINE, "partial-update"}, {Options::IGNORE_DELETE, "true"}}); AddOptions(&context_builder); @@ -1091,7 +1091,7 @@ TEST_P(MergeFileSplitReadTest, TestPartialUpdateMergeEngineWithRemoveRecordOnDel auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k0", "k1", "v0", "v1", "v2"}); + context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); AddOptions(&context_builder); @@ -1131,7 +1131,7 @@ TEST_P(MergeFileSplitReadTest, TestEmptyPlan) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k0", "k1", "v0", "v1", "v2"}); + context_builder.SetReadFieldNames({"k0", "k1", "v0", "v1", "v2"}); context_builder.SetOptions({{Options::MERGE_ENGINE, "partial-update"}, {Options::PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE, "true"}}); AddOptions(&context_builder); @@ -1158,7 +1158,7 @@ TEST_P(MergeFileSplitReadTest, TestIOException) { auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"k1", "p1", "s1", "v0", "v1"}); + context_builder.SetReadFieldNames({"k1", "p1", "s1", "v0", "v1"}); context_builder.SetOptions({{Options::SEQUENCE_FIELD, "s0,s1"}, {Options::MERGE_ENGINE, "deduplicate"}, {Options::IGNORE_DELETE, "true"}}); @@ -1212,7 +1212,7 @@ TEST_P(MergeFileSplitReadTest, Test09VersionWithoutInlineFieldId) { DataField(1, arrow::field("f1", arrow::int32()))}; auto read_schema = DataField::ConvertDataFieldsToArrowSchema(raw_read_fields); ASSERT_TRUE(read_schema); - context_builder.SetReadSchema({"f3", "f2", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f2", "f0", "f1"}); context_builder.SetOptions({{Options::FILE_FORMAT, "orc"}, {Options::MERGE_ENGINE, "deduplicate"}, {"orc.read.enable-metrics", "true"}}); diff --git a/src/paimon/core/operation/raw_file_split_read_test.cpp b/src/paimon/core/operation/raw_file_split_read_test.cpp index 3f557b5f..569f8dfd 100644 --- a/src/paimon/core/operation/raw_file_split_read_test.cpp +++ b/src/paimon/core/operation/raw_file_split_read_test.cpp @@ -135,7 +135,7 @@ class RawFileSplitReadTest : public ::testing::Test { "/orc/multi_partition_append_table.db/" "multi_partition_append_table"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema(read_schema->field_names()); + context_builder.SetReadFieldNames(read_schema->field_names()); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); @@ -430,7 +430,7 @@ TEST_F(RawFileSplitReadTest, TestMatch) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_total_buckets.db/pk_table_with_total_buckets"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f0", "f1", "f2", "f3"}); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, context_builder.Finish()); SchemaManager schema_manager(std::make_shared(), read_context->GetPath()); ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0)); diff --git a/src/paimon/core/operation/read_context.cpp b/src/paimon/core/operation/read_context.cpp index bb9cca77..ad7d1e3a 100644 --- a/src/paimon/core/operation/read_context.cpp +++ b/src/paimon/core/operation/read_context.cpp @@ -20,6 +20,8 @@ #include +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/executor.h" @@ -30,19 +32,20 @@ namespace paimon { class Predicate; ReadContext::ReadContext( - const std::string& path, const std::string& branch, const std::vector& read_schema, - const std::vector& read_field_ids, const std::shared_ptr& predicate, - bool enable_predicate_filter, bool enable_prefetch, uint32_t prefetch_batch_count, - uint32_t prefetch_max_parallel_num, bool enable_multi_thread_row_to_batch, - uint32_t row_to_batch_thread_number, const std::optional& table_schema, - const std::shared_ptr& memory_pool, const std::shared_ptr& executor, + const std::string& path, const std::string& branch, + const std::vector& read_field_names, const std::vector& read_field_ids, + const std::shared_ptr& predicate, bool enable_predicate_filter, bool enable_prefetch, + uint32_t prefetch_batch_count, uint32_t prefetch_max_parallel_num, + bool enable_multi_thread_row_to_batch, uint32_t row_to_batch_thread_number, + const std::optional& table_schema, const std::shared_ptr& memory_pool, + const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, const std::map& fs_scheme_to_identifier_map, const std::map& options, PrefetchCacheMode prefetch_cache_mode, const CacheConfig& cache_config, const std::shared_ptr& cache) : path_(path), branch_(branch), - read_schema_(read_schema), + read_field_names_(read_field_names), read_field_ids_(read_field_ids), predicate_(predicate), enable_predicate_filter_(enable_predicate_filter), @@ -61,7 +64,23 @@ ReadContext::ReadContext( cache_config_(cache_config), cache_(cache) {} -ReadContext::~ReadContext() = default; +ReadContext::~ReadContext() { + if (read_schema_ && read_schema_->release) { + read_schema_->release(read_schema_.get()); + } +} + +void ReadContext::SetReadSchema(std::unique_ptr schema) { + if (schema && schema->release) { + if (schema.get() == read_schema_.get()) { + return; + } + if (read_schema_ && read_schema_->release) { + read_schema_->release(read_schema_.get()); + } + read_schema_ = std::move(schema); + } +} class ReadContextBuilder::Impl { public: @@ -70,6 +89,7 @@ class ReadContextBuilder::Impl { branch_ = BranchManager::DEFAULT_MAIN_BRANCH; read_field_names_.clear(); read_field_ids_.clear(); + read_schema_.reset(); fs_scheme_to_identifier_map_.clear(); options_.clear(); predicate_.reset(); @@ -93,6 +113,7 @@ class ReadContextBuilder::Impl { std::string branch_ = BranchManager::DEFAULT_MAIN_BRANCH; std::vector read_field_names_; std::vector read_field_ids_; + std::unique_ptr read_schema_; std::map fs_scheme_to_identifier_map_; std::map options_; std::shared_ptr predicate_; @@ -132,7 +153,7 @@ ReadContextBuilder& ReadContextBuilder::SetOptions(const std::map& read_field_names) { impl_->read_field_names_ = read_field_names; return *this; @@ -144,6 +165,13 @@ ReadContextBuilder& ReadContextBuilder::SetReadFieldIds( return *this; } +ReadContextBuilder& ReadContextBuilder::SetReadSchema(std::unique_ptr read_schema) { + if (read_schema && read_schema->release) { + impl_->read_schema_ = std::move(read_schema); + } + return *this; +} + ReadContextBuilder& ReadContextBuilder::SetPredicate(const std::shared_ptr& predicate) { impl_->predicate_ = predicate; return *this; @@ -264,6 +292,9 @@ Result> ReadContextBuilder::Finish() { impl_->table_schema_, impl_->memory_pool_, impl_->executor_, impl_->specific_file_system_, impl_->fs_scheme_to_identifier_map_, impl_->options_, impl_->prefetch_cache_mode_, impl_->cache_config_, impl_->cache_); + if (impl_->read_schema_ && impl_->read_schema_->release) { + ctx->SetReadSchema(std::move(impl_->read_schema_)); + } impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/read_context_test.cpp b/src/paimon/core/operation/read_context_test.cpp index f1945cc0..8d5a78e1 100644 --- a/src/paimon/core/operation/read_context_test.cpp +++ b/src/paimon/core/operation/read_context_test.cpp @@ -20,6 +20,8 @@ #include +#include "arrow/c/bridge.h" +#include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/common/io/cache/lru_cache.h" #include "paimon/defs.h" @@ -37,7 +39,7 @@ TEST(ReadContextTest, TestDefaultValue) { ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); - ASSERT_TRUE(ctx->GetReadSchema().empty()); + ASSERT_TRUE(ctx->GetReadFieldNames().empty()); ASSERT_TRUE(ctx->GetReadFieldIds().empty()); ASSERT_TRUE(ctx->GetOptions().empty()); ASSERT_FALSE(ctx->GetPredicate()); @@ -61,7 +63,7 @@ TEST(ReadContextTest, TestSetContent) { /*hole_size_limit=*/128, /*pre_buffer_limit=*/2048); builder.AddOption("key", "value"); - builder.SetReadSchema({"f1", "f2"}); + builder.SetReadFieldNames({"f1", "f2"}); builder.SetReadFieldIds({0, 1}); auto predicate = PredicateBuilder::IsNull(/*field_index=*/0, /*field_name=*/"f1", FieldType::INT); @@ -88,7 +90,7 @@ TEST(ReadContextTest, TestSetContent) { ASSERT_EQ(ctx->GetPath(), "table_root_path"); ASSERT_TRUE(ctx->GetMemoryPool()); ASSERT_TRUE(ctx->GetExecutor()); - ASSERT_EQ(ctx->GetReadSchema(), std::vector({"f1", "f2"})); + ASSERT_EQ(ctx->GetReadFieldNames(), std::vector({"f1", "f2"})); ASSERT_EQ(ctx->GetReadFieldIds(), std::vector({0, 1})); ASSERT_EQ(*predicate, *(ctx->GetPredicate())); ASSERT_TRUE(ctx->EnablePredicateFilter()); @@ -151,4 +153,32 @@ TEST(ReadContextTest, TestPrefetchMaxParallelNumZero) { ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num should be greater than 0"); } +TEST(ReadContextTest, TestSetReadSchemaAndHasReadSchema) { + auto projected_schema = arrow::schema({arrow::field("f0", arrow::utf8())}); + auto c_schema = std::make_unique(); + auto* c_schema_raw = c_schema.get(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + { + ReadContextBuilder builder("table_root_path"); + builder.SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + ASSERT_TRUE(ctx->HasReadSchema()); + ASSERT_EQ(ctx->GetReadSchema(), c_schema_raw); + } + + ASSERT_EQ(c_schema, nullptr); +} + +TEST(ReadContextTest, TestSetInvalidReadSchemaIgnored) { + auto invalid_schema = std::make_unique(); + + ReadContextBuilder builder("table_root_path"); + builder.SetReadSchema(std::move(invalid_schema)); + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + + ASSERT_FALSE(ctx->HasReadSchema()); + ASSERT_EQ(ctx->GetReadSchema(), nullptr); +} + } // namespace paimon::test diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index a0294617..1e089cb9 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -48,9 +48,6 @@ Result> TableSchema::Create( int64_t schema_id, const std::shared_ptr& schema, const std::vector& partition_keys, const std::vector& primary_keys, const std::map& options) { - if (schema_id != 0) { - return Status::NotImplemented("do not support schema evolution, schema_id must be 0"); - } std::vector data_fields; int32_t field_id = 0; std::set primary_key_set; diff --git a/src/paimon/core/table/source/table_read_test.cpp b/src/paimon/core/table/source/table_read_test.cpp index 691e4d79..bc2c7f6f 100644 --- a/src/paimon/core/table/source/table_read_test.cpp +++ b/src/paimon/core/table/source/table_read_test.cpp @@ -44,7 +44,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { { // read with non-exist field ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f0", "f1", "non-exist"}); + context_builder.SetReadFieldNames({"f0", "f1", "non-exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "Get field non-exist failed: not exist in table schema"); @@ -75,7 +75,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { auto predicate = PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"f3", FieldType::DOUBLE, Literal(15.0)); ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPredicate(predicate); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG( @@ -95,7 +95,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { { // schema with duplicate field f3 ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f1", "f3"}); + context_builder.SetReadFieldNames({"f3", "f1", "f3"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "validate schema failed: read schema has duplicate field f3"); @@ -105,7 +105,7 @@ TEST(TableReadTest, TestReadWithInvalidContext) { TEST(TableReadTest, TestReadWithSpecifiedInvalidSchema) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"field_no_exist"}); + context_builder.SetReadFieldNames({"field_no_exist"}); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), "Get field field_no_exist failed: not exist in table schema"); @@ -115,7 +115,7 @@ TEST(TableReadTest, TestCreateKeyValueTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/pk_table_with_dv_cardinality.db/pk_table_with_dv_cardinality/"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f0", "f1", "f2", "f3"}); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("read.batch-size", "2"); context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); @@ -127,7 +127,7 @@ TEST(TableReadTest, TestCreateKeyValueTableRead) { TEST(TableReadTest, TestCreateAppendOnlyTableRead) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f0", "f1", "f2", "f3"}); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("read.batch-size", "2"); context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); ASSERT_OK_AND_ASSIGN(auto read_context, context_builder.Finish()); @@ -139,7 +139,7 @@ TEST(TableReadTest, TestCreateAppendOnlyTableRead) { TEST(TableReadTest, TestMergeOptions) { std::string path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f0", "f1", "f2", "f3"}); + context_builder.SetReadFieldNames({"f0", "f1", "f2", "f3"}); context_builder.AddOption("read.batch-size", "2"); context_builder.AddOption("orc.read.enable-lazy-decoding", "true"); context_builder.AddOption("bucket", "10"); diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp index a2581eb2..1f89d5b7 100644 --- a/src/paimon/core/table/system/audit_log_system_table.cpp +++ b/src/paimon/core/table/system/audit_log_system_table.cpp @@ -287,7 +287,7 @@ Result> AuditLogSystemTable::NewChangelogRead( PAIMON_ASSIGN_OR_RAISE(StringMap read_options, ReadOptions()); PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(read_options)); builder.SetOptions(read_options) - .SetReadSchema(base_read_schema->field_names()) + .SetReadFieldNames(base_read_schema->field_names()) .WithBranch(core_options.GetBranch()) .WithMemoryPool(context->GetMemoryPool()) .WithExecutor(context->GetExecutor()) diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp index 5ab695ba..34934a95 100644 --- a/src/paimon/core/utils/field_mapping.cpp +++ b/src/paimon/core/utils/field_mapping.cpp @@ -30,6 +30,7 @@ #include "paimon/common/utils/object_utils.h" #include "paimon/core/casting/cast_executor_factory.h" #include "paimon/core/casting/casting_utils.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/defs.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" @@ -75,8 +76,8 @@ Result> FieldMappingBuilder::CreateFieldMapping( // generate non-exist field info std::optional non_exist_field_info = CreateNonExistFieldInfo(data_fields); - // generate exist field info - ExistFieldInfo exist_field_info = CreateExistFieldInfo(data_fields); + // generate exist field info (includes nested type pruning) + PAIMON_ASSIGN_OR_RAISE(ExistFieldInfo exist_field_info, CreateExistFieldInfo(data_fields)); // key: partition key, value: partition idx std::map partition_key_to_idx = @@ -90,7 +91,7 @@ Result> FieldMappingBuilder::CreateFieldMapping( return std::make_unique(partition_info, non_partition_info, non_exist_field_info); } -ExistFieldInfo FieldMappingBuilder::CreateExistFieldInfo( +Result FieldMappingBuilder::CreateExistFieldInfo( const std::vector& data_fields) const { // key:field id, value: {target_idx, read field} std::map> field_id_to_read_fields; @@ -104,8 +105,22 @@ ExistFieldInfo FieldMappingBuilder::CreateExistFieldInfo( auto iter = field_id_to_read_fields.find(data_field.Id()); if (iter != field_id_to_read_fields.end()) { const auto& [target_idx, read_field] = iter->second; + + // Recursively prune nested types in data_field to match read_field's + // projection. For atomic types this is a no-op. + PAIMON_ASSIGN_OR_RAISE( + std::optional> pruned_type, + NestedProjectionUtils::PruneDataType(read_field.Type(), data_field.Type())); + if (!pruned_type.has_value()) { + // All sub-fields pruned away — treat as non-existent. + continue; + } + + DataField pruned_data_field(data_field.Id(), + data_field.ArrowField()->WithType(pruned_type.value()), + data_field.Description()); exist_field_info.exist_read_schema.push_back(read_field); - exist_field_info.exist_data_schema.push_back(data_field); + exist_field_info.exist_data_schema.push_back(pruned_data_field); exist_field_info.idx_in_target_read_schema.push_back(target_idx); } } @@ -127,6 +142,19 @@ std::optional FieldMappingBuilder::CreateNonExistFieldInfo( if (iter == field_id_to_data_fields.end()) { non_exist_field_info.non_exist_read_schema.push_back(read_field); non_exist_field_info.idx_in_target_read_schema.push_back(i); + continue; + } + + // Empty STRUCT projection (f1: struct<>) is a valid request, but + // cannot be read from data files directly. Materialize it as nulls. + if (read_field.Type()->id() == arrow::Type::STRUCT && + iter->second.Type()->id() == arrow::Type::STRUCT) { + auto read_struct = std::static_pointer_cast(read_field.Type()); + auto data_struct = std::static_pointer_cast(iter->second.Type()); + if (read_struct->num_fields() == 0 && data_struct->num_fields() > 0) { + non_exist_field_info.non_exist_read_schema.push_back(read_field); + non_exist_field_info.idx_in_target_read_schema.push_back(i); + } } } if (non_exist_field_info.idx_in_target_read_schema.empty()) { @@ -147,9 +175,12 @@ Result>> FieldMappingBuilder::CreateDa FieldTypeUtils::ConvertToFieldType(data_fields[i].Type()->id())); if (!read_fields[i].Type()->Equals(data_fields[i].Type())) { - if (read_type == FieldType::MAP || read_type == FieldType::ARRAY || - read_type == FieldType::STRUCT) { - return Status::Invalid("Only support column type evolution in atomic data type."); + if (read_type == FieldType::STRUCT) { + // STRUCT may still differ by nested pruning shape. No cast is + // needed — type pruning is handled by PruneDataType during + // field mapping construction. + cast_executors.push_back(nullptr); + continue; } auto executor_factory = CastExecutorFactory::GetCastExecutorFactory(); auto cast_executor = @@ -173,13 +204,16 @@ Result FieldMappingBuilder::CreateNonPartitionInfo( const std::vector& data_fields, const ExistFieldInfo& exist_field_info, const std::map& partition_keys) const { NonPartitionInfo non_partition_info; + const std::vector propagated_metadata_keys = {DataField::MAP_SELECTED_KEYS}; for (size_t i = 0; i < exist_field_info.exist_data_schema.size(); i++) { const auto& data_field = exist_field_info.exist_data_schema[i]; const auto& read_field = exist_field_info.exist_read_schema[i]; auto iter = partition_keys.find(read_field.Name()); if (iter == partition_keys.end()) { non_partition_info.non_partition_read_schema.push_back(read_field); - non_partition_info.non_partition_data_schema.push_back(data_field); + non_partition_info.non_partition_data_schema.push_back( + DataField::MergeFieldMetadataByWhitelist(data_field, read_field, + propagated_metadata_keys)); non_partition_info.idx_in_target_read_schema.push_back( exist_field_info.idx_in_target_read_schema[i]); } diff --git a/src/paimon/core/utils/field_mapping.h b/src/paimon/core/utils/field_mapping.h index 9445add3..55b89253 100644 --- a/src/paimon/core/utils/field_mapping.h +++ b/src/paimon/core/utils/field_mapping.h @@ -83,7 +83,7 @@ class FieldMappingBuilder { std::optional CreateNonExistFieldInfo( const std::vector& data_fields) const; - ExistFieldInfo CreateExistFieldInfo(const std::vector& data_fields) const; + Result CreateExistFieldInfo(const std::vector& data_fields) const; Result CreateNonPartitionInfo( const std::vector& data_fields, const ExistFieldInfo& exist_field_info, diff --git a/src/paimon/core/utils/field_mapping_test.cpp b/src/paimon/core/utils/field_mapping_test.cpp index 153932df..2de53d25 100644 --- a/src/paimon/core/utils/field_mapping_test.cpp +++ b/src/paimon/core/utils/field_mapping_test.cpp @@ -645,4 +645,35 @@ TEST_F(FieldMappingTest, TestCompoundPredicateWithoutPushDown) { CheckNonPartitionInfo(mapping->non_partition_info, expected_non_part_info); } +TEST_F(FieldMappingTest, TestMapSelectedKeysMetadataPropagatedToDataSchema) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto selected_keys_metadata = arrow::KeyValueMetadata::Make( + {DataField::MAP_SELECTED_KEYS, "custom.key"}, {"k1,k2", "should_not_propagate"}); + + std::vector read_fields = { + DataField(0, arrow::field("m", map_type)->WithMetadata(selected_keys_metadata)), + }; + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + + std::vector data_fields = { + DataField(0, arrow::field("m", map_type)), + }; + + ASSERT_OK_AND_ASSIGN( + auto mapping_builder, + FieldMappingBuilder::Create(read_schema, /*partition_keys=*/{}, /*predicate=*/nullptr)); + ASSERT_OK_AND_ASSIGN(auto mapping, mapping_builder->CreateFieldMapping(data_fields)); + + ASSERT_EQ(mapping->non_partition_info.non_partition_data_schema.size(), 1); + auto propagated_field = mapping->non_partition_info.non_partition_data_schema[0].ArrowField(); + ASSERT_TRUE(propagated_field->HasMetadata()); + ASSERT_TRUE(propagated_field->metadata()); + auto selected_keys_result = propagated_field->metadata()->Get(DataField::MAP_SELECTED_KEYS); + ASSERT_TRUE(selected_keys_result.ok()); + std::string selected_keys = selected_keys_result.ValueUnsafe(); + ASSERT_EQ(selected_keys, "k1,k2"); + auto custom_metadata_result = propagated_field->metadata()->Get("custom.key"); + ASSERT_FALSE(custom_metadata_result.ok()); +} + } // namespace paimon::test diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp new file mode 100644 index 00000000..70fb4bce --- /dev/null +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/nested_projection_utils.h" + +#include +#include +#include +#include +#include + +#include "arrow/array/array_nested.h" +#include "arrow/array/array_primitive.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/array/concatenate.h" +#include "arrow/type.h" +#include "fmt/format.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/status.h" + +namespace paimon { + +std::shared_ptr NestedProjectionUtils::FindFieldByName( + const arrow::FieldVector& fields, const std::string& name) { + for (const auto& field : fields) { + if (field->name() == name) { + return field; + } + } + return nullptr; +} + +int32_t NestedProjectionUtils::GetPaimonFieldId(const std::shared_ptr& field) { + if (!field || !field->HasMetadata() || !field->metadata()) { + return -1; + } + auto result = field->metadata()->Get(DataField::FIELD_ID); + if (!result.ok()) { + return -1; + } + std::optional field_id = StringUtils::StringToValue(result.ValueUnsafe()); + return field_id.value_or(-1); +} + +std::shared_ptr NestedProjectionUtils::FindFieldByPaimonId( + const std::shared_ptr& struct_type, int32_t field_id) { + if (!struct_type || struct_type->id() != arrow::Type::STRUCT) { + return nullptr; + } + for (const auto& child : struct_type->fields()) { + if (GetPaimonFieldId(child) == field_id) { + return child; + } + } + return nullptr; +} + +Result NestedProjectionUtils::HasNestedSubfieldProjectionType( + const std::shared_ptr& file_type, + const std::shared_ptr& read_type) { + switch (file_type->id()) { + case arrow::Type::STRUCT: { + if (read_type->id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjectionType requires same nested type kind, but file " + "type is {} and read type is {}", + file_type->ToString(), read_type->ToString())); + } + auto file_struct = std::static_pointer_cast(file_type); + auto read_struct = std::static_pointer_cast(read_type); + bool field_count_diff = read_struct->num_fields() != file_struct->num_fields(); + for (const auto& read_child : read_struct->fields()) { + auto file_child = FindFieldByName(file_struct->fields(), read_child->name()); + if (!file_child) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjectionType found requested struct child '{}' " + "missing in file type {}", + read_child->name(), file_type->ToString())); + } + PAIMON_ASSIGN_OR_RAISE( + bool child_has_nested_projection, + HasNestedSubfieldProjectionType(file_child->type(), read_child->type())); + if (child_has_nested_projection) { + return true; + } + } + return field_count_diff; + } + case arrow::Type::LIST: { + if (read_type->id() != arrow::Type::LIST) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjectionType requires same nested type kind, but file " + "type is {} and read type is {}", + file_type->ToString(), read_type->ToString())); + } + auto file_list = std::static_pointer_cast(file_type); + auto read_list = std::static_pointer_cast(read_type); + return HasNestedSubfieldProjectionType(file_list->value_type(), + read_list->value_type()); + } + case arrow::Type::MAP: { + if (read_type->id() != arrow::Type::MAP) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjectionType requires same nested type kind, but file " + "type is {} and read type is {}", + file_type->ToString(), read_type->ToString())); + } + auto file_map = std::static_pointer_cast(file_type); + auto read_map = std::static_pointer_cast(read_type); + PAIMON_ASSIGN_OR_RAISE( + bool key_has_nested_projection, + HasNestedSubfieldProjectionType(file_map->key_type(), read_map->key_type())); + if (key_has_nested_projection) { + return true; + } + return HasNestedSubfieldProjectionType(file_map->item_type(), read_map->item_type()); + } + default: + return false; + } +} + +Result>> NestedProjectionUtils::PruneDataType( + const std::shared_ptr& read_type, + const std::shared_ptr& data_type) { + // Identical types need no pruning. + if (read_type->Equals(data_type)) { + return std::optional>(data_type); + } + + switch (read_type->id()) { + case arrow::Type::STRUCT: { + arrow::FieldVector pruned_fields; + for (const auto& read_child : read_type->fields()) { + int32_t read_child_id = GetPaimonFieldId(read_child); + if (read_child_id < 0) { + return Status::Invalid(fmt::format( + "PruneDataType requires paimon.id for nested struct field '{}', but it " + "is missing or invalid", + read_child->name())); + } + std::shared_ptr data_child = + FindFieldByPaimonId(data_type, read_child_id); + if (!data_child) { + return Status::Invalid(fmt::format( + "PruneDataType does not support schema evolution inside struct: nested " + "field '{}' (id={}) does not exist in data type {}", + read_child->name(), read_child_id, data_type->ToString())); + } + if (read_child->name() != data_child->name()) { + return Status::Invalid(fmt::format( + "PruneDataType does not support schema evolution inside struct: nested " + "field id {} name mismatch: read '{}' vs data '{}'", + read_child_id, read_child->name(), data_child->name())); + } + if (read_child->type()->id() != data_child->type()->id()) { + return Status::Invalid(fmt::format( + "PruneDataType nested field type mismatch for '{}': read {} vs data {}", + read_child->name(), read_child->type()->ToString(), + data_child->type()->ToString())); + } + PAIMON_ASSIGN_OR_RAISE( + std::optional> pruned_child_type, + PruneDataType(read_child->type(), data_child->type())); + if (!pruned_child_type.has_value()) { + // All sub-fields of this child were pruned away; skip it. + continue; + } + pruned_fields.push_back(data_child->WithType(pruned_child_type.value())); + } + if (pruned_fields.empty()) { + // All fields pruned — return nullopt so the caller can skip this field. + return std::optional>(std::nullopt); + } + return std::optional>(arrow::struct_(pruned_fields)); + } + + case arrow::Type::LIST: { + // Keep behavior aligned with format readers: partial projection inside + // LIST is unsupported and must fail fast. + return Status::Invalid( + fmt::format("PruneDataType does not support partial projection inside list: src {} " + "vs target {}", + data_type->ToString(), read_type->ToString())); + } + + case arrow::Type::MAP: { + // Keep behavior aligned with format readers: partial projection inside + // MAP is unsupported and must fail fast. + return Status::Invalid(fmt::format( + "PruneDataType does not support partial projection inside map: src {} vs target {}", + data_type->ToString(), read_type->ToString())); + } + + default: + // Atomic type: return data_type as-is (type evolution is handled + // separately by CastExecutor). + return std::optional>(data_type); + } +} + +Result NestedProjectionUtils::HasNestedSubfieldProjection( + const std::shared_ptr& file_schema, + const std::shared_ptr& read_schema) { + for (const auto& read_field : read_schema->fields()) { + auto file_field = file_schema->GetFieldByName(read_field->name()); + if (!file_field) { + return Status::Invalid(fmt::format( + "HasNestedSubfieldProjection found read field '{}' missing in file schema {}", + read_field->name(), file_schema->ToString())); + } + if (read_field->type()->id() == arrow::Type::STRUCT || + read_field->type()->id() == arrow::Type::LIST || + read_field->type()->id() == arrow::Type::MAP) { + PAIMON_ASSIGN_OR_RAISE( + bool has_nested_projection, + HasNestedSubfieldProjectionType(file_field->type(), read_field->type())); + if (has_nested_projection) { + return true; + } + } + } + return false; +} + +// Map selected-keys support + +Result> NestedProjectionUtils::GetMapSelectedKeys( + const std::shared_ptr& field) { + std::vector result; + if (!field || !field->HasMetadata() || !field->metadata()) { + return result; + } + auto get_result = field->metadata()->Get(DataField::MAP_SELECTED_KEYS); + if (!get_result.ok()) { + return result; + } + std::string value = get_result.ValueUnsafe(); + if (value.empty()) { + // Metadata is explicitly present but empty: select the empty-string key. + result.push_back(""); + return result; + } + + auto tokens = StringUtils::Split(value, ",", /*ignore_empty=*/false); + std::unordered_set deduplicated; + deduplicated.reserve(tokens.size()); + for (auto& token : tokens) { + if (!deduplicated.insert(token).second) { + return Status::Invalid(fmt::format("Duplicate selected key '{}' in {} metadata", token, + DataField::MAP_SELECTED_KEYS)); + } + result.push_back(token); + } + return result; +} + +namespace { + +struct MapKeyAccessor { + std::shared_ptr string_keys; + std::shared_ptr dict_keys; + std::shared_ptr dict_values; + std::shared_ptr dict_large_values; +}; + +Result BuildMapKeyAccessor(const std::shared_ptr& key_array) { + MapKeyAccessor accessor; + if (key_array->type_id() == arrow::Type::STRING) { + accessor.string_keys = std::static_pointer_cast(key_array); + return accessor; + } + if (key_array->type_id() == arrow::Type::DICTIONARY) { + auto dict_type = std::static_pointer_cast(key_array->type()); + if (dict_type->value_type()->id() != arrow::Type::STRING && + dict_type->value_type()->id() != arrow::Type::LARGE_STRING) { + return Status::Invalid( + fmt::format("FilterMapArrayBySelectedKeys only supports string keys or " + "dictionary keys, got {}", + key_array->type()->ToString())); + } + accessor.dict_keys = std::static_pointer_cast(key_array); + if (dict_type->value_type()->id() == arrow::Type::STRING) { + accessor.dict_values = + std::static_pointer_cast(accessor.dict_keys->dictionary()); + } else { + accessor.dict_large_values = + std::static_pointer_cast(accessor.dict_keys->dictionary()); + } + return accessor; + } + return Status::Invalid( + fmt::format("FilterMapArrayBySelectedKeys only supports string keys or " + "dictionary keys, got {}", + key_array->type()->ToString())); +} + +Result GetMapKeyViewAt(const MapKeyAccessor& accessor, int64_t entry_idx) { + if (accessor.string_keys) { + if (accessor.string_keys->IsNull(entry_idx)) { + return Status::Invalid("FilterMapArrayBySelectedKeys found null map key at entry " + + std::to_string(entry_idx)); + } + return accessor.string_keys->GetView(entry_idx); + } + + if (accessor.dict_keys->IsNull(entry_idx)) { + return Status::Invalid("FilterMapArrayBySelectedKeys found null map key at entry " + + std::to_string(entry_idx)); + } + int64_t dict_idx = accessor.dict_keys->GetValueIndex(entry_idx); + if (accessor.dict_values) { + if (accessor.dict_values->IsNull(dict_idx)) { + return Status::Invalid( + "FilterMapArrayBySelectedKeys found null dictionary map key at dictionary index " + + std::to_string(dict_idx)); + } + return accessor.dict_values->GetView(dict_idx); + } + + if (accessor.dict_large_values->IsNull(dict_idx)) { + return Status::Invalid( + "FilterMapArrayBySelectedKeys found null dictionary map key at dictionary index " + + std::to_string(dict_idx)); + } + return accessor.dict_large_values->GetView(dict_idx); +} + +} // namespace + +Result> NestedProjectionUtils::FilterMapArrayBySelectedKeys( + const std::shared_ptr& array, const std::vector& selected_keys, + arrow::MemoryPool* pool) { + if (selected_keys.empty() || !array || array->length() == 0) { + return array; + } + if (pool == nullptr) { + return Status::Invalid("FilterMapArrayBySelectedKeys requires a non-null memory pool"); + } + + if (array->type_id() != arrow::Type::MAP) { + return Status::Invalid(fmt::format( + "FilterMapArrayBySelectedKeys requires map array, got {}", array->type()->ToString())); + } + + auto map_array = std::static_pointer_cast(array); + auto map_type = std::static_pointer_cast(array->type()); + assert(map_array && map_type); + + auto key_array = map_array->keys(); + PAIMON_ASSIGN_OR_RAISE(MapKeyAccessor key_accessor, BuildMapKeyAccessor(key_array)); + + auto values_array = map_array->items(); + int64_t num_maps = map_array->length(); + + std::unordered_set deduplicated; + deduplicated.reserve(selected_keys.size()); + for (const auto& selected_key : selected_keys) { + if (!deduplicated.insert(selected_key).second) { + return Status::Invalid(fmt::format("Duplicate selected key '{}' in {} metadata", + selected_key, DataField::MAP_SELECTED_KEYS)); + } + } + + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr key_builder_u, + arrow::MakeBuilder(arrow::utf8(), pool)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr value_builder_u, + arrow::MakeBuilder(values_array->type(), pool)); + arrow::MapBuilder map_builder(pool, std::move(key_builder_u), std::move(value_builder_u)); + auto* key_builder = static_cast(map_builder.key_builder()); + auto* value_builder = map_builder.item_builder(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Reserve(num_maps)); + + for (int64_t map_idx = 0; map_idx < num_maps; ++map_idx) { + if (map_array->IsNull(map_idx)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.AppendNull()); + continue; + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Append()); + int64_t start = map_array->value_offset(map_idx); + int64_t end = map_array->value_offset(map_idx + 1); + + // Keep selected keys in the exact selected_keys order. + for (const auto& selected_key : selected_keys) { + for (int64_t entry_idx = start; entry_idx < end; ++entry_idx) { + PAIMON_ASSIGN_OR_RAISE(std::string_view key_view, + GetMapKeyViewAt(key_accessor, entry_idx)); + if (key_view == selected_key) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(key_builder->Append( + key_view.data(), static_cast(key_view.size()))); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + value_builder->AppendArraySlice(*values_array->data(), entry_idx, 1)); + } + } + } + } + + std::shared_ptr result_map; + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder.Finish(&result_map)); + return result_map; +} + +} // namespace paimon diff --git a/src/paimon/core/utils/nested_projection_utils.h b/src/paimon/core/utils/nested_projection_utils.h new file mode 100644 index 00000000..0677ea75 --- /dev/null +++ b/src/paimon/core/utils/nested_projection_utils.h @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "arrow/memory_pool.h" +#include "arrow/type.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/result.h" + +namespace paimon { + +/// Utility class for nested column pruning and map key selection. +class PAIMON_EXPORT NestedProjectionUtils { + public: + NestedProjectionUtils() = delete; + ~NestedProjectionUtils() = delete; + + static std::shared_ptr FindFieldByName(const arrow::FieldVector& fields, + const std::string& name); + + /// Extract the paimon field ID from an Arrow field's metadata ("paimon.id"). + /// Returns -1 if the metadata key is not present. + static int32_t GetPaimonFieldId(const std::shared_ptr& field); + + /// Find a child field in a STRUCT DataType by paimon field ID. + /// Returns nullptr if no child has the given ID. + static std::shared_ptr FindFieldByPaimonId( + const std::shared_ptr& struct_type, int32_t field_id); + + /// Recursively prune `data_type` so that only the sub-fields requested by + /// `read_type` are retained. Matching is done by paimon field ID to support + /// schema evolution (field renames). + /// + /// Supported nesting: STRUCT, LIST (element recurse), MAP (key/value recurse). + /// For atomic types, `data_type` is returned as-is. + /// + /// Returns std::nullopt when all sub-fields of a STRUCT are pruned away + /// (caller should skip this field entirely, mirroring Java's null return). + static Result>> PruneDataType( + const std::shared_ptr& read_type, + const std::shared_ptr& data_type); + + /// Returns true if `read_schema` requests a nested sub-field projection against + /// `file_schema` (same top-level field, but nested STRUCT/LIST/MAP subtree is pruned). + static Result HasNestedSubfieldProjection( + const std::shared_ptr& file_schema, + const std::shared_ptr& read_schema); + + /// Parse the "paimon.map.selected-keys" metadata from an Arrow field. + /// Returns an empty vector if the field is null, has no metadata, or the metadata key + /// is absent. + /// The metadata value is a comma-separated string, e.g. "key1,key2". + /// Empty tokens are preserved ("" means selecting empty-string keys), and duplicate + /// selected keys are rejected as invalid. + static Result> GetMapSelectedKeys( + const std::shared_ptr& field); + + /// Filter a MapArray so that only entries whose key is in `selected_keys` are kept. + /// Supports string keys and dictionary keys. + /// The output map entry order follows + /// `selected_keys` order, and duplicate selected keys are rejected. + /// Returns the original array unchanged if `selected_keys` is empty. + static Result> FilterMapArrayBySelectedKeys( + const std::shared_ptr& map_array, + const std::vector& selected_keys, arrow::MemoryPool* pool); + + private: + static Result HasNestedSubfieldProjectionType( + const std::shared_ptr& file_type, + const std::shared_ptr& read_type); +}; + +} // namespace paimon diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp new file mode 100644 index 00000000..14dd8858 --- /dev/null +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -0,0 +1,511 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/utils/nested_projection_utils.h" + +#include "arrow/array/array_nested.h" +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_dict.h" +#include "arrow/array/builder_nested.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/memory_pool.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +// Helper: create an arrow::Field with paimon.id metadata +static std::shared_ptr MakeField(const std::string& name, + const std::shared_ptr& type, + int32_t paimon_id) { + DataField data_field(paimon_id, arrow::field(name, type)); + return DataField::ConvertDataFieldToArrowField(data_field); +} + +// ============== GetPaimonFieldId ============== + +TEST(NestedProjectionUtilsTest, GetPaimonFieldIdPresent) { + auto field = MakeField("col", arrow::int32(), 42); + ASSERT_EQ(NestedProjectionUtils::GetPaimonFieldId(field), 42); +} + +TEST(NestedProjectionUtilsTest, GetPaimonFieldIdMissing) { + auto field = arrow::field("col", arrow::int32()); + ASSERT_EQ(NestedProjectionUtils::GetPaimonFieldId(field), -1); +} + +TEST(NestedProjectionUtilsTest, GetPaimonFieldIdNullptr) { + ASSERT_EQ(NestedProjectionUtils::GetPaimonFieldId(nullptr), -1); +} + +// ============== FindFieldByPaimonId ============== + +TEST(NestedProjectionUtilsTest, FindFieldByPaimonIdFound) { + auto struct_type = + arrow::struct_({MakeField("x", arrow::int32(), 1), MakeField("y", arrow::utf8(), 2)}); + auto found = NestedProjectionUtils::FindFieldByPaimonId(struct_type, 2); + ASSERT_NE(found, nullptr); + ASSERT_EQ(found->name(), "y"); +} + +TEST(NestedProjectionUtilsTest, FindFieldByPaimonIdNotFound) { + auto struct_type = arrow::struct_({MakeField("x", arrow::int32(), 1)}); + ASSERT_EQ(NestedProjectionUtils::FindFieldByPaimonId(struct_type, 99), nullptr); +} + +TEST(NestedProjectionUtilsTest, FindFieldByPaimonIdNonStruct) { + ASSERT_EQ(NestedProjectionUtils::FindFieldByPaimonId(arrow::int32(), 1), nullptr); +} + +// ============== PruneDataType ============== + +TEST(NestedProjectionUtilsTest, PruneDataTypeIdenticalTypes) { + auto type = arrow::int32(); + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(type, type)); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(result.value()->Equals(type)); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeAtomicType) { + // Different atomic types: return data_type + auto read_type = arrow::int64(); + auto data_type = arrow::int32(); + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_TRUE(result.value()->Equals(data_type)); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeStructPruneSubset) { + // data: STRUCT + // read: STRUCT + // expected: STRUCT + auto data_type = + arrow::struct_({MakeField("x", arrow::int32(), 1), MakeField("y", arrow::utf8(), 2), + MakeField("z", arrow::float64(), 3)}); + auto read_type = arrow::struct_({MakeField("x", arrow::int32(), 1)}); + + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(result.value()->num_fields(), 1); + ASSERT_EQ(result.value()->field(0)->name(), "x"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeStructAllFieldsPruned) { + // data: STRUCT + // read: STRUCT — no match + // expected: fail-fast (struct-internal schema evolution unsupported) + auto data_type = arrow::struct_({MakeField("x", arrow::int32(), 1)}); + auto read_type = arrow::struct_({MakeField("y", arrow::int32(), 99)}); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "does not support schema evolution inside struct"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeNestedStruct) { + // data: STRUCT(id=1)> + // read: STRUCT(id=1)> + // expected: STRUCT(id=1)> + auto inner_data = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto data_type = arrow::struct_({MakeField("inner", inner_data, 1)}); + + auto inner_read = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto read_type = arrow::struct_({MakeField("inner", inner_read, 1)}); + + ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(result.value()->num_fields(), 1); + auto pruned_inner = result.value()->field(0)->type(); + ASSERT_EQ(pruned_inner->num_fields(), 1); + ASSERT_EQ(pruned_inner->field(0)->name(), "a"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListWithStructElement) { + // data: LIST> + // read: LIST> + auto inner_data = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto data_type = arrow::list(arrow::field("item", inner_data)); + + auto inner_read = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto read_type = arrow::list(arrow::field("item", inner_read)); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "partial projection inside list"); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeMapWithStructValue) { + // data: MAP> + // read: MAP> + auto inner_data = + arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)}); + auto data_type = arrow::map(arrow::utf8(), inner_data); + + auto inner_read = arrow::struct_({MakeField("a", arrow::int32(), 10)}); + auto read_type = arrow::map(arrow::utf8(), inner_read); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "partial projection inside map"); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField("f1", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 3), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField("f1", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 3), + }); + ASSERT_OK_AND_ASSIGN( + auto has_nested_projection, + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema)); + ASSERT_FALSE(has_nested_projection); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionWithProjection) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField( + "f1", + arrow::struct_({MakeField("a", arrow::int32(), 2), MakeField("b", arrow::utf8(), 4)}), + 3), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField("f1", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 3), + }); + ASSERT_OK_AND_ASSIGN( + auto has_nested_projection, + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema)); + ASSERT_TRUE(has_nested_projection); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionTypeMismatchReturnsInvalid) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 1), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::list(arrow::field("item", arrow::int32())), 1), + }); + + ASSERT_NOK_WITH_MSG( + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema), + "requires same nested type kind"); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionAtomicTypeMismatchReturnsFalse) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::map(arrow::utf8(), arrow::int32()), 1), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::map(arrow::utf8(), arrow::int16()), 1), + }); + + ASSERT_OK_AND_ASSIGN( + auto has_nested_projection, + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema)); + ASSERT_FALSE(has_nested_projection); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionMissingStructChildReturnsInvalid) { + auto file_schema = arrow::schema({ + MakeField( + "f0", + arrow::struct_({MakeField("a", arrow::int32(), 2), MakeField("b", arrow::utf8(), 3)}), + 1), + }); + auto read_schema = arrow::schema({ + MakeField( + "f0", + arrow::struct_({MakeField("a", arrow::int32(), 2), MakeField("c", arrow::utf8(), 4)}), + 1), + }); + + ASSERT_NOK_WITH_MSG( + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema), + "requested struct child"); +} + +TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionMissingTopLevelFieldReturnsInvalid) { + auto file_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + }); + auto read_schema = arrow::schema({ + MakeField("f0", arrow::int32(), 1), + MakeField("f1", arrow::struct_({MakeField("a", arrow::int32(), 2)}), 3), + }); + + ASSERT_NOK_WITH_MSG( + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, read_schema), + "missing in file schema"); +} + +// ============== GetMapSelectedKeys ============== + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysPresent) { + auto metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"key1,key2,key3"}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_EQ(keys.size(), 3); + ASSERT_EQ(keys[0], "key1"); + ASSERT_EQ(keys[1], "key2"); + ASSERT_EQ(keys[2], "key3"); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysAbsent) { + auto field = arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_TRUE(keys.empty()); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysEmptyString) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {""}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_EQ(keys.size(), 1); + ASSERT_EQ(keys[0], ""); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysContainsEmptyToken) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a, ,b"}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_EQ(keys.size(), 3); + ASSERT_EQ(keys[0], "a"); + ASSERT_EQ(keys[1], " "); + ASSERT_EQ(keys[2], "b"); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysWhitespaceOnly) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {" "}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(field)); + ASSERT_EQ(keys.size(), 1); + ASSERT_EQ(keys[0], " "); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysDuplicateKey) { + auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,b,a"}); + auto field = + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32()), /*nullable=*/true, metadata); + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::GetMapSelectedKeys(field), + "Duplicate selected key 'a'"); +} + +TEST(NestedProjectionUtilsTest, GetMapSelectedKeysNullptr) { + ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(nullptr)); + ASSERT_TRUE(keys.empty()); +} + +// ============== FilterMapArrayBySelectedKeys ============== + +class NestedProjectionUtilsMapArrayTest : public ::testing::Test { + protected: + // Helper to build a MapArray from vectors of key-value pairs. + static std::shared_ptr BuildStringInt32MapArray( + const std::vector>>& maps, + const std::vector& null_mask = {}) { + auto key_builder = std::make_shared(); + auto value_builder = std::make_shared(); + arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, value_builder); + for (size_t i = 0; i < maps.size(); ++i) { + if (!null_mask.empty() && !null_mask[i]) { + EXPECT_TRUE(map_builder.AppendNull().ok()); + continue; + } + EXPECT_TRUE(map_builder.Append().ok()); + for (const auto& [k, v] : maps[i]) { + EXPECT_TRUE(key_builder->Append(k).ok()); + EXPECT_TRUE(value_builder->Append(v).ok()); + } + } + std::shared_ptr result; + EXPECT_TRUE(map_builder.Finish(&result).ok()); + return result; + } +}; + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysBasic) { + // Map with 3 entries each, select only "a" and "c" + auto map_array = BuildStringInt32MapArray({ + {{"a", 1}, {"b", 2}, {"c", 3}}, + {{"a", 10}, {"d", 40}}, + }); + + std::vector selected = {"a", "c"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + + auto expected = BuildStringInt32MapArray({ + {{"a", 1}, {"c", 3}}, + {{"a", 10}}, + }); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysEmptySelectedKeys) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}}}); + std::vector empty_keys; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, empty_keys, arrow::default_memory_pool())); + // Should return original array unchanged + ASSERT_EQ(filtered.get(), map_array.get()); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysAllKept) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}}}); + std::vector selected = {"a", "b"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + ASSERT_TRUE(filtered->Equals(map_array)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysNoneKept) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}}}); + std::vector selected = {"x", "y"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{}}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysEmptyStringKeySelected) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"", 9}, {"b", 2}}}); + std::vector selected = {""}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{{"", 9}}}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysWithNull) { + // maps[0] = {"a":1}, maps[1] = null, maps[2] = {"b":2,"c":3} + auto map_array = + BuildStringInt32MapArray({{{"a", 1}}, {}, {{"b", 2}, {"c", 3}}}, {true, false, true}); + + std::vector selected = {"a", "c"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{{"a", 1}}, {}, {{"c", 3}}}, {true, false, true}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysEmptyArray) { + auto map_array = BuildStringInt32MapArray({}); + std::vector selected = {"a"}; + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysSelectedOrderWins) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}, {"c", 3}}}); + std::vector selected = {"c", "a"}; + + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{{"c", 3}, {"a", 1}}}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysDuplicateSelectedKeys) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}}}); + std::vector selected = {"a", "a"}; + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::FilterMapArrayBySelectedKeys( + map_array, selected, arrow::default_memory_pool()), + "Duplicate selected key 'a'"); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysDictionaryStringKey) { + auto map_array = BuildStringInt32MapArray({ + {{"a", 1}, {"b", 2}, {"c", 3}}, + {{"c", 30}, {"a", 10}}, + }); + auto map = std::static_pointer_cast(map_array); + auto string_keys = std::static_pointer_cast(map->keys()); + + arrow::StringDictionaryBuilder dict_builder(arrow::default_memory_pool()); + for (int64_t i = 0; i < string_keys->length(); ++i) { + ASSERT_TRUE(dict_builder.Append(string_keys->GetView(i)).ok()); + } + auto dict_keys_result = dict_builder.Finish(); + ASSERT_TRUE(dict_keys_result.ok()); + auto dict_keys = dict_keys_result.ValueUnsafe(); + + auto dict_map_type = arrow::map(dict_keys->type(), map->items()->type()); + auto dict_map_array = std::make_shared( + dict_map_type, map->length(), map->value_offsets(), dict_keys, map->items(), + map->null_bitmap(), map->null_count(), map->offset()); + + std::vector selected = {"c", "a"}; + ASSERT_OK_AND_ASSIGN(auto filtered, + NestedProjectionUtils::FilterMapArrayBySelectedKeys( + dict_map_array, selected, arrow::default_memory_pool())); + + auto expected = BuildStringInt32MapArray({ + {{"c", 3}, {"a", 1}}, + {{"c", 30}, {"a", 10}}, + }); + ASSERT_TRUE(filtered->Equals(expected)); +} + +TEST_F(NestedProjectionUtilsMapArrayTest, FilterMapArrayBySelectedKeysDictionaryLargeStringKey) { + auto map_array = BuildStringInt32MapArray({{{"a", 1}, {"b", 2}}}); + auto map = std::static_pointer_cast(map_array); + + arrow::LargeStringBuilder dict_value_builder(arrow::default_memory_pool()); + ASSERT_TRUE(dict_value_builder.Append("a").ok()); + ASSERT_TRUE(dict_value_builder.Append("b").ok()); + std::shared_ptr dict_values; + ASSERT_TRUE(dict_value_builder.Finish(&dict_values).ok()); + + arrow::Int64Builder index_builder(arrow::default_memory_pool()); + ASSERT_TRUE(index_builder.Append(0).ok()); + ASSERT_TRUE(index_builder.Append(1).ok()); + std::shared_ptr indices; + ASSERT_TRUE(index_builder.Finish(&indices).ok()); + + auto dict_type = arrow::dictionary(arrow::int64(), arrow::large_utf8()); + auto large_string_dict_keys_result = + arrow::DictionaryArray::FromArrays(dict_type, indices, dict_values); + ASSERT_TRUE(large_string_dict_keys_result.ok()); + auto large_string_dict_keys = large_string_dict_keys_result.ValueUnsafe(); + + auto dict_map_type = arrow::map(large_string_dict_keys->type(), map->items()->type()); + auto dict_map_array = std::make_shared( + dict_map_type, map->length(), map->value_offsets(), large_string_dict_keys, map->items(), + map->null_bitmap(), map->null_count(), map->offset()); + + ASSERT_OK_AND_ASSIGN(auto filtered, NestedProjectionUtils::FilterMapArrayBySelectedKeys( + dict_map_array, {"a"}, arrow::default_memory_pool())); + auto expected = BuildStringInt32MapArray({{{"a", 1}}}); + ASSERT_TRUE(filtered->Equals(expected)); +} + +} // namespace paimon::test diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp index 5daf1b89..74a3b10f 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader.cpp @@ -29,6 +29,7 @@ #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/format/avro/avro_input_stream_impl.h" #include "paimon/format/avro/avro_schema_converter.h" #include "paimon/reader/batch_reader.h" @@ -148,7 +149,14 @@ Status AvroFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, arrow::ImportSchema(read_schema)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, ArrowUtils::DataTypeToSchema(file_data_type_)); - PAIMON_ASSIGN_OR_RAISE(std::set read_fields_projection, + PAIMON_ASSIGN_OR_RAISE( + bool has_nested_projection, + NestedProjectionUtils::HasNestedSubfieldProjection(file_schema, arrow_read_schema)); + if (has_nested_projection) { + return Status::Invalid( + "SetReadSchema failed: avro reader does not support nested sub-field projection"); + } + PAIMON_ASSIGN_OR_RAISE(read_fields_projection_, CalculateReadFieldsProjection(file_schema, arrow_read_schema->fields())); std::shared_ptr<::arrow::DataType> read_data_type = arrow::struct_(arrow_read_schema->fields()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr array_builder, @@ -160,7 +168,6 @@ Status AvroFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, reader_->close(); } reader_ = std::move(reader); - read_fields_projection_ = std::move(read_fields_projection); array_builder_ = std::move(array_builder); previous_first_row_ = std::numeric_limits::max(); next_row_to_read_ = std::numeric_limits::max(); diff --git a/src/paimon/format/avro/avro_file_batch_reader_test.cpp b/src/paimon/format/avro/avro_file_batch_reader_test.cpp index b1849931..13fb7656 100644 --- a/src/paimon/format/avro/avro_file_batch_reader_test.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader_test.cpp @@ -297,6 +297,38 @@ TEST_F(AvroFileBatchReaderTest, TestReadMapTypes) { ASSERT_TRUE(expected_array->Equals(result_array)); } +TEST_F(AvroFileBatchReaderTest, TestSetReadSchemaRejectNestedSubFieldProjection) { + std::string path = PathUtil::JoinPath(dir_->Str(), "nested_projection_unsupported.avro"); + + arrow::FieldVector write_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8())}))}; + auto write_type = arrow::struct_(write_fields); + auto write_array = arrow::ipc::internal::json::ArrayFromJSON(write_type, R"([ + [1, [10, "x"]], + [2, [20, "y"]] + ])") + .ValueOrDie(); + WriteData(write_array, path, /*compression=*/"null"); + + ASSERT_OK_AND_ASSIGN(auto reader_builder, + file_format_->CreateReaderBuilder(/*batch_size=*/1024)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(path)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, reader_builder->Build(in)); + + arrow::FieldVector read_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({arrow::field("a", arrow::int32())}))}; + auto read_schema = arrow::schema(read_fields); + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + + ASSERT_NOK_WITH_MSG(batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt), + "does not support nested sub-field projection"); +} + TEST_F(AvroFileBatchReaderTest, TestGetPreviousBatchFirstRowNumber) { std::string path = paimon::test::GetDataDir() + "/avro/append_simple.db/" diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h index 373d0159..6cc5464f 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.h +++ b/src/paimon/format/parquet/file_reader_wrapper.h @@ -116,6 +116,10 @@ class FileReaderWrapper { /// Prepare for immediate reading of the specified row groups and columns. /// Initializes the reader and starts pre-buffering I/O. + /// + /// Note: when the read schema has nested sub-field projection, + /// page-level filtering is disabled temporarily due to known offset + /// calculation issues for nested pages. Status PrepareForReading(const std::vector& target_row_groups, const std::vector& column_indices); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 456c4d3f..32ec420c 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -41,7 +41,9 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/schema/arrow_schema_validator.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/format/parquet/parquet_field_id_converter.h" #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_timestamp_converter.h" @@ -133,7 +135,6 @@ Status ParquetFileBatchReader::SetReadSchema( arrow::ImportSchema(schema)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema()); - std::unordered_map> field_index_map; bool has_nested_field = false; for (const auto& field : read_schema->fields()) { if (ArrowSchemaValidator::IsNestedType(field->type())) { @@ -141,22 +142,20 @@ Status ParquetFileBatchReader::SetReadSchema( break; } } - int32_t i = 0; - for (const auto& field : file_schema->fields()) { - std::vector v; - FlattenSchema(field->type(), &i, &v); - field_index_map[field->name()] = v; - } - std::vector column_indices; - for (const auto& field : read_schema->field_names()) { - if (field_index_map.find(field) != field_index_map.end()) { - for (int32_t index : field_index_map[field]) { - column_indices.push_back(index); - } - } else { - return Status::Invalid(fmt::format("Field {} is not found in schema.", field)); - } + // Recursively match read_schema against file_schema by field names. + // STRUCT supports sub-field projection; LIST/MAP require exact type match. + PAIMON_ASSIGN_OR_RAISE(std::vector column_indices, + ComputeNestedColumnIndices(read_schema, file_schema)); + + // Build column name to index map for page-level filtering. + // We still need the full per-top-level-field leaf indices for predicate pushdown. + std::unordered_map> field_index_map; + int32_t flat_idx = 0; + for (const auto& field : file_schema->fields()) { + std::vector leaf_indices; + FlattenSchema(field->type(), &flat_idx, &leaf_indices); + field_index_map[field->name()] = leaf_indices; } std::vector row_groups = arrow::internal::Iota(reader_->GetNumberOfRowGroups()); @@ -442,4 +441,91 @@ Result<::parquet::ArrowReaderProperties> ParquetFileBatchReader::CreateArrowRead return arrow_reader_props; } +// Nested column index computation + +Status ParquetFileBatchReader::CollectLeafIndices(const std::shared_ptr& read_type, + const std::shared_ptr& file_type, + int32_t* leaf_index, + std::vector* indices) { + if (file_type->id() == arrow::Type::STRUCT) { + for (const auto& file_child : file_type->fields()) { + std::shared_ptr read_child = nullptr; + for (const auto& candidate : read_type->fields()) { + if (candidate->name() == file_child->name()) { + read_child = candidate; + break; + } + } + if (read_child) { + PAIMON_RETURN_NOT_OK(CollectLeafIndices(read_child->type(), file_child->type(), + leaf_index, indices)); + } else { + SkipLeafIndices(file_child->type(), leaf_index); + } + } + } else if (file_type->id() == arrow::Type::LIST || file_type->id() == arrow::Type::MAP) { + // Keep behavior aligned with ORC path: list/map inner partial projection + // is currently unsupported and should fail-fast. + if (!read_type->Equals(file_type)) { + return Status::Invalid(fmt::format( + "Parquet does not support partial projection inside list/map: src {} vs target {}", + file_type->ToString(), read_type->ToString())); + } + for (int32_t i = 0; i < file_type->num_fields(); i++) { + PAIMON_RETURN_NOT_OK(CollectLeafIndices( + read_type->field(i)->type(), file_type->field(i)->type(), leaf_index, indices)); + } + } else { + // Leaf column — collect its index. + indices->push_back((*leaf_index)++); + } + return Status::OK(); +} + +void ParquetFileBatchReader::SkipLeafIndices(const std::shared_ptr& file_type, + int32_t* leaf_index) { + if (file_type->id() == arrow::Type::STRUCT || file_type->id() == arrow::Type::LIST || + file_type->id() == arrow::Type::MAP) { + for (int32_t i = 0; i < file_type->num_fields(); i++) { + SkipLeafIndices(file_type->field(i)->type(), leaf_index); + } + } else { + (*leaf_index)++; + } +} + +Result> ParquetFileBatchReader::ComputeNestedColumnIndices( + const std::shared_ptr& read_schema, + const std::shared_ptr& file_schema) { + std::vector indices; + std::vector file_field_leaf_starts; + file_field_leaf_starts.reserve(file_schema->num_fields()); + + int32_t file_leaf_index = 0; + for (const auto& file_field : file_schema->fields()) { + file_field_leaf_starts.push_back(file_leaf_index); + SkipLeafIndices(file_field->type(), &file_leaf_index); + } + + const auto& file_fields = file_schema->fields(); + for (const auto& read_field : read_schema->fields()) { + int32_t file_field_idx = -1; + for (int32_t i = 0; i < static_cast(file_fields.size()); ++i) { + if (file_fields[i]->name() == read_field->name()) { + file_field_idx = i; + break; + } + } + if (file_field_idx < 0) { + return Status::Invalid( + fmt::format("Field '{}' in read schema does not exist in parquet file schema", + read_field->name())); + } + int32_t leaf_index = file_field_leaf_starts[file_field_idx]; + PAIMON_RETURN_NOT_OK(CollectLeafIndices( + read_field->type(), file_fields[file_field_idx]->type(), &leaf_index, &indices)); + } + return indices; +} + } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 7bfc2e1e..592c128e 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -156,6 +156,25 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { } } + /// Recursively collect leaf column indices for the sub-fields in read_type + /// that match file_type by paimon field ID. Unmatched sub-fields in file_type + /// have their leaf indices skipped. Partial projection inside LIST/MAP is + /// not supported and will return Invalid. + static Status CollectLeafIndices(const std::shared_ptr& read_type, + const std::shared_ptr& file_type, + int32_t* leaf_index, std::vector* indices); + + /// Skip over all leaf column indices of the given file_type without collecting. + static void SkipLeafIndices(const std::shared_ptr& file_type, + int32_t* leaf_index); + + /// Compute leaf column indices by recursively matching read_schema against + /// file_schema using paimon field IDs. STRUCT supports sub-field projection + /// (unmatched sub-fields are skipped). LIST/MAP require exact type match. + static Result> ComputeNestedColumnIndices( + const std::shared_ptr& read_schema, + const std::shared_ptr& file_schema); + // precondition: predicate supposed not be empty Result> FilterRowGroupsByPredicate( const std::shared_ptr& predicate, diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index 2041166e..61a7caef 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -42,6 +42,7 @@ #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/defs.h" +#include "paimon/format/parquet/parquet_field_id_converter.h" #include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/parquet_format_writer.h" #include "paimon/format/parquet/parquet_reader_builder.h" @@ -114,6 +115,19 @@ class FailedUriInputStream : public InputStream { class ParquetFileBatchReaderTest : public ::testing::Test, public ::testing::WithParamInterface { public: + static std::shared_ptr WithMapSelectedKeys( + const std::shared_ptr& field, const std::string& selected_keys) { + auto metadata = + field->metadata() ? field->metadata()->Copy() : arrow::key_value_metadata({}); + auto set_status = metadata->Set(DataField::MAP_SELECTED_KEYS, selected_keys); + EXPECT_TRUE(set_status.ok()) << set_status.ToString(); + return field->WithMetadata(metadata); + } + + static std::shared_ptr MakeReadSchema(const arrow::FieldVector& fields) { + return arrow::schema(fields); + } + void SetUp() override { dir_ = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(dir_); @@ -397,6 +411,36 @@ TEST_F(ParquetFileBatchReaderTest, TestSetReadSchema) { ASSERT_FALSE(result_with_read_schema); } +TEST_F(ParquetFileBatchReaderTest, TestSetReadSchemaWithLegacyParquetMissingFieldIds) { + std::string file_name = paimon::test::GetDataDir() + + "/parquet/append_09.db/append_09/f1=20/bucket-0/" + "data-b446f78a-2cfb-4b3b-add8-31295d24a277-0.parquet"; + + std::vector read_fields = { + DataField(0, arrow::field("f0", arrow::utf8())), + DataField(2, arrow::field("f2", arrow::int32())), + DataField(3, arrow::field("f3", arrow::float64())), + }; + auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields); + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_name, read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, batch_size_); + + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(read_schema->fields()), {R"([ + ["Lucy", 1, 14.1] + ])"}, + &expected_array) + .ok()); + ASSERT_TRUE(result_array->Equals(expected_array)) + << "expected: " << expected_array->ToString() << "\nactual: " << result_array->ToString(); +} + TEST_F(ParquetFileBatchReaderTest, TestNextBatchSimple) { std::string file_name = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table/bucket-0/" @@ -503,6 +547,85 @@ TEST_F(ParquetFileBatchReaderTest, TestNextBatchWithDictionary) { check_result(false); } +TEST_F(ParquetFileBatchReaderTest, TestNestedStructChildProjectionRecall) { + auto f0 = arrow::field("f0", arrow::int32()); + auto f1 = arrow::field( + "f1", arrow::struct_({arrow::field("c0", arrow::int64()), arrow::field("c1", arrow::utf8()), + arrow::field("c2", arrow::float64())})); + auto f2 = arrow::field("f2", arrow::utf8()); + + auto write_schema = arrow::schema({f0, f1, f2}); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_schema->fields()), R"([ + [1, [100, "a", 1.1], "x"], + [2, [200, "b", 2.2], "y"], + [3, [300, null, 3.3], "z"] + ])") + .ValueOrDie()); + + WriteArray(file_path_, write_array, write_schema, + /*write_batch_size=*/write_array->length(), + /*enable_dictionary=*/false, /*max_row_group_length=*/write_array->length()); + + auto read_schema = MakeReadSchema({ + f0, + arrow::field("f1", arrow::struct_({arrow::field("c1", arrow::utf8())})), + }); + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, /*batch_size=*/2); + + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + arrow::struct_(read_schema->fields()), {R"([ + [1, ["a"]], + [2, ["b"]], + [3, [null]] + ])"}, + &expected_array) + .ok()); + + ASSERT_TRUE(result_array->Equals(expected_array)) + << "expected: " << expected_array->ToString() << "\nactual: " << result_array->ToString(); +} + +TEST_F(ParquetFileBatchReaderTest, TestReadSchemaWithMapSelectedKeysMetadata) { + auto id_field = arrow::field("id", arrow::int32()); + auto map_field = arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())); + + auto write_schema = arrow::schema({id_field, map_field}); + auto write_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_schema->fields()), R"([ + [1, [["k1", 10], ["k2", 20], ["k3", 30]]], + [2, [["k2", 200]]], + [3, null] + ])") + .ValueOrDie()); + + WriteArray(file_path_, write_array, write_schema, + /*write_batch_size=*/write_array->length(), + /*enable_dictionary=*/false, /*max_row_group_length=*/write_array->length()); + + // selected-keys metadata is consumed by upper-level field mapping; format reader should + // still accept the schema and read data correctly. + auto read_schema = MakeReadSchema( + {id_field, WithMapSelectedKeys(map_field, "k1,k3")}); // NOLINT(whitespace/comma) + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, read_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt, /*batch_size=*/2); + + ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( + parquet_batch_reader.get())); + auto expected_array = arrow::ChunkedArray::Make({write_array}).ValueOrDie(); + ASSERT_TRUE(result_array->Equals(expected_array)) + << "expected: " << expected_array->ToString() << "\nactual: " << result_array->ToString(); +} + TEST_F(ParquetFileBatchReaderTest, TestGetFileSchemaWithFieldId) { std::string file_name = paimon::test::GetDataDir() + "parquet/parquet_append_table.db/parquet_append_table/bucket-0/" diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt index e9e48146..27960afe 100644 --- a/test/inte/CMakeLists.txt +++ b/test/inte/CMakeLists.txt @@ -99,4 +99,11 @@ if(PAIMON_BUILD_TESTS) test_utils_static ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(nested_column_pruning_inte_test + STATIC_LINK_LIBS + paimon_shared + ${TEST_STATIC_LINK_LIBS} + test_utils_static + ${GTEST_LINK_TOOLCHAIN}) + endif() diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 9cf19492..9bd0797c 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -205,7 +205,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter const std::map& options = {}) const { auto splits = plan->Splits(); ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetReadSchema(read_schema).SetPredicate(predicate); + read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); if (!options.empty()) { read_context_builder.SetOptions(options); } diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 4c040fcf..f6a7b9b6 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -149,7 +149,7 @@ class DataEvolutionTableTest : public ::testing::Test, // read auto splits = result_plan->Splits(); ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetReadSchema(read_schema).SetPredicate(predicate); + read_context_builder.SetReadFieldNames(read_schema).SetPredicate(predicate); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index def51bf3..e1b76b68 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -197,7 +197,9 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter const std::shared_ptr& result_plan) const { auto splits = result_plan->Splits(); ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetReadSchema(read_schema).SetPredicate(predicate).WithFileSystem(fs_); + read_context_builder.SetReadFieldNames(read_schema) + .SetPredicate(predicate) + .WithFileSystem(fs_); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp new file mode 100644 index 00000000..c18eb752 --- /dev/null +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -0,0 +1,1649 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "arrow/ipc/json_simple.h" +#include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/defs.h" +#include "paimon/fs/file_system_factory.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/status.h" +#include "paimon/table/source/startup_mode.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/testing/utils/dict_array_converter.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/test_helper.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon { +class DataSplit; +class RecordBatch; +} // namespace paimon + +namespace paimon::test { + +class NestedColumnPruningInteTest : public ::testing::Test, + public ::testing::WithParamInterface { + void SetUp() override { + file_format_ = GetParam(); + dir_ = UniqueTestDirectory::Create("local"); + test_dir_ = dir_->Str(); + table_path_ = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + } + void TearDown() override { + dir_.reset(); + } + + void AssertChunkedArrayEquals(const std::shared_ptr& expected, + const std::shared_ptr& actual) const { + arrow::EqualOptions equal_options = arrow::EqualOptions::Defaults(); + bool is_equal = expected->Equals(actual, equal_options.diff_sink(&std::cout)); + if (!is_equal) { + std::cout << "[expected_type] " << expected->type()->ToString() << std::endl; + std::cout << "[actual_type] " << actual->type()->ToString() << std::endl; + std::cout << "[expected] " << expected->ToString() << std::endl; + std::cout << "[actual] " << actual->ToString() << std::endl; + } + ASSERT_TRUE(is_equal); + } + + protected: + std::string file_format_; + std::string test_dir_; + std::string table_path_; + std::unique_ptr dir_; +}; + +// Test: Table has struct field with 3 sub-fields, read only 1 sub-field via SetReadSchema. +TEST_P(NestedColumnPruningInteTest, PruneStructSubFields) { + // Table schema: f0 (int32), f1 (struct{a: int32, b: utf8, c: float64}) + auto struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + arrow::field("c", arrow::float64()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Write data + std::string data = R"([ + [1, [10, "hello", 1.1]], + [2, [20, "world", 2.2]], + [3, [30, "foo", 3.3]], + [4, [40, "bar", 4.4]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Scan to get splits + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Build projected schema: only read f0 (full) and f1.a (sub-field of struct) + auto pruned_struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + // Export to C ArrowSchema + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + // Read with projected schema + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + // Expected: struct with _VALUE_KIND, f0, f1{a} + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({arrow::field("a", arrow::int32())})), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [10]], + [0, 2, [20]], + [0, 3, [30]], + [0, 4, [40]] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: Projecting a STRUCT column as empty struct should return this column +// as all null values. +TEST_P(NestedColumnPruningInteTest, ProjectStructColumnAsEmptyStructReturnsNullColumn) { + auto struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "x"]], + [2, [22, "y"]], + [3, [33, "z"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Project f1 as empty struct. + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({})), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({})), + }); + std::string expected_data = R"([ + [0, 1, null], + [0, 2, null], + [0, 3, null] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: Two top-level struct columns have the same nested field name; projection should +// distinguish by parent column. +TEST_P(NestedColumnPruningInteTest, PruneSameNestedFieldNameFromDifferentStructColumns) { + // Table schema: f0 (int32), s0 (struct{f1: int32, a: utf8}), s1 (struct{f1: int32, b: utf8}) + auto s0_type = arrow::struct_({ + arrow::field("f1", arrow::int32()), + arrow::field("a", arrow::utf8()), + }); + auto s1_type = arrow::struct_({ + arrow::field("f1", arrow::int32()), + arrow::field("b", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("s0", s0_type), + arrow::field("s1", s1_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "left-1"], [101, "right-1"]], + [2, [22, "left-2"], [202, "right-2"]], + [3, [33, "left-3"], [303, "right-3"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Project only s0.f1 and s1.f1; both nested field names are identical. + auto projected_s0 = arrow::struct_({arrow::field("f1", arrow::int32())}); + auto projected_s1 = arrow::struct_({arrow::field("f1", arrow::int32())}); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("s0", projected_s0), + arrow::field("s1", projected_s1), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("s0", arrow::struct_({arrow::field("f1", arrow::int32())})), + arrow::field("s1", arrow::struct_({arrow::field("f1", arrow::int32())})), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [11], [101]], + [0, 2, [22], [202]], + [0, 3, [33], [303]] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: Querying only non-existent struct sub-fields should fail fast. +TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsAllNonExistent) { + // Table schema: f0 (int32), f1 (struct{f1: int32, f2: utf8, f3: float64}) + auto struct_type = arrow::struct_({ + arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::float64()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "a", 1.1]], + [2, [22, "b", 2.2]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Query struct sub-fields that do not exist in table schema. + auto projected_struct_type = arrow::struct_({ + arrow::field("f4", arrow::int32()), + arrow::field("f5", arrow::int64()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + "does not support schema evolution inside struct"); +} + +// Test: Querying a mix of existent and non-existent struct sub-fields should fail fast. +TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithNonExistentField) { + // Table schema: f0 (int32), f1 (struct{f1: int32, f2: utf8, f3: float64}) + auto struct_type = arrow::struct_({ + arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::float64()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "a", 1.1]], + [2, [22, "b", 2.2]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Query struct sub-fields f2,f3,f4 where f4 does not exist in table schema. + auto projected_struct_type = arrow::struct_({ + arrow::field("f2", arrow::utf8()), + arrow::field("f3", arrow::float64()), + arrow::field("f4", arrow::int32()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_NOK_WITH_MSG(TableRead::Create(std::move(read_context)), + "does not support schema evolution inside struct"); +} + +// Test: Nested schema divergence (simulated evolution mismatch) must fail fast +// instead of silently skipping nested fields. +TEST_P(NestedColumnPruningInteTest, QueryStructSubFieldsWithTypeMismatchShouldFail) { + // File schema (old): f1.a is INT32. + auto struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "x"]], + [2, [22, "y"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Persist schema-2 on disk so table latest schema diverges from old data files. + std::string file_system_identifier = "local"; + auto fs_iter = options.find(Options::FILE_SYSTEM); + if (fs_iter != options.end()) { + file_system_identifier = StringUtils::ToLowerCase(fs_iter->second); + } + ASSERT_OK_AND_ASSIGN(auto file_system, + FileSystemFactory::Get(file_system_identifier, table_path_, options)); + std::shared_ptr schema_fs(std::move(file_system)); + + SchemaManager schema_manager(schema_fs, table_path_); + ASSERT_OK_AND_ASSIGN(auto latest_schema_opt, schema_manager.Latest()); + ASSERT_TRUE(latest_schema_opt.has_value()); + auto latest_schema = latest_schema_opt.value(); + + auto schema_v2_arrow = arrow::schema({ + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("a", arrow::utf8()), + arrow::field("b", arrow::utf8()), + })), + }); + ASSERT_OK_AND_ASSIGN( + auto schema_v2, + TableSchema::Create(/*schema_id=*/latest_schema->Id() + 1, schema_v2_arrow, + latest_schema->PartitionKeys(), latest_schema->PrimaryKeys(), + latest_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto schema_v2_json, schema_v2->ToJsonString()); + auto schema_v2_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(schema_v2->Id())); + ASSERT_OK(schema_fs->AtomicStore(schema_v2_path, schema_v2_json)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + + ASSERT_NOK_WITH_MSG(table_read->CreateReader(data_splits), + "PruneDataType nested field type mismatch for 'a': read string vs " + "data int32"); +} + +// Test: With SetReadSchema using the new schema, context build should pass, +// and mismatch against old file type should be rejected in reader creation. +TEST_P(NestedColumnPruningInteTest, + QueryStructSubFieldsWithTypeMismatchAndSetReadSchemaFailAtContext) { + // File schema (old): f1.a is INT32. + auto struct_type = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("b", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [11, "x"]], + [2, [22, "y"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Persist schema-2 on disk so table latest schema diverges from old data files. + std::string file_system_identifier = "local"; + auto fs_iter = options.find(Options::FILE_SYSTEM); + if (fs_iter != options.end()) { + file_system_identifier = StringUtils::ToLowerCase(fs_iter->second); + } + ASSERT_OK_AND_ASSIGN(auto file_system, + FileSystemFactory::Get(file_system_identifier, table_path_, options)); + std::shared_ptr schema_fs(std::move(file_system)); + + SchemaManager schema_manager(schema_fs, table_path_); + ASSERT_OK_AND_ASSIGN(auto latest_schema_opt, schema_manager.Latest()); + ASSERT_TRUE(latest_schema_opt.has_value()); + auto latest_schema = latest_schema_opt.value(); + + auto schema_v2_arrow = arrow::schema({ + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("a", arrow::utf8()), + arrow::field("b", arrow::utf8()), + })), + }); + ASSERT_OK_AND_ASSIGN( + auto schema_v2, + TableSchema::Create(/*schema_id=*/latest_schema->Id() + 1, schema_v2_arrow, + latest_schema->PartitionKeys(), latest_schema->PrimaryKeys(), + latest_schema->Options())); + ASSERT_OK_AND_ASSIGN(auto schema_v2_json, schema_v2->ToJsonString()); + auto schema_v2_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(schema_v2->Id())); + ASSERT_OK(schema_fs->AtomicStore(schema_v2_path, schema_v2_json)); + + // User-provided read schema uses latest nested type (a:string). + auto projected_schema = arrow::schema({ + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("a", arrow::utf8()), + arrow::field("b", arrow::utf8()), + })), + }); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + ASSERT_NOK_WITH_MSG(table_read->CreateReader(data_splits), + "PruneDataType nested field type mismatch for 'a': read string vs " + "data int32"); +} + +// Test: Read only top-level fields, skip struct entirely. +TEST_P(NestedColumnPruningInteTest, PruneEntireStructField) { + auto struct_type = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("y", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + arrow::field("f2", arrow::float64()), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [100, [1, "aa"], 0.1], + [200, [2, "bb"], 0.2], + [300, [3, "cc"], 0.3] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + + // Only read f0 and f2, skip f1 entirely. + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f2", arrow::float64()), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f2", arrow::float64()), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 100, 0.1], + [0, 200, 0.2], + [0, 300, 0.3] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: Nested struct — prune sub-fields of a struct inside another struct. +TEST_P(NestedColumnPruningInteTest, PruneDeepNestedStruct) { + // Table schema: f0 (int32), f1 (struct{a: int32, inner: struct{x: int64, y: utf8}}) + auto inner_struct = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("y", arrow::utf8()), + }); + auto outer_struct = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("inner", inner_struct), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", outer_struct), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [10, [100, "aaa"]]], + [2, [20, [200, "bbb"]]], + [3, [30, [300, "ccc"]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + + // Projected: f0, f1{inner{x}} — skip f1.a and f1.inner.y + auto pruned_inner = arrow::struct_({ + arrow::field("x", arrow::int64()), + }); + auto pruned_outer = arrow::struct_({ + arrow::field("inner", pruned_inner), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_outer), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("inner", arrow::struct_({ + arrow::field("x", arrow::int64()), + })), + })), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [[100]]], + [0, 2, [[200]]], + [0, 3, [[300]]] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: Nested projected schema with special fields under row tracking. +TEST_P(NestedColumnPruningInteTest, PruneNestedStructWithSpecialFields) { + // Table schema: f0 (int32), f1 (struct{a: int32, inner: struct{x: int64, y: utf8}}) + auto inner_struct = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("y", arrow::utf8()), + }); + auto outer_struct = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("inner", inner_struct), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", outer_struct), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [10, [100, "aaa"]]], + [2, [20, [200, "bbb"]]], + [3, [30, [300, "ccc"]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + + // Projected: f0, f1{inner{x}}, _SEQUENCE_NUMBER, _ROW_ID + auto pruned_inner = arrow::struct_({ + arrow::field("x", arrow::int64()), + }); + auto pruned_outer = arrow::struct_({ + arrow::field("inner", pruned_inner), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_outer), + arrow::field("_SEQUENCE_NUMBER", arrow::int64()), + arrow::field("_ROW_ID", arrow::int64()), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + ASSERT_EQ(read_result->num_chunks(), 1); + auto result_array = std::dynamic_pointer_cast(read_result->chunk(0)); + ASSERT_TRUE(result_array); + + ASSERT_TRUE(result_array->GetFieldByName("_SEQUENCE_NUMBER")); + ASSERT_TRUE(result_array->GetFieldByName("_ROW_ID")); + auto nested_col = result_array->GetFieldByName("f1"); + ASSERT_TRUE(nested_col); + + auto expected_nested_type = arrow::struct_({ + arrow::field("inner", arrow::struct_({arrow::field("x", arrow::int64())})), + }); + ASSERT_TRUE(nested_col->type()->Equals(expected_nested_type)); + + auto expected_nested_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_nested_type, R"([ + [[100]], + [[200]], + [[300]] + ])") + .ValueOrDie(); + ASSERT_TRUE(nested_col->Equals(expected_nested_array)); +} + +// Test: Table has MAP field, read with selected keys filter. +TEST_P(NestedColumnPruningInteTest, MapSelectedKeys) { + // Table schema: f0 (int32), f1 (map) + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Write data: each row has a map with keys "a", "b", "c" + std::string data = R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]], + [2, [["a", 100], ["c", 300]]], + [3, [["b", 200], ["c", 400], ["d", 500]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Scan to get splits + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Build projected schema: read f0 and f1 with selected keys "a,c" + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type)->WithMetadata(selected_keys_metadata), + }; + auto projected_schema = arrow::schema(projected_fields); + + // Export to C ArrowSchema + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + // Read with projected schema + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + // Expected: only keys "a" and "c" remain in each map + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::map(arrow::utf8(), arrow::int32())), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [["a", 10], ["c", 30]]], + [0, 2, [["a", 100], ["c", 300]]], + [0, 3, [["c", 400]]] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: Selected-keys metadata on MAP nested inside STRUCT should be applied. +TEST_P(NestedColumnPruningInteTest, NestedMapSelectedKeysInStruct) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto struct_type = arrow::struct_({ + arrow::field("m", map_type), + arrow::field("tag", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [[ ["a", 10], ["b", 20], ["c", 30] ], "r1"]], + [2, [[ ["a", 100], ["c", 300] ], "r2"]], + [3, [[ ["b", 200], ["c", 400], ["d", 500] ], "r3"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + auto projected_struct_type = arrow::struct_({ + arrow::field("m", map_type)->WithMetadata(selected_keys_metadata), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())), + })), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [[ ["a", 10], ["c", 30] ]]], + [0, 2, [[ ["a", 100], ["c", 300] ]]], + [0, 3, [[ ["c", 400] ]]] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: Partial STRUCT sub-field recall where one recalled child is MAP with selected keys. +TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKeys) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto struct_type = arrow::struct_({ + arrow::field("m", map_type), + arrow::field("keep", arrow::int64()), + arrow::field("drop", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [[ ["a", 10], ["b", 20], ["c", 30] ], 1001, "x1"]], + [2, [[ ["a", 100], ["c", 300] ], 1002, "x2"]], + [3, [[ ["b", 200], ["c", 400], ["d", 500] ], 1003, "x3"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + auto projected_struct_type = arrow::struct_({ + arrow::field("m", map_type)->WithMetadata(selected_keys_metadata), + arrow::field("keep", arrow::int64()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())), + arrow::field("keep", arrow::int64()), + })), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [[ ["a", 10], ["c", 30] ], 1001]], + [0, 2, [[ ["a", 100], ["c", 300] ], 1002]], + [0, 3, [[ ["c", 400] ], 1003]] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: Null semantics should be preserved when pruning STRUCT sub-fields and +// applying selected-keys filtering on nested MAP. +TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKeysAndNulls) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto struct_type = arrow::struct_({ + arrow::field("m", map_type), + arrow::field("keep", arrow::int64()), + arrow::field("drop", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [[ ["a", 10], ["b", 20], ["c", 30] ], 1001, "x1"]], + [2, null], + [3, [null, 1003, "x3"]], + [4, [[ ["b", 200], ["c", 400], ["d", 500] ], null, "x4"]], + [5, [[ ["a", 500], ["c", null] ], 1005, "x5"]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + auto projected_struct_type = arrow::struct_({ + arrow::field("m", map_type)->WithMetadata(selected_keys_metadata), + arrow::field("keep", arrow::int64()), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", projected_struct_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::struct_({ + arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())), + arrow::field("keep", arrow::int64()), + })), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [[ ["a", 10], ["c", 30] ], 1001]], + [0, 2, null], + [0, 3, [null, 1003]], + [0, 4, [[ ["c", 400] ], null]], + [0, 5, [[ ["a", 500], ["c", null] ], 1005]] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: MAP_SELECTED_KEYS metadata value is empty string, select empty-string map key. +TEST_P(NestedColumnPruningInteTest, MapSelectedKeysEmptyStringKey) { + // Table schema: f0 (int32), f1 (map) + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Write data: each row has a map that may contain empty-string key. + std::string data = R"([ + [1, [["", 9], ["a", 10], ["c", 30]]], + [2, [["a", 100], ["", 99], ["c", 300]]], + [3, [["b", 200], ["c", 400], ["d", 500]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + // Scan to get splits + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Build projected schema: read f0 and f1 with selected keys metadata set to empty string. + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {""}); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type)->WithMetadata(selected_keys_metadata), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + // Read with projected schema + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + // Expected: only empty-string key remains. + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::map(arrow::utf8(), arrow::int32())), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [["", 9]]], + [0, 2, [["", 99]]], + [0, 3, []] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: MAP_SELECTED_KEYS output map entry order should follow selected key order. +TEST_P(NestedColumnPruningInteTest, MapSelectedKeysPreserveOrder) { + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Write data with map key order different from selected key order. + std::string data = R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]], + [2, [["a", 100], ["c", 300]]], + [3, [["c", 400], ["a", 500], ["d", 600]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + // Query key order is c,a and output should follow this order. + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"c,a"}); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type)->WithMetadata(selected_keys_metadata), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::map(arrow::utf8(), arrow::int32())), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [["c", 30], ["a", 10]]], + [0, 2, [["c", 300], ["a", 100]]], + [0, 3, [["c", 400], ["a", 500]]] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: ORC dictionary-encoded map key/value should work with MAP_SELECTED_KEYS. +TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) { + if (file_format_ != "orc") { + GTEST_SKIP() << "ORC-only dictionary encoding case"; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::utf8()); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"orc.dictionary-key-size-threshold", "1.0"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + // Low-cardinality map keys/values increase dictionary-encoding probability for ORC. + std::string data = R"([ + [1, [["a", "v1"], ["b", "v2"], ["c", "v3"]]], + [2, [["a", "v1"], ["c", "v3"], ["d", "v4"]]], + [3, [["a", "v1"], ["b", "v2"], ["e", "v5"]]], + [4, [["a", "v1"], ["c", "v3"], ["e", "v5"]]], + [5, [["a", "v1"], ["b", "v2"], ["c", "v3"]]], + [6, [["a", "v1"], ["c", "v3"], ["d", "v4"]]], + [7, [["a", "v1"], ["b", "v2"], ["e", "v5"]]], + [8, [["a", "v1"], ["c", "v3"], ["e", "v5"]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); + auto projected_schema = arrow::schema({ + arrow::field("f0", arrow::int32()), + arrow::field("f1", map_type)->WithMetadata(selected_keys_metadata), + }); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + ASSERT_OK_AND_ASSIGN( + auto decoded_result, + DictArrayConverter::ConvertDictArray(read_result->chunk(0), arrow::default_memory_pool())); + auto actual_chunked = std::make_shared(decoded_result); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::map(arrow::utf8(), arrow::utf8())), + }); + auto expected_array = arrow::ipc::internal::json::ArrayFromJSON(expected_type, R"([ + [0, 1, [["a", "v1"], ["c", "v3"]]], + [0, 2, [["a", "v1"], ["c", "v3"]]], + [0, 3, [["a", "v1"]]], + [0, 4, [["a", "v1"], ["c", "v3"]]], + [0, 5, [["a", "v1"], ["c", "v3"]]], + [0, 6, [["a", "v1"], ["c", "v3"]]], + [0, 7, [["a", "v1"]]], + [0, 8, [["a", "v1"], ["c", "v3"]]] + ])") + .ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, actual_chunked); +} + +// Test: Deeper nested struct — prune sub-fields of a struct inside a struct inside another struct. +TEST_P(NestedColumnPruningInteTest, PruneDeeperNestedStruct) { + // Table schema: f0 (int32), f1 (struct{a: int32, inner1: struct{x: int64, inner2: struct{p: + // utf8, q: float64}}}) + auto inner2_struct = arrow::struct_({ + arrow::field("p", arrow::utf8()), + arrow::field("q", arrow::float64()), + }); + auto inner1_struct = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("inner2", inner2_struct), + }); + auto outer_struct = arrow::struct_({ + arrow::field("a", arrow::int32()), + arrow::field("inner1", inner1_struct), + }); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", outer_struct), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [10, [100, ["ppp", 1.1]]]], + [2, [20, [200, ["qqq", 2.2]]]], + [3, [30, [300, ["rrr", 3.3]]]] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + + // Projected: f0, f1{inner1{inner2{p}}} + auto pruned_inner2 = arrow::struct_({ + arrow::field("p", arrow::utf8()), + }); + auto pruned_inner1 = arrow::struct_({ + arrow::field("inner2", pruned_inner2), + }); + auto pruned_outer = arrow::struct_({ + arrow::field("inner1", pruned_inner1), + }); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_outer), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = { + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", arrow::int32()), + arrow::field( + "f1", arrow::struct_({ + arrow::field("inner1", + arrow::struct_({ + arrow::field("inner2", arrow::struct_({ + arrow::field("p", arrow::utf8()), + })), + })), + })), + }; + auto expected_type = arrow::struct_(expected_fields); + std::string expected_data = R"([ + [0, 1, [[[ "ppp" ]]]], + [0, 2, [[[ "qqq" ]]]], + [0, 3, [[[ "rrr" ]]]] + ])"; + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); + auto expected_chunked = std::make_shared(expected_array); + + AssertChunkedArrayEquals(expected_chunked, read_result); +} + +// Test: Nested pruning for LIST> in integration path. +TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { + auto list_elem_struct = arrow::struct_({ + arrow::field("x", arrow::int64()), + arrow::field("y", arrow::utf8()), + arrow::field("z", arrow::float64()), + }); + auto list_struct_type = arrow::list(arrow::field("item", list_elem_struct)); + arrow::FieldVector table_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", list_struct_type), + }; + auto table_schema = arrow::schema(table_fields); + + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [[100, "a", 1.1], [200, "b", 2.2]]], + [2, [[300, "c", 3.3]]], + [3, []] + ])"; + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_FALSE(data_splits.empty()); + + auto pruned_list_elem_struct = arrow::struct_({arrow::field("x", arrow::int64())}); + auto pruned_list_type = arrow::list(arrow::field("item", pruned_list_elem_struct)); + arrow::FieldVector projected_fields = { + arrow::field("f0", arrow::int32()), + arrow::field("f1", pruned_list_type), + }; + auto projected_schema = arrow::schema(projected_fields); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + auto create_reader_result = table_read->CreateReader(data_splits); + ASSERT_NOK_WITH_MSG(create_reader_result, "partial projection inside list"); +} + +INSTANTIATE_TEST_SUITE_P(FileFormats, NestedColumnPruningInteTest, + ::testing::Values("parquet", "orc")); + +} // namespace paimon::test diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 0958014b..813e9f16 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -508,7 +508,7 @@ TEST_P(ReadInteTest, TestReadOnlyPartitionField) { ReadContextBuilder context_builder(path); context_builder.AddOption(Options::FILE_FORMAT, param.file_format); - context_builder.SetReadSchema({"dt"}); + context_builder.SetReadFieldNames({"dt"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.EnablePrefetch(param.enable_prefetch) .AddOption(Options::FILE_FORMAT, param.file_format) @@ -1369,7 +1369,7 @@ TEST_P(ReadInteTest, TestAppendReadWithMultipleBuckets) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2") @@ -1449,7 +1449,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicate) { paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .SetPredicate(predicate) @@ -1553,7 +1553,7 @@ TEST_P(ReadInteTest, TestAppendReadWithComplexTypePredicate) { "/append_complex_data.db/append_complex_data"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f6", "f2", "f4", "f3", "f5"}); + context_builder.SetReadFieldNames({"f6", "f2", "f4", "f3", "f5"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPredicate(predicate); @@ -1626,7 +1626,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateOnlyPushdown) { ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2") .AddOption("test.enable-adaptive-prefetch-strategy", @@ -1702,7 +1702,7 @@ TEST_P(ReadInteTest, TestAppendReadWithPredicateAllFiltered) { ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2") .AddOption("test.enable-adaptive-prefetch-strategy", @@ -1787,7 +1787,7 @@ TEST_P(ReadInteTest, TestAppendReadIOException) { io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); ReadContextBuilder context_builder(paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09/"); - context_builder.SetReadSchema({"f3", "f0", "f1"}); + context_builder.SetReadFieldNames({"f3", "f0", "f1"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2") @@ -2031,7 +2031,7 @@ TEST_P(ReadInteTest, TestPkTableWithSnapshot8) { std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/pk_09.db/pk_09"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f0", "f3", "f1"}); + context_builder.SetReadFieldNames({"f0", "f3", "f1"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -2205,7 +2205,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateFilter) { "/append_table_with_alter_table.db/append_table_with_alter_table/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"a", "k", "key1", "d", "key0", "c"}); + context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPredicate(predicate); @@ -2284,7 +2284,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithPredicateOnlyPushDown) "append_table_with_alter_table/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"a", "k", "key1", "d", "key0", "c"}); + context_builder.SetReadFieldNames({"a", "k", "key1", "d", "key0", "c"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPredicate(predicate); @@ -2357,7 +2357,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot5WithSchemaEvolution) { "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); + context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -2442,7 +2442,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolution) { "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); + context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -2526,7 +2526,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateOnlyPush ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({equal, less_than})); ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); + context_builder.SetReadFieldNames({{"key1", "k", "key_2", "c", "d", "a", "key0", "e"}}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPrefetchCacheMode(param.cache_mode); @@ -2609,7 +2609,7 @@ TEST_P(ReadInteTest, TestPkReadSnapshot6WithSchemaEvolutionWithPredicateFilter) ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); + context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.SetPredicate(predicate); @@ -2701,7 +2701,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithBuildInFieldId) { ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"key0", "key1", "k", "c", "d", "a", "e"}); + context_builder.SetReadFieldNames({"key0", "key1", "k", "c", "d", "a", "e"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -2819,7 +2819,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCast) { "append_table_alter_table_with_cast/"; ReadContextBuilder context_builder(path); context_builder.SetPrefetchCacheMode(param.cache_mode); - context_builder.SetReadSchema({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); + context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); context_builder.EnablePrefetch(param.enable_prefetch) @@ -2900,7 +2900,7 @@ TEST_P(ReadInteTest, TestAppendReadWithSchemaEvolutionWithCastWithPredicatePushD "/append_table_alter_table_with_cast.db/" "append_table_alter_table_with_cast/"; ReadContextBuilder context_builder(path); - context_builder.SetReadSchema({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); + context_builder.SetReadFieldNames({"f4", "key0", "key1", "f3", "f1", "f2", "f0", "f6"}); context_builder.SetPrefetchCacheMode(param.cache_mode); context_builder.AddOption(Options::FILE_FORMAT, param.file_format) .AddOption("read.batch-size", "2"); diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index 56715571..bd0d5cb6 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -1028,7 +1028,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { AddReadOptionsForPrefetch(&read_context_builder); ASSERT_OK_AND_ASSIGN( auto read_context, - read_context_builder.SetReadSchema({"shopId", "dt", "hr", "col0", "col1", "col2"}) + read_context_builder.SetReadFieldNames({"shopId", "dt", "hr", "col0", "col1", "col2"}) .Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); @@ -2180,7 +2180,7 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.SetReadSchema({"f10", "f8", "f4", "f13"}); + read_context_builder.SetReadFieldNames({"f10", "f8", "f4", "f13"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); @@ -2236,7 +2236,7 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { ReadContextBuilder read_context_builder(table_path); AddReadOptionsForPrefetch(&read_context_builder); - read_context_builder.SetReadSchema({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); + read_context_builder.SetReadFieldNames({"key1", "k", "key_2", "c", "d", "a", "key0", "e"}); ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 2c7fcf64..6529fc11 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -288,7 +288,7 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface const std::string& blob_field, const std::vector>& expected_blobs) const { ReadContextBuilder read_context_builder(table_path); - read_context_builder.SetOptions(options).SetReadSchema({blob_field}); + read_context_builder.SetOptions(options).SetReadFieldNames({blob_field}); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); From 73f72675bff2aea2c28ad86456a5e9d14935d8ef Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:44:12 +0800 Subject: [PATCH 072/138] fix: close input/output streams when remote lookup file read/write failed --- cmake_modules/CorrosionFetch.cmake | 2 +- .../lookup_merge_tree_compact_rewriter.h | 4 +++- ...ookup_merge_tree_compact_rewriter_test.cpp | 2 ++ .../lookup/remote_lookup_file_manager.cpp | 19 +++++++++++++++++-- .../lookup/remote_lookup_file_manager.h | 1 - .../lucene/lucene_global_index_reader.cpp | 5 ++++- .../tantivy/tantivy_streaming_test.cpp | 4 ++-- .../tantivy/tantivy_writer_test.cpp | 7 ------- test/inte/append_compaction_inte_test.cpp | 1 + test/inte/write_and_read_inte_test.cpp | 5 ----- 10 files changed, 30 insertions(+), 20 deletions(-) diff --git a/cmake_modules/CorrosionFetch.cmake b/cmake_modules/CorrosionFetch.cmake index 926919ad..40ab1d2b 100644 --- a/cmake_modules/CorrosionFetch.cmake +++ b/cmake_modules/CorrosionFetch.cmake @@ -18,7 +18,7 @@ # targets. Used to bring in third_party/tantivy_ffi for the tantivy-fulltext # global index (see docs/dev/tantivy_fts_migration_plan.md). # -# Pinned to v0.5.0 (stable release). Requires CMake >= 3.22. +# Pinned to v0.5.2 (stable release). Requires CMake >= 3.22. include(FetchContent) diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h index 7890ed25..85e9aba0 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h @@ -17,6 +17,7 @@ */ #pragma once + #include "arrow/api.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" @@ -26,7 +27,9 @@ #include "paimon/core/mergetree/lookup/remote_lookup_file_manager.h" #include "paimon/core/mergetree/lookup_levels.h" #include "paimon/core/schema/table_schema.h" + namespace paimon { + /// A `MergeTreeCompactRewriter` which produces changelog files by lookup for the compaction /// involving level 0 files. template @@ -88,7 +91,6 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter { Result>> NotifyRewriteCompactAfter( const std::vector>& files) override; - private: std::unique_ptr> lookup_levels_; std::shared_ptr dv_maintainer_; std::shared_ptr remote_lookup_file_manager_; diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp index 48e880b8..f33b8b86 100644 --- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp +++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp @@ -55,7 +55,9 @@ #include "paimon/testing/utils/io_exception_helper.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" + namespace paimon::test { + class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParam { public: void SetUp() override { diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp index 2d686852..6d962572 100644 --- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp +++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.cpp @@ -19,8 +19,10 @@ #include "fmt/format.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/scope_guard.h" #include "paimon/core/mergetree/lookup/file_position.h" #include "paimon/core/mergetree/lookup/positioned_key_value.h" + namespace paimon { RemoteLookupFileManager::RemoteLookupFileManager( @@ -105,6 +107,15 @@ Status RemoteLookupFileManager::CopyRemoteToLocal(const std::string& remote_path Status RemoteLookupFileManager::CopyFromInputToOutput( std::unique_ptr&& input_stream, std::unique_ptr&& output_stream) const { + ScopeGuard input_close_guard([&input_stream]() -> void { + Status s = input_stream->Close(); + (void)s; + }); + ScopeGuard output_close_guard([&output_stream]() -> void { + Status s = output_stream->Close(); + (void)s; + }); + auto buffer = std::make_shared(kBufferSize, pool_.get()); PAIMON_ASSIGN_OR_RAISE(int64_t total_length, input_stream->Length()); PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(total_length, "input stream length")); @@ -130,8 +141,12 @@ Status RemoteLookupFileManager::CopyFromInputToOutput( write_size += current_read_size; } PAIMON_RETURN_NOT_OK(output_stream->Flush()); - PAIMON_RETURN_NOT_OK(output_stream->Close()); - PAIMON_RETURN_NOT_OK(input_stream->Close()); + Status output_close_status = output_stream->Close(); + output_close_guard.Release(); + PAIMON_RETURN_NOT_OK(output_close_status); + Status input_close_status = input_stream->Close(); + input_close_guard.Release(); + PAIMON_RETURN_NOT_OK(input_close_status); return Status::OK(); } template Result> diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.h b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.h index 1d0b50a8..d8f9e7da 100644 --- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.h +++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager.h @@ -52,7 +52,6 @@ class RemoteLookupFileManager { static constexpr uint64_t kBufferSize = 1024 * 1024; - private: int32_t level_threshold_; std::shared_ptr pool_; std::shared_ptr path_factory_; diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index 02edd4d3..41daa8b8 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -210,7 +210,10 @@ std::shared_ptr LuceneGlobalIndexReader::SearchWithNoLimit( Result> LuceneGlobalIndexReader::VisitFullTextSearch( const std::shared_ptr& full_text_search) { - if (full_text_search && full_text_search->min_score.has_value()) { + if (!full_text_search) { + return Status::Invalid("VisitFullTextSearch: null FullTextSearch pointer"); + } + if (full_text_search->min_score.has_value()) { // The lucene backend does not support min_score pushdown. Fail loudly // instead of silently ignoring the threshold and returning unfiltered // results, which would be a correctness bug for the caller. diff --git a/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp b/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp index 40283fcb..36ca7a3a 100644 --- a/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp +++ b/src/paimon/global_index/tantivy/tantivy_streaming_test.cpp @@ -83,7 +83,7 @@ struct WriteResult { class StreamingTestFixture : public ::testing::Test { public: WriteResult BuildArchive(std::size_t n_docs, - const std::string& text_template = "apple banana cherry {}") { + const std::string& text_template = "apple banana cherry %zu") { auto root_dir = paimon::test::UniqueTestDirectory::Create(); EXPECT_TRUE(root_dir); std::string root = root_dir->Str(); @@ -229,7 +229,7 @@ TEST(ParseArchiveHeaderFuzz, PayloadLenNegative) { TEST_F(StreamingTestFixture, ConcurrentQueryOnSameReader) { // 50 docs containing "apple" in every one (all should match) - auto wr = BuildArchive(50, "apple banana {}"); + auto wr = BuildArchive(50, "apple banana %zu"); auto reader = OpenReader(wr.root_dir, wr.meta); auto fts = BuildMatchAll("apple"); diff --git a/src/paimon/global_index/tantivy/tantivy_writer_test.cpp b/src/paimon/global_index/tantivy/tantivy_writer_test.cpp index 1b51fc7f..0d623b88 100644 --- a/src/paimon/global_index/tantivy/tantivy_writer_test.cpp +++ b/src/paimon/global_index/tantivy/tantivy_writer_test.cpp @@ -127,13 +127,6 @@ std::vector ParsePacked(const std::vector& bytes) { class TantivyGlobalIndexWriterTest : public ::testing::Test { public: - std::unique_ptr<::ArrowSchema> CreateArrowSchema( - const std::shared_ptr& data_type) const { - auto c_schema = std::make_unique<::ArrowSchema>(); - EXPECT_TRUE(arrow::ExportType(*data_type, c_schema.get()).ok()); - return c_schema; - } - Result> WriteIndex( const std::string& root, const std::shared_ptr& data_type, const std::map& options, diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index a80ebe90..418a0e96 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -119,6 +119,7 @@ class AppendCompactionInteTest : public testing::Test, auto partition = BinaryRowGenerator::GenerateRow({10}, pool_.get()); int32_t bucket = 1; auto abstract_write = dynamic_cast(helper->write_.get()); + ASSERT_NE(abstract_write, nullptr); ASSERT_OK_AND_ASSIGN(auto restore_files, abstract_write->ScanExistingFileMetas(partition, bucket)); ASSERT_OK_AND_ASSIGN( diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index d1c7956e..e85c2344 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -49,11 +49,6 @@ #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" -namespace paimon { -class DataSplit; -class RecordBatch; -} // namespace paimon - namespace paimon::test { // This is a sdk end-to-end test demo that supports write, commit, scan, and read operations. class WriteAndReadInteTest From 699b1ee5e991810e0636a31c7102d2e6fdb19557 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:02:32 +0800 Subject: [PATCH 073/138] fix(testing): address review issues --- include/paimon/memory/memory_pool.h | 3 +- .../prefetch_file_batch_reader_impl_test.cpp | 31 +++++++++++----- src/paimon/common/utils/arrow/arrow_utils.cpp | 1 + src/paimon/common/utils/arrow/arrow_utils.h | 1 - .../common/utils/arrow/arrow_utils_test.cpp | 1 + src/paimon/common/utils/arrow/mem_utils.cpp | 27 +++++++++++++- src/paimon/common/utils/arrow/mem_utils.h | 3 -- .../format/orc/orc_file_batch_reader_test.cpp | 7 +++- .../testing/mock/mock_file_batch_reader.h | 37 +++++++++++++------ .../testing/mock/mock_format_writer.cpp | 3 +- .../testing/mock/mock_format_writer_builder.h | 2 - .../mock_key_value_data_file_record_reader.h | 1 + test/inte/blob_table_inte_test.cpp | 10 ++--- 13 files changed, 88 insertions(+), 39 deletions(-) diff --git a/include/paimon/memory/memory_pool.h b/include/paimon/memory/memory_pool.h index b4ed71c7..54176dff 100644 --- a/include/paimon/memory/memory_pool.h +++ b/include/paimon/memory/memory_pool.h @@ -58,7 +58,8 @@ class PAIMON_EXPORT MemoryPool { /// /// @param size Number of bytes to allocate. /// @param alignment Memory alignment requirement (0 for default alignment). - /// @return Pointer to allocated memory, or nullptr on failure. + /// @return Pointer to allocated memory. + /// @throws std::bad_alloc if the allocation fails. virtual void* Malloc(uint64_t size, uint64_t alignment = 0) = 0; /// Reallocate memory to a new size. diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp index 4d9fcf09..3386a1bc 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp @@ -19,6 +19,7 @@ #include "paimon/common/reader/prefetch_file_batch_reader_impl.h" #include +#include #include #include "arrow/compute/api.h" @@ -281,7 +282,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestSimple) { /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult( reader.get(), /*max simulated data processing time*/ 100)); @@ -603,7 +605,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithLargeBatchSize) { prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult( reader.get(), /*max simulated data processing time*/ 100)); @@ -631,7 +634,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPartialReaderSuccessRead) { } arrow::ArrayVector result_array_vector; - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_OK_AND_ASSIGN(auto batch_with_bitmap, reader->NextBatchWithBitmap()); auto& [batch, bitmap] = batch_with_bitmap; ASSERT_EQ(batch.first->length, bitmap.Cardinality()); @@ -676,9 +680,11 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestAllReaderFailedWithIOError) { ->SetNextBatchStatus(Status::IOError("mock error")); } - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); auto batch_result = reader->NextBatchWithBitmap(); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_FALSE(batch_result.ok()); ASSERT_TRUE(batch_result.status().IsIOError()); ASSERT_FALSE(prefetch_reader->is_shutdown_); @@ -687,7 +693,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestAllReaderFailedWithIOError) { // call NextBatch again, will still return error status auto batch_result2 = reader->NextBatchWithBitmap(); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_FALSE(batch_result2.ok()); ASSERT_TRUE(batch_result2.status().IsIOError()); } @@ -704,7 +711,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithEmptyData) { prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult( reader.get(), /*max simulated data processing time*/ 100)); @@ -724,7 +732,8 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCallNextBatchAfterReadingEof) { prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult( reader.get(), /*max simulated data processing time*/ 100)); @@ -830,7 +839,8 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCom PreparePrefetchReader(file_format, schema.get(), predicate, /*selection_bitmap=*/std::nullopt, /*batch_size=*/10, /*prefetch_max_parallel_num=*/3, cache_mode); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult( reader.get(), /*max simulated data processing time*/ 100)); @@ -866,7 +876,8 @@ TEST_P(PrefetchFileBatchReaderImplTest, /*selection_bitmap=*/std::nullopt, /*batch_size=*/10, /*prefetch_max_parallel_num=*/3, cache_mode); ASSERT_OK(reader->RefreshReadRanges()); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult( reader.get(), /*max simulated data processing time*/ 100)); diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp index e9f4f1a9..94849f59 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils.cpp @@ -22,6 +22,7 @@ #include "arrow/array/array_base.h" #include "arrow/array/array_nested.h" #include "arrow/util/compression.h" +#include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/string_utils.h" diff --git a/src/paimon/common/utils/arrow/arrow_utils.h b/src/paimon/common/utils/arrow/arrow_utils.h index 52c521d2..439609f9 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.h +++ b/src/paimon/common/utils/arrow/arrow_utils.h @@ -23,7 +23,6 @@ #include "arrow/api.h" #include "arrow/util/type_fwd.h" -#include "fmt/format.h" #include "paimon/result.h" namespace paimon { diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp b/src/paimon/common/utils/arrow/arrow_utils_test.cpp index 8908f679..fc1b1c24 100644 --- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp @@ -24,6 +24,7 @@ #include "gtest/gtest.h" #include "paimon/common/types/data_field.h" #include "paimon/testing/utils/testharness.h" + namespace paimon::test { TEST(ArrowUtilsTest, TestCreateProjection) { diff --git a/src/paimon/common/utils/arrow/mem_utils.cpp b/src/paimon/common/utils/arrow/mem_utils.cpp index 4ecb644f..7e8986be 100644 --- a/src/paimon/common/utils/arrow/mem_utils.cpp +++ b/src/paimon/common/utils/arrow/mem_utils.cpp @@ -21,10 +21,12 @@ #include #include +#include #include #include "arrow/memory_pool.h" #include "arrow/status.h" +#include "fmt/format.h" #include "paimon/memory/memory_pool.h" namespace paimon { @@ -35,14 +37,35 @@ class ArrowMemPoolAdaptor : public arrow::MemoryPool { : pool_(*pool), life_holder_(pool) {} arrow::Status Allocate(int64_t size, int64_t alignment, uint8_t** out) override { - *out = reinterpret_cast(pool_.Malloc(size, alignment)); + uint8_t* new_out = nullptr; + try { + new_out = reinterpret_cast(pool_.Malloc(size, alignment)); + } catch (const std::bad_alloc&) { + return arrow::Status::OutOfMemory(fmt::format("failed to allocate {} bytes", size)); + } + if (size > 0 && new_out == nullptr) { + return arrow::Status::OutOfMemory(fmt::format("failed to allocate {} bytes", size)); + } + *out = new_out; stats_.DidAllocateBytes(size); return arrow::Status::OK(); } arrow::Status Reallocate(int64_t old_size, int64_t new_size, int64_t alignment, uint8_t** ptr) override { - *ptr = reinterpret_cast(pool_.Realloc(*ptr, old_size, new_size, alignment)); + uint8_t* new_ptr = nullptr; + try { + new_ptr = + reinterpret_cast(pool_.Realloc(*ptr, old_size, new_size, alignment)); + } catch (const std::bad_alloc&) { + return arrow::Status::OutOfMemory( + fmt::format("failed to reallocate memory from {} to {} bytes", old_size, new_size)); + } + if (new_size > 0 && new_ptr == nullptr) { + return arrow::Status::OutOfMemory( + fmt::format("failed to reallocate memory from {} to {} bytes", old_size, new_size)); + } + *ptr = new_ptr; stats_.DidReallocateBytes(old_size, new_size); return arrow::Status::OK(); } diff --git a/src/paimon/common/utils/arrow/mem_utils.h b/src/paimon/common/utils/arrow/mem_utils.h index b59c4031..96b59e3e 100644 --- a/src/paimon/common/utils/arrow/mem_utils.h +++ b/src/paimon/common/utils/arrow/mem_utils.h @@ -26,9 +26,6 @@ #include "paimon/visibility.h" namespace paimon { -class MemoryPool; - -PAIMON_EXPORT std::unique_ptr GetArrowPool(MemoryPool& pool); PAIMON_EXPORT std::unique_ptr GetArrowPool( const std::shared_ptr& pool); diff --git a/src/paimon/format/orc/orc_file_batch_reader_test.cpp b/src/paimon/format/orc/orc_file_batch_reader_test.cpp index b1655850..135e7c5a 100644 --- a/src/paimon/format/orc/orc_file_batch_reader_test.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader_test.cpp @@ -18,6 +18,7 @@ #include "paimon/format/orc/orc_file_batch_reader.h" +#include #include #include #include @@ -494,7 +495,8 @@ TEST_P(OrcFileBatchReaderTest, TestNextBatchSimple) { for (auto batch_size : {1, 2, 3, 5, 8, 10}) { auto orc_batch_reader = PrepareOrcFileBatchReader(file_name, &read_schema, batch_size, natural_read_size); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + orc_batch_reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( orc_batch_reader.get())); ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 8); @@ -768,7 +770,8 @@ TEST_F(OrcFileBatchReaderTest, TestReadNoField) { auto orc_batch_reader = PrepareOrcFileBatchReader(file_name, &read_schema, /*batch_size=*/3, /*natural_read_size=*/10); // read 3 rows - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), -1); + ASSERT_EQ(std::numeric_limits::max(), + orc_batch_reader->GetPreviousBatchFirstRowNumber().value()); ASSERT_OK_AND_ASSIGN(auto batch1, orc_batch_reader->NextBatch()); ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 0); // read 3 rows diff --git a/src/paimon/testing/mock/mock_file_batch_reader.h b/src/paimon/testing/mock/mock_file_batch_reader.h index 40a7bcb9..b50e053a 100644 --- a/src/paimon/testing/mock/mock_file_batch_reader.h +++ b/src/paimon/testing/mock/mock_file_batch_reader.h @@ -19,7 +19,11 @@ #pragma once #include +#include +#include +#include #include +#include #include #include @@ -28,8 +32,8 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/common/utils/date_time_utils.h" #include "paimon/reader/prefetch_file_batch_reader.h" + namespace paimon::test { class MockFileBatchReader : public PrefetchFileBatchReader { @@ -41,13 +45,14 @@ class MockFileBatchReader : public PrefetchFileBatchReader { file_schema_(file_schema), read_schema_(arrow::schema(file_schema->fields())), batch_size_(read_batch_size) { + assert(read_batch_size > 0); // add all valid bitmap - int32_t data_length = data_ ? data_->length() : 0; + int64_t data_length = data_ ? data_->length() : 0; + assert(data_length >= 0); + assert(data_length <= static_cast(std::numeric_limits::max())); bitmap_ = RoaringBitmap32(); - bitmap_.AddRange(0, data_length); - read_end_pos_ = data_length; - int64_t seed = DateTimeUtils::GetCurrentUTCTimeUs(); - std::srand(seed); + bitmap_.AddRange(0, static_cast(data_length)); + read_end_pos_ = static_cast(data_length); } MockFileBatchReader(const std::shared_ptr& data, @@ -95,7 +100,8 @@ class MockFileBatchReader : public PrefetchFileBatchReader { } Status SeekToRow(uint64_t row_number) override { - current_pos_ = row_number; + assert(row_number <= static_cast(std::numeric_limits::max())); + current_pos_ = static_cast(row_number); return Status::OK(); } @@ -119,7 +125,8 @@ class MockFileBatchReader : public PrefetchFileBatchReader { } int32_t actual_batch_size = batch_size_; if (enable_randomize_batch_size_) { - actual_batch_size = std::rand() % batch_size_ + 1; + std::uniform_int_distribution distribution(1, batch_size_); + actual_batch_size = distribution(random_engine_); } int32_t batch_end_pos = std::min(read_end_pos_, current_pos_ + actual_batch_size); auto slice = data_->Slice(current_pos_, batch_end_pos - current_pos_); @@ -152,14 +159,14 @@ class MockFileBatchReader : public PrefetchFileBatchReader { } Result GetPreviousBatchFirstRowNumber() const override { - return previous_batch_first_row_num_; + return ToReaderRowNumber(previous_batch_first_row_num_); } Result GetNumberOfRows() const override { - return data_ ? data_->length() : 0; + return ToReaderRowNumber(read_end_pos_); } uint64_t GetNextRowToRead() const override { - return current_pos_; + return ToReaderRowNumber(current_pos_); } void Close() override {} @@ -172,6 +179,13 @@ class MockFileBatchReader : public PrefetchFileBatchReader { } private: + static uint64_t ToReaderRowNumber(int32_t row_number) { + if (row_number < 0) { + return std::numeric_limits::max(); + } + return static_cast(row_number); + } + std::shared_ptr data_; std::shared_ptr file_schema_; std::shared_ptr read_schema_; @@ -183,6 +197,7 @@ class MockFileBatchReader : public PrefetchFileBatchReader { Status next_batch_status_; bool enable_randomize_batch_size_ = true; std::vector> read_ranges_; + std::mt19937 random_engine_{std::random_device{}()}; // NOLINT(whitespace/braces) }; } // namespace paimon::test diff --git a/src/paimon/testing/mock/mock_format_writer.cpp b/src/paimon/testing/mock/mock_format_writer.cpp index f763e437..c7829f46 100644 --- a/src/paimon/testing/mock/mock_format_writer.cpp +++ b/src/paimon/testing/mock/mock_format_writer.cpp @@ -20,7 +20,6 @@ #include #include -#include #include "arrow/c/helpers.h" #include "paimon/common/utils/date_time_utils.h" @@ -36,7 +35,7 @@ class MemoryPool; namespace paimon::test { MockFormatWriter::MockFormatWriter(const std::shared_ptr& out, const std::shared_ptr& pool) - : FormatWriter(), out_(std::move(out)), pool_(pool) {} + : FormatWriter(), out_(out), pool_(pool) {} Status MockFormatWriter::AddBatch(ArrowArray* batch) { ArrowArrayRelease(batch); diff --git a/src/paimon/testing/mock/mock_format_writer_builder.h b/src/paimon/testing/mock/mock_format_writer_builder.h index d61867f4..a61567d6 100644 --- a/src/paimon/testing/mock/mock_format_writer_builder.h +++ b/src/paimon/testing/mock/mock_format_writer_builder.h @@ -44,8 +44,6 @@ class MockFormatWriterBuilder : public WriterBuilder { const std::string& compression) override; private: - Status Prepare(); - std::shared_ptr memory_pool_; }; diff --git a/src/paimon/testing/mock/mock_key_value_data_file_record_reader.h b/src/paimon/testing/mock/mock_key_value_data_file_record_reader.h index ec8d615c..a2cbfda4 100644 --- a/src/paimon/testing/mock/mock_key_value_data_file_record_reader.h +++ b/src/paimon/testing/mock/mock_key_value_data_file_record_reader.h @@ -23,6 +23,7 @@ #include "paimon/common/data/columnar/columnar_batch_context.h" #include "paimon/core/io/key_value_data_file_record_reader.h" + namespace paimon::test { // mock reader hold data array class MockKeyValueDataFileRecordReader : public KeyValueDataFileRecordReader { diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 9bd0797c..d8ffd614 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -77,6 +77,7 @@ #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" + namespace paimon { class DataSplit; class RecordBatch; @@ -1191,10 +1192,9 @@ TEST_P(BlobTableInteTest, TestIOException) { .ValueOrDie()); auto commit_msgs1_result = WriteArray(table_path, {}, write_cols1, {src_array1}); CHECK_HOOK_STATUS(commit_msgs1_result.status(), i); - SetFirstRowId(/*reset_first_row_id=*/0, - const_cast>&>( - commit_msgs1_result.value())); - CHECK_HOOK_STATUS(Commit(table_path, commit_msgs1_result.value()), i); + auto commit_msgs1 = std::move(commit_msgs1_result).value(); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + CHECK_HOOK_STATUS(Commit(table_path, commit_msgs1), i); write_run_complete = true; break; } @@ -1514,7 +1514,7 @@ TEST_P(BlobTableInteTest, TestWithRowIdsForMultipleBlobFiles) { /*row_ranges=*/{Range(1l, 1l), Range(5l, 5l)})); } { - // test ont read blob field + // test not read blob field auto expected_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([ ["aaa1"], From 2076d4f6010cb0fe7ae7f24d2cc42743e5402648 Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Fri, 26 Jun 2026 00:36:23 -0700 Subject: [PATCH 074/138] feat(shared_shredding): add read inte test for shared_shredding --- src/paimon/CMakeLists.txt | 4 +- ...p => map_shared_shredding_file_reader.cpp} | 178 +-- ...r.h => map_shared_shredding_file_reader.h} | 29 +- ...map_shared_shredding_file_reader_test.cpp} | 131 +- src/paimon/core/io/field_mapping_reader.cpp | 20 +- src/paimon/core/io/field_mapping_reader.h | 6 +- .../core/io/field_mapping_reader_test.cpp | 25 +- .../core/operation/abstract_split_read.cpp | 87 +- .../core/operation/abstract_split_read.h | 8 +- .../operation/data_evolution_split_read.h | 2 +- .../core/operation/merge_file_split_read.h | 4 +- .../core/operation/raw_file_split_read.h | 2 +- .../core/utils/nested_projection_utils.cpp | 65 +- .../core/utils/nested_projection_utils.h | 18 +- .../utils/nested_projection_utils_test.cpp | 28 +- .../lucene/lucene_global_index_reader.cpp | 4 +- test/inte/blob_table_inte_test.cpp | 270 +++++ test/inte/nested_column_pruning_inte_test.cpp | 11 +- test/inte/write_and_read_inte_test.cpp | 1071 +++++++++++++++++ 19 files changed, 1670 insertions(+), 293 deletions(-) rename src/paimon/common/data/shredding/{shared_shredding_file_reader.cpp => map_shared_shredding_file_reader.cpp} (70%) rename src/paimon/common/data/shredding/{shared_shredding_file_reader.h => map_shared_shredding_file_reader.h} (77%) rename src/paimon/common/data/shredding/{shared_shredding_file_reader_test.cpp => map_shared_shredding_file_reader_test.cpp} (82%) diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 7e075415..d2e3787c 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -144,7 +144,7 @@ set(PAIMON_COMMON_SRCS common/data/shredding/map_shared_shredding_context.cpp common/data/shredding/map_shared_shredding_batch_converter.cpp common/data/shredding/map_shared_shredding_column_allocator.cpp - common/data/shredding/shared_shredding_file_reader.cpp + common/data/shredding/map_shared_shredding_file_reader.cpp common/utils/delta_varint_compressor.cpp common/utils/fields_comparator.cpp common/utils/path_util.cpp @@ -552,7 +552,7 @@ if(PAIMON_BUILD_TESTS) common/data/shredding/map_shared_shredding_column_allocator_test.cpp common/data/shredding/map_shared_shredding_field_dict_test.cpp common/data/shredding/map_shared_shredding_context_test.cpp - common/data/shredding/shared_shredding_file_reader_test.cpp + common/data/shredding/map_shared_shredding_file_reader_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static diff --git a/src/paimon/common/data/shredding/shared_shredding_file_reader.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp similarity index 70% rename from src/paimon/common/data/shredding/shared_shredding_file_reader.cpp rename to src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp index 5e80ad27..c1933cf1 100644 --- a/src/paimon/common/data/shredding/shared_shredding_file_reader.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/data/shredding/shared_shredding_file_reader.h" +#include "paimon/common/data/shredding/map_shared_shredding_file_reader.h" #include #include @@ -34,58 +34,16 @@ #include "paimon/core/casting/casting_utils.h" namespace paimon { -namespace { -// TODO(lisizhuo.lsz): rm kSelectedKeysMetadataKey after zhanyu pr -constexpr const char* kSelectedKeysMetadataKey = "paimon.map.selected-keys"; - -Result>> GetSelectedKeys( - const std::shared_ptr& field) { - if (!field->HasMetadata() || !field->metadata()) { - return std::optional>(); - } - int32_t index = field->metadata()->FindKey(kSelectedKeysMetadataKey); - if (index < 0) { - return std::optional>(); - } - std::string selected_keys = field->metadata()->value(index); - // paimon will not ignore empty for '' is a valid key - return std::optional>( - StringUtils::Split(selected_keys, ",", /*ignore_empty=*/false)); -} - -} // namespace - -Result> SharedShreddingFileReader::Create( - std::unique_ptr&& reader, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, reader->GetFileSchema()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_arrow_schema, - arrow::ImportSchema(file_schema.get())); - - std::map shared_shredding_name_to_meta; - for (const auto& field : file_arrow_schema->fields()) { - std::shared_ptr metadata = - std::const_pointer_cast(field->metadata()); - if (MapSharedShreddingUtils::HasShreddingMetadata(metadata)) { - PAIMON_ASSIGN_OR_RAISE( - MapSharedShreddingFieldMeta meta, - MapSharedShreddingUtils::DeserializeMetadata( - metadata, MapSharedShreddingDefine::kDefaultDictCompression)); - shared_shredding_name_to_meta[field->name()] = std::move(meta); - } - } - return std::unique_ptr( - new SharedShreddingFileReader(std::move(reader), shared_shredding_name_to_meta, pool)); -} - -SharedShreddingFileReader::SharedShreddingFileReader( +MapSharedShreddingFileReader::MapSharedShreddingFileReader( std::unique_ptr&& reader, - const std::map& shared_shredding_name_to_meta, + std::map&& + shared_shredding_name_to_context, const std::shared_ptr& pool) : arrow_pool_(GetArrowPool(pool)), reader_(std::move(reader)), - shared_shredding_name_to_meta_(shared_shredding_name_to_meta) {} + shared_shredding_name_to_context_(std::move(shared_shredding_name_to_context)) {} -Result> SharedShreddingFileReader::GetFileSchema() const { +Result> MapSharedShreddingFileReader::GetFileSchema() const { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> physical_schema, reader_->GetFileSchema()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr physical_arrow_schema, @@ -108,14 +66,13 @@ Result> SharedShreddingFileReader::GetFileSchema( return c_logical_schema; } -Result> SharedShreddingFileReader::ToLogicalMapField( +Result> MapSharedShreddingFileReader::ToLogicalMapField( const std::shared_ptr& physical_field) { auto physical_type = std::dynamic_pointer_cast(physical_field->type()); if (!physical_type) { return Status::Invalid(fmt::format("shared-shredding field {} is not a physical struct", physical_field->name())); } - std::shared_ptr value_type; bool value_nullable = true; for (const auto& child : physical_type->fields()) { @@ -137,101 +94,79 @@ Result> SharedShreddingFileReader::ToLogicalMapFie physical_field->nullable()); } -Status SharedShreddingFileReader::SetReadSchema( +Status MapSharedShreddingFileReader::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { if (!read_schema) { - return Status::Invalid("invalid read schema in SharedShreddingFileReader, cannot be null"); + return Status::Invalid( + "invalid read schema in MapSharedShreddingFileReader, cannot be null"); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_read_schema, arrow::ImportSchema(read_schema)); std::vector shared_shredding_names; for (const auto& field : logical_read_schema->fields()) { - if (shared_shredding_name_to_meta_.find(field->name()) != - shared_shredding_name_to_meta_.end()) { + if (shared_shredding_name_to_context_.find(field->name()) != + shared_shredding_name_to_context_.end()) { shared_shredding_names.push_back(field->name()); } } if (shared_shredding_names.empty()) { - // suppose not fall into SharedShreddingFileReader + // suppose not fall into MapSharedShreddingFileReader return Status::Invalid("do not exist shared shredding columns in read schema"); } - shared_shredding_name_to_selected_keys_.clear(); - shared_shredding_name_to_map_type_.clear(); arrow::FieldVector resolved_fields = logical_read_schema->fields(); for (const auto& name : shared_shredding_names) { const auto& field = logical_read_schema->GetFieldByName(name); if (!field) { return Status::Invalid( - fmt::format("cannot find shared-shredding field in read schema")); + fmt::format("cannot find shared-shredding field {} in read schema", name)); } - auto meta_iter = shared_shredding_name_to_meta_.find(field->name()); - if (meta_iter == shared_shredding_name_to_meta_.end()) { + auto context_iter = shared_shredding_name_to_context_.find(field->name()); + if (context_iter == shared_shredding_name_to_context_.end()) { return Status::Invalid( fmt::format("cannot find shared-shredding metadata for field {}", field->name())); } - PAIMON_ASSIGN_OR_RAISE(std::optional> selected_keys_opt, - GetSelectedKeys(field)); - std::vector selected_keys; - if (!selected_keys_opt) { - // select all keys - selected_keys.reserve(meta_iter->second.name_to_id.size()); - for (const auto& [key_name, _] : meta_iter->second.name_to_id) { - selected_keys.push_back(key_name); - } - } else { - std::set seen_keys; - for (const auto& selected_key : selected_keys_opt.value()) { - if (!seen_keys.insert(selected_key).second) { - return Status::Invalid( - fmt::format("duplicate key [{}] in paimon.map.selected-keys for field {}", - selected_key, field->name())); - } - selected_keys.push_back(selected_key); - } - } - shared_shredding_name_to_selected_keys_[field->name()] = selected_keys; - - auto map_type = arrow::internal::checked_pointer_cast(field->type()); - shared_shredding_name_to_map_type_[field->name()] = map_type; std::set selected_physical_column_ids; bool include_overflow = false; - for (const auto& selected_key : selected_keys) { - auto name_iter = meta_iter->second.name_to_id.find(selected_key); - if (name_iter == meta_iter->second.name_to_id.end()) { + for (const auto& selected_key : context_iter->second.selected_keys) { + // check if selected_key in file + auto name_iter = context_iter->second.meta.name_to_id.find(selected_key); + if (name_iter == context_iter->second.meta.name_to_id.end()) { continue; } - auto column_iter = meta_iter->second.field_to_columns.find(name_iter->second); - if (column_iter == meta_iter->second.field_to_columns.end()) { + // check if selected_key in overflow_field + PAIMON_ASSIGN_OR_RAISE( + bool is_overflow_field, + MapSharedShreddingUtils::IsOverflowField(context_iter->second.meta, selected_key)); + include_overflow = include_overflow || is_overflow_field; + // check if selected_key in field_to_columns + auto column_iter = context_iter->second.meta.field_to_columns.find(name_iter->second); + if (column_iter == context_iter->second.meta.field_to_columns.end()) { continue; } const std::vector& physical_column_ids = column_iter->second; selected_physical_column_ids.insert(physical_column_ids.begin(), physical_column_ids.end()); - PAIMON_ASSIGN_OR_RAISE(bool is_overflow_field, MapSharedShreddingUtils::IsOverflowField( - meta_iter->second, selected_key)); - include_overflow = include_overflow || is_overflow_field; } std::shared_ptr resolved_type = MapSharedShreddingUtils::BuildSpecificPhysicalStructType( - map_type->item_type(), selected_physical_column_ids, - map_type->item_field()->nullable(), include_overflow); + context_iter->second.map_type->item_type(), selected_physical_column_ids, + context_iter->second.map_type->item_field()->nullable(), include_overflow); resolved_fields[logical_read_schema->GetFieldIndex(name)] = arrow::field(field->name(), resolved_type, field->nullable()); } - auto resolved_schema = arrow::schema(std::move(resolved_fields)); std::unique_ptr c_resolved_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*resolved_schema, c_resolved_schema.get())); return reader_->SetReadSchema(c_resolved_schema.get(), predicate, selection_bitmap); } -Result SharedShreddingFileReader::NextBatch() { +Result MapSharedShreddingFileReader::NextBatch() { return Status::Invalid( - "paimon inner reader SharedShreddingFileReader should use NextBatchWithBitmap"); + "paimon inner reader MapSharedShreddingFileReader should use NextBatchWithBitmap"); } -Result SharedShreddingFileReader::NextBatchWithBitmap() { +Result MapSharedShreddingFileReader::NextBatchWithBitmap() { PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, reader_->NextBatchWithBitmap()); if (BatchReader::IsEofBatch(batch_with_bitmap)) { @@ -244,15 +179,15 @@ Result SharedShreddingFileReader::NextBatchWit arrow::ImportArray(c_array.get(), c_schema.get())); auto struct_array = std::dynamic_pointer_cast(arrow_array); if (!struct_array) { - return Status::Invalid("cannot cast batch to StructArray in SharedShreddingFileReader"); + return Status::Invalid("cannot cast batch to StructArray in MapSharedShreddingFileReader"); } arrow::ArrayVector resolved_arrays = struct_array->fields(); arrow::FieldVector resolved_fields = struct_array->struct_type()->fields(); for (int32_t field_idx = 0; field_idx < struct_array->num_fields(); ++field_idx) { const auto& physical_field = struct_array->struct_type()->field(field_idx); - auto iter = shared_shredding_name_to_selected_keys_.find(physical_field->name()); - if (iter == shared_shredding_name_to_selected_keys_.end()) { + auto iter = shared_shredding_name_to_context_.find(physical_field->name()); + if (iter == shared_shredding_name_to_context_.end()) { continue; } auto physical_struct_array = @@ -261,14 +196,12 @@ Result SharedShreddingFileReader::NextBatchWit return Status::Invalid(fmt::format( "cannot cast physical shredding field {} to StructArray", physical_field->name())); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr logical_map_array, RebuildLogicalMapArray(physical_field, physical_struct_array)); resolved_arrays[field_idx] = logical_map_array; resolved_fields[field_idx] = arrow::field(physical_field->name(), logical_map_array->type(), physical_field->nullable()); } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr new_struct_array, arrow::StructArray::Make(resolved_arrays, resolved_fields)); auto new_c_array = std::make_unique(); @@ -279,28 +212,18 @@ Result SharedShreddingFileReader::NextBatchWit return batch_with_bitmap; } -Result> SharedShreddingFileReader::RebuildLogicalMapArray( +Result> MapSharedShreddingFileReader::RebuildLogicalMapArray( const std::shared_ptr& physical_field, const std::shared_ptr& physical_struct_array) const { std::string shredding_field_name = physical_field->name(); - auto meta_iter = shared_shredding_name_to_meta_.find(shredding_field_name); - if (meta_iter == shared_shredding_name_to_meta_.end()) { - return Status::Invalid(fmt::format("cannot find shared-shredding metadata for field {}", - shredding_field_name)); - } - auto selected_iter = shared_shredding_name_to_selected_keys_.find(shredding_field_name); - if (selected_iter == shared_shredding_name_to_selected_keys_.end()) { - return Status::Invalid( - fmt::format("cannot find selected keys for field {}", shredding_field_name)); - } - auto map_type_iter = shared_shredding_name_to_map_type_.find(shredding_field_name); - if (map_type_iter == shared_shredding_name_to_map_type_.end()) { + auto iter = shared_shredding_name_to_context_.find(shredding_field_name); + if (iter == shared_shredding_name_to_context_.end()) { return Status::Invalid( - fmt::format("cannot find logical map type for field {}", shredding_field_name)); + fmt::format("cannot find shared-shredding context for field {}", shredding_field_name)); } - const MapSharedShreddingFieldMeta& meta = meta_iter->second; - const std::vector& selected_keys = selected_iter->second; - const auto& map_type = map_type_iter->second; + const MapSharedShreddingFieldMeta& meta = iter->second.meta; + const std::vector& selected_keys = iter->second.selected_keys; + const auto& map_type = iter->second.map_type; auto field_mapping_array = std::dynamic_pointer_cast( physical_struct_array->GetFieldByName(MapSharedShreddingDefine::kFieldMapping)); @@ -315,7 +238,6 @@ Result> SharedShreddingFileReader::RebuildLogicalM } auto selected_key_ids = ResolveSelectedKeyIds(meta, selected_keys); - std::map> physical_column_name_to_array; std::shared_ptr overflow_array; CollectPhysicalColumns(physical_struct_array, &physical_column_name_to_array, &overflow_array); @@ -431,7 +353,7 @@ Result> SharedShreddingFileReader::RebuildLogicalM return map_array; } -std::vector> SharedShreddingFileReader::ResolveSelectedKeyIds( +std::vector> MapSharedShreddingFileReader::ResolveSelectedKeyIds( const MapSharedShreddingFieldMeta& meta, const std::vector& selected_keys) { std::vector> selected_key_ids; selected_key_ids.reserve(selected_keys.size()); @@ -445,7 +367,7 @@ std::vector> SharedShreddingFileReader::ResolveS return selected_key_ids; } -void SharedShreddingFileReader::CollectPhysicalColumns( +void MapSharedShreddingFileReader::CollectPhysicalColumns( const std::shared_ptr& physical_struct_array, std::map>* physical_column_name_to_array, std::shared_ptr* overflow_array) { @@ -464,23 +386,23 @@ void SharedShreddingFileReader::CollectPhysicalColumns( } } -std::shared_ptr SharedShreddingFileReader::GetReaderMetrics() const { +std::shared_ptr MapSharedShreddingFileReader::GetReaderMetrics() const { return reader_->GetReaderMetrics(); } -void SharedShreddingFileReader::Close() { +void MapSharedShreddingFileReader::Close() { reader_->Close(); } -Result SharedShreddingFileReader::GetPreviousBatchFirstRowNumber() const { +Result MapSharedShreddingFileReader::GetPreviousBatchFirstRowNumber() const { return reader_->GetPreviousBatchFirstRowNumber(); } -Result SharedShreddingFileReader::GetNumberOfRows() const { +Result MapSharedShreddingFileReader::GetNumberOfRows() const { return reader_->GetNumberOfRows(); } -bool SharedShreddingFileReader::SupportPreciseBitmapSelection() const { +bool MapSharedShreddingFileReader::SupportPreciseBitmapSelection() const { return reader_->SupportPreciseBitmapSelection(); } diff --git a/src/paimon/common/data/shredding/shared_shredding_file_reader.h b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h similarity index 77% rename from src/paimon/common/data/shredding/shared_shredding_file_reader.h rename to src/paimon/common/data/shredding/map_shared_shredding_file_reader.h index 7826e269..3ab92753 100644 --- a/src/paimon/common/data/shredding/shared_shredding_file_reader.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h @@ -21,7 +21,9 @@ #include #include +#include #include +#include #include #include "arrow/api.h" @@ -31,10 +33,22 @@ namespace paimon { -class SharedShreddingFileReader : public FileBatchReader { +class MapSharedShreddingFileReader : public FileBatchReader { public: - static Result> Create( - std::unique_ptr&& reader, const std::shared_ptr& pool); + struct SharedShreddingContext { + SharedShreddingContext(const MapSharedShreddingFieldMeta& _meta, + const std::vector& _selected_keys, + const std::shared_ptr& _map_type) + : meta(_meta), selected_keys(_selected_keys), map_type(_map_type) {} + MapSharedShreddingFieldMeta meta; + std::vector selected_keys; + std::shared_ptr map_type; + }; + + MapSharedShreddingFileReader( + std::unique_ptr&& reader, + std::map&& shared_shredding_name_to_context, + const std::shared_ptr& pool); Result> GetFileSchema() const override; @@ -56,11 +70,6 @@ class SharedShreddingFileReader : public FileBatchReader { bool SupportPreciseBitmapSelection() const override; private: - SharedShreddingFileReader( - std::unique_ptr&& reader, - const std::map& shared_shredding_name_to_meta, - const std::shared_ptr& pool); - Result> RebuildLogicalMapArray( const std::shared_ptr& physical_field, const std::shared_ptr& physical_struct_array) const; @@ -79,9 +88,7 @@ class SharedShreddingFileReader : public FileBatchReader { private: std::shared_ptr arrow_pool_; std::unique_ptr reader_; - std::map shared_shredding_name_to_meta_; - std::map> shared_shredding_name_to_selected_keys_; - std::map> shared_shredding_name_to_map_type_; + std::map shared_shredding_name_to_context_; }; } // namespace paimon diff --git a/src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp similarity index 82% rename from src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp rename to src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp index 5f8e274b..7169ffa8 100644 --- a/src/paimon/common/data/shredding/shared_shredding_file_reader_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/data/shredding/shared_shredding_file_reader.h" +#include "paimon/common/data/shredding/map_shared_shredding_file_reader.h" #include #include @@ -50,7 +50,7 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -class SharedShreddingFileReaderTest : public ::testing::Test { +class MapSharedShreddingFileReaderTest : public ::testing::Test { public: void SetUp() override { pool_ = GetDefaultPool(); @@ -97,9 +97,56 @@ class SharedShreddingFileReaderTest : public ::testing::Test { .ValueOrDie(); } - std::unique_ptr CreateReader( + std::unique_ptr WrapReader( + std::unique_ptr&& reader, + const std::optional& selected_keys_str = std::nullopt) const { + EXPECT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + std::map + shared_shredding_name_to_context; + for (const auto& field : file_schema->fields()) { + auto metadata = std::const_pointer_cast(field->metadata()); + if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata)) { + continue; + } + EXPECT_OK_AND_ASSIGN(auto meta, + MapSharedShreddingUtils::DeserializeMetadata( + metadata, MapSharedShreddingDefine::kDefaultDictCompression)); + auto physical_type = + arrow::internal::checked_pointer_cast(field->type()); + std::shared_ptr item_field; + for (const auto& child : physical_type->fields()) { + if (child->name() != MapSharedShreddingDefine::kFieldMapping && + child->name() != MapSharedShreddingDefine::kOverflow) { + item_field = child; + break; + } + } + EXPECT_TRUE(item_field); + auto map_type = arrow::internal::checked_pointer_cast(arrow::map( + arrow::utf8(), arrow::field("value", item_field->type(), item_field->nullable()))); + std::vector selected_keys; + if (selected_keys_str.has_value()) { + selected_keys = StringUtils::Split(selected_keys_str.value(), ",", + /*ignore_empty=*/false); + } else { + selected_keys.reserve(meta.name_to_id.size()); + for (const auto& [key_name, _] : meta.name_to_id) { + selected_keys.push_back(key_name); + } + } + shared_shredding_name_to_context.emplace( + field->name(), MapSharedShreddingFileReader::SharedShreddingContext( + meta, selected_keys, map_type)); + } + return std::make_unique( + std::move(reader), std::move(shared_shredding_name_to_context), pool_); + } + + std::unique_ptr CreateReader( std::shared_ptr physical_array = nullptr, - std::shared_ptr physical_schema = nullptr) const { + std::shared_ptr physical_schema = nullptr, + const std::optional& selected_keys = std::nullopt) const { if (!physical_schema) { physical_schema = PhysicalSchemaWithMetadata(); } @@ -109,9 +156,7 @@ class SharedShreddingFileReaderTest : public ::testing::Test { auto mock_reader = std::make_unique( physical_array, arrow::struct_(physical_schema->fields()), /*read_batch_size=*/10); mock_reader->EnableRandomizeBatchSize(false); - EXPECT_OK_AND_ASSIGN(auto shared_shredding_reader, - SharedShreddingFileReader::Create(std::move(mock_reader), pool_)); - return shared_shredding_reader; + return WrapReader(std::move(mock_reader), selected_keys); } std::shared_ptr ReadSchema( @@ -199,7 +244,7 @@ class SharedShreddingFileReaderTest : public ::testing::Test { }; }; -TEST_F(SharedShreddingFileReaderTest, TestGetFileSchemaReturnsLogicalMapSchema) { +TEST_F(MapSharedShreddingFileReaderTest, TestGetFileSchemaReturnsLogicalMapSchema) { auto reader = CreateReader(); ASSERT_OK_AND_ASSIGN(auto c_schema, reader->GetFileSchema()); @@ -212,8 +257,9 @@ TEST_F(SharedShreddingFileReaderTest, TestGetFileSchemaReturnsLogicalMapSchema) ASSERT_FALSE(schema->field(1)->HasMetadata()); } -TEST_F(SharedShreddingFileReaderTest, TestAllExistSelectedKeysWithoutOverflow) { - auto reader = CreateReader(); +TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithoutOverflow) { + auto reader = CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"b"); auto read_schema = ExportSchema(ReadSchema("b")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -232,8 +278,9 @@ TEST_F(SharedShreddingFileReaderTest, TestAllExistSelectedKeysWithoutOverflow) { AssertChunkedArrayEquals(expected, actual); } -TEST_F(SharedShreddingFileReaderTest, TestAllExistSelectedKeysWithOverflow) { - auto reader = CreateReader(); +TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithOverflow) { + auto reader = CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"a,c"); auto read_schema = ExportSchema(ReadSchema("a,c")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -252,8 +299,9 @@ TEST_F(SharedShreddingFileReaderTest, TestAllExistSelectedKeysWithOverflow) { AssertChunkedArrayEquals(expected, actual); } -TEST_F(SharedShreddingFileReaderTest, TestPartialExistSelectedKeys) { - auto reader = CreateReader(); +TEST_F(MapSharedShreddingFileReaderTest, TestPartialExistSelectedKeys) { + auto reader = CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"a,c,missing"); auto read_schema = ExportSchema(ReadSchema("a,c,missing")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -273,15 +321,7 @@ TEST_F(SharedShreddingFileReaderTest, TestPartialExistSelectedKeys) { AssertChunkedArrayEquals(expected, actual); } -TEST_F(SharedShreddingFileReaderTest, TestDuplicatedSelectedKeys) { - auto reader = CreateReader(); - auto read_schema = ExportSchema(ReadSchema("a,c,a")); - ASSERT_NOK_WITH_MSG(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, - /*selection_bitmap=*/std::nullopt), - "duplicate key [a] in paimon.map.selected-keys for field tags"); -} - -TEST_F(SharedShreddingFileReaderTest, TestMissingSelectedKeysReadsWholeMap) { +TEST_F(MapSharedShreddingFileReaderTest, TestMissingSelectedKeysReadsWholeMap) { auto reader = CreateReader(); auto read_schema = ExportSchema(ReadSchema(std::nullopt)); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, @@ -301,7 +341,7 @@ TEST_F(SharedShreddingFileReaderTest, TestMissingSelectedKeysReadsWholeMap) { AssertChunkedArrayEquals(expected, actual); } -TEST_F(SharedShreddingFileReaderTest, TestSpecialSelectedKeys) { +TEST_F(MapSharedShreddingFileReaderTest, TestSpecialSelectedKeys) { MapSharedShreddingFieldMeta meta; meta.name_to_id = {{"", 0}, {" ", 1}, {".", 2}, {"a", 3}}; meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}, {3, {1}}}; @@ -318,7 +358,7 @@ TEST_F(SharedShreddingFileReaderTest, TestSpecialSelectedKeys) { .ValueOrDie(); auto assert_read = [&](const std::string& selected_keys, const std::string& expected_json) { - auto reader = CreateReader(physical_array, physical_schema); + auto reader = CreateReader(physical_array, physical_schema, selected_keys); auto read_schema = ExportSchema(ReadSchema(selected_keys)); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -353,18 +393,9 @@ TEST_F(SharedShreddingFileReaderTest, TestSpecialSelectedKeys) { ])"); } -TEST_F(SharedShreddingFileReaderTest, TestSpecialSelectedKeysWithDuplicatedEmptyKey) { - for (const auto& selected_keys : {",", ",,"}) { - auto reader = CreateReader(); - auto read_schema = ExportSchema(ReadSchema(selected_keys)); - ASSERT_NOK_WITH_MSG(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, - /*selection_bitmap=*/std::nullopt), - "duplicate key [] in paimon.map.selected-keys for field tags"); - } -} - -TEST_F(SharedShreddingFileReaderTest, TestUnknownSelectedKeyReturnsEmptyMap) { - auto reader = CreateReader(); +TEST_F(MapSharedShreddingFileReaderTest, TestUnknownSelectedKeyReturnsEmptyMap) { + auto reader = CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"missing"); auto read_schema = ExportSchema(ReadSchema("missing")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -384,7 +415,7 @@ TEST_F(SharedShreddingFileReaderTest, TestUnknownSelectedKeyReturnsEmptyMap) { AssertChunkedArrayEquals(expected, actual); } -TEST_F(SharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) { +TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) { auto physical_schema = PhysicalSchemaWithMetadata(); std::string json = R"([ [1, [null, 10, null, null]] @@ -392,7 +423,7 @@ TEST_F(SharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) { auto physical_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) .ValueOrDie(); - auto reader = CreateReader(physical_array, physical_schema); + auto reader = CreateReader(physical_array, physical_schema, /*selected_keys=*/"a"); auto read_schema = ExportSchema(ReadSchema("a")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -400,7 +431,7 @@ TEST_F(SharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) { "__field_mapping cannot be null"); } -TEST_F(SharedShreddingFileReaderTest, TestInvalidNullFieldMappingFieldElement) { +TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingFieldElement) { auto physical_schema = PhysicalSchemaWithMetadata(); std::string json = R"([ [1, [[0, null], 10, null, null]] @@ -408,7 +439,7 @@ TEST_F(SharedShreddingFileReaderTest, TestInvalidNullFieldMappingFieldElement) { auto physical_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) .ValueOrDie(); - auto reader = CreateReader(physical_array, physical_schema); + auto reader = CreateReader(physical_array, physical_schema, /*selected_keys=*/"b"); auto read_schema = ExportSchema(ReadSchema("b")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -416,7 +447,7 @@ TEST_F(SharedShreddingFileReaderTest, TestInvalidNullFieldMappingFieldElement) { "__field_mapping element cannot be null"); } -TEST_F(SharedShreddingFileReaderTest, TestListValue) { +TEST_F(MapSharedShreddingFileReaderTest, TestListValue) { std::shared_ptr logical_schema = arrow::schema({ arrow::field("id", arrow::int32()), arrow::field("tags", arrow::map(arrow::utf8(), arrow::list(arrow::int32()))), @@ -446,7 +477,8 @@ TEST_F(SharedShreddingFileReaderTest, TestListValue) { [4, [[1, 0], [8], [9, 10], [[2, [null]]]]] ])") .ValueOrDie(); - auto reader = CreateReader(physical_array, physical_schema); + auto reader = CreateReader(physical_array, physical_schema, + /*selected_keys=*/"a,c"); // NOLINT(whitespace/comma) auto read_metadata = std::make_shared(); read_metadata->Append("paimon.map.selected-keys", "a,c"); @@ -470,7 +502,7 @@ TEST_F(SharedShreddingFileReaderTest, TestListValue) { AssertChunkedArrayEquals(expected, actual); } -TEST_F(SharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) { +TEST_F(MapSharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) { std::shared_ptr logical_schema = arrow::schema({ arrow::field("id", arrow::int32()), arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), @@ -506,9 +538,8 @@ TEST_F(SharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) { std::string data_file_path = path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map reader_options = {{"orc.read.enable-lazy-decoding", "true"}}; - ASSERT_OK_AND_ASSIGN(auto reader, - SharedShreddingFileReader::Create( - OpenFormatReader(data_file_path, format, reader_options), pool_)); + auto reader = WrapReader(OpenFormatReader(data_file_path, format, reader_options), + /*selected_keys_str=*/"a,c"); auto read_metadata = std::make_shared(); read_metadata->Append("paimon.map.selected-keys", "a,c"); @@ -531,7 +562,7 @@ TEST_F(SharedShreddingFileReaderTest, TestOrcDictionaryEncodedStringValue) { AssertChunkedArrayEquals(expected, actual); } -TEST_F(SharedShreddingFileReaderTest, TestReadsRealFormatFile) { +TEST_F(MapSharedShreddingFileReaderTest, TestReadsRealFormatFile) { // TODO(lisizhuo.lsz): support other format auto options = options_; std::string format = "orc"; @@ -562,8 +593,8 @@ TEST_F(SharedShreddingFileReaderTest, TestReadsRealFormatFile) { std::string data_file_path = path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); - ASSERT_OK_AND_ASSIGN(auto reader, SharedShreddingFileReader::Create( - OpenFormatReader(data_file_path, format), pool_)); + auto reader = WrapReader(OpenFormatReader(data_file_path, format), + /*selected_keys_str=*/"a,c"); auto read_schema = ExportSchema(ReadSchema("a,c")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp index 4fe5427d..738a99a0 100644 --- a/src/paimon/core/io/field_mapping_reader.cpp +++ b/src/paimon/core/io/field_mapping_reader.cpp @@ -125,12 +125,16 @@ Result> FieldMappingReader::FilterMapSelectedKeysR Result> FieldMappingReader::Create( int32_t field_count, std::unique_ptr&& reader, const BinaryRow& partition, - std::unique_ptr&& mapping, const std::shared_ptr& pool) { + std::unique_ptr&& mapping, + std::set&& skip_map_selected_keys_filter_field_ids, + const std::shared_ptr& pool) { auto mapping_reader = std::unique_ptr(new FieldMappingReader( field_count, std::move(reader), partition, std::move(mapping), pool)); mapping_reader->need_mapping_ = false; mapping_reader->need_casting_ = false; + mapping_reader->skip_map_selected_keys_filter_field_ids_ = + std::move(skip_map_selected_keys_filter_field_ids); if (mapping_reader->non_exist_field_info_ != std::nullopt || mapping_reader->partition_info_ != std::nullopt) { @@ -163,7 +167,9 @@ Result> FieldMappingReader::Create( bool has_map_selected_keys, mapping_reader->HasMapSelectedKeysRecursively( mapping_reader->non_partition_info_.non_partition_read_schema[i].ArrowField())); - if (has_map_selected_keys) { + if (has_map_selected_keys && + mapping_reader->skip_map_selected_keys_filter_field_ids_.count( + mapping_reader->non_partition_info_.non_partition_read_schema[i].Id()) == 0) { mapping_reader->need_mapping_ = true; } } @@ -401,14 +407,16 @@ Status FieldMappingReader::MappingFields(const std::shared_ptr& da assert(struct_array->fields().size() == idx_in_target_schema.size()); for (size_t i = 0; i < idx_in_target_schema.size(); i++) { std::shared_ptr field_array = struct_array->field(i); + const DataField& read_field = read_fields_of_data_array[i]; // Filter map entries by selected keys recursively (supports MAP nested in STRUCT). - PAIMON_ASSIGN_OR_RAISE(field_array, - FilterMapSelectedKeysRecursively( - field_array, read_fields_of_data_array[i].ArrowField())); + if (skip_map_selected_keys_filter_field_ids_.count(read_field.Id()) == 0) { + PAIMON_ASSIGN_OR_RAISE(field_array, FilterMapSelectedKeysRecursively( + field_array, read_field.ArrowField())); + } (*target_array)[idx_in_target_schema[i]] = std::move(field_array); - (*target_field_names)[idx_in_target_schema[i]] = read_fields_of_data_array[i].Name(); + (*target_field_names)[idx_in_target_schema[i]] = read_field.Name(); } return Status::OK(); } diff --git a/src/paimon/core/io/field_mapping_reader.h b/src/paimon/core/io/field_mapping_reader.h index bec1af24..5c09974f 100644 --- a/src/paimon/core/io/field_mapping_reader.h +++ b/src/paimon/core/io/field_mapping_reader.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -50,7 +51,9 @@ class FieldMappingReader : public FileBatchReader { public: static Result> Create( int32_t field_count, std::unique_ptr&& reader, const BinaryRow& partition, - std::unique_ptr&& mapping, const std::shared_ptr& pool); + std::unique_ptr&& mapping, + std::set&& skip_map_selected_keys_filter_field_ids, + const std::shared_ptr& pool); Result NextBatch() override { return Status::Invalid( @@ -126,6 +129,7 @@ class FieldMappingReader : public FileBatchReader { std::optional partition_info_; NonPartitionInfo non_partition_info_; std::optional non_exist_field_info_; + std::set skip_map_selected_keys_filter_field_ids_; std::shared_ptr partition_array_; std::shared_ptr non_exist_array_; diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp index 7fbe7c94..15fc669d 100644 --- a/src/paimon/core/io/field_mapping_reader_test.cpp +++ b/src/paimon/core/io/field_mapping_reader_test.cpp @@ -136,9 +136,10 @@ class FieldMappingReaderTest : public ::testing::Test { /*batch_size=*/1); ASSERT_OK_AND_ASSIGN( - auto reader, FieldMappingReader::Create( - /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), - partition_, std::move(mapping), pool_)); + auto reader, + FieldMappingReader::Create( + /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), partition_, + std::move(mapping), /*skip_map_selected_keys_filter_field_ids=*/{}, pool_)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); if (expect_array == nullptr && result_array == nullptr) { // expect empty result @@ -176,9 +177,10 @@ class FieldMappingReaderTest : public ::testing::Test { /*predicate=*/mapping->non_partition_info.non_partition_filter, /*batch_size=*/1); ASSERT_OK_AND_ASSIGN( - auto reader, FieldMappingReader::Create( - /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), - partition, std::move(mapping), pool_)); + auto reader, + FieldMappingReader::Create( + /*field_count=*/read_schema->num_fields(), std::move(orc_batch_reader), partition, + std::move(mapping), /*skip_map_selected_keys_filter_field_ids=*/{}, pool_)); ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get())); if (expect_array == nullptr && result_array == nullptr) { // expect empty result @@ -236,9 +238,11 @@ TEST_F(FieldMappingReaderTest, TestGenerateSinglePartitionArray) { {false, static_cast(1), static_cast(2), static_cast(3), static_cast(4), std::string("5"), std::make_shared("6", pool_.get()), 100}, pool_.get()); - ASSERT_OK_AND_ASSIGN(auto mapping_reader, FieldMappingReader::Create( - /*field_count=*/8, /*reader=*/nullptr, partition, - std::move(field_mapping), pool_)); + ASSERT_OK_AND_ASSIGN( + auto mapping_reader, + FieldMappingReader::Create( + /*field_count=*/8, /*reader=*/nullptr, partition, std::move(field_mapping), + /*skip_map_selected_keys_filter_field_ids=*/{}, pool_)); { ASSERT_OK_AND_ASSIGN(auto p7_array, mapping_reader->GenerateSinglePartitionArray( @@ -785,7 +789,8 @@ TEST_F(FieldMappingReaderTest, TestCreateFailFastOnInvalidMapSelectedKeysMetadat ASSERT_NOK_WITH_MSG(FieldMappingReader::Create( /*field_count=*/1, /*reader=*/nullptr, - /*partition=*/BinaryRow::EmptyRow(), std::move(field_mapping), pool_), + /*partition=*/BinaryRow::EmptyRow(), std::move(field_mapping), + /*skip_map_selected_keys_filter_field_ids=*/{}, pool_), "Duplicate selected key 'a'"); } } // namespace paimon::test diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index 53e6c5d7..a55e1440 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -20,10 +20,15 @@ #include #include +#include +#include #include #include "arrow/type.h" +#include "fmt/format.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/shredding/map_shared_shredding_file_reader.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/reader/delegating_prefetch_reader.h" #include "paimon/common/reader/predicate_batch_reader.h" #include "paimon/common/reader/prefetch_file_batch_reader_impl.h" @@ -39,6 +44,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/nested_projection_utils.h" #include "paimon/format/file_format.h" #include "paimon/format/file_format_factory.h" #include "paimon/fs/file_system.h" @@ -124,9 +130,8 @@ Result> AbstractSplitRead::PrepareReaderBuilder( } Result> AbstractSplitRead::CreateFileBatchReader( - const std::shared_ptr& file_meta, const std::string& data_file_path, + const std::string& file_format_identifier, const std::string& data_file_path, const ReaderBuilder* reader_builder) const { - PAIMON_ASSIGN_OR_RAISE(std::string file_format_identifier, file_meta->FileFormat()); if (context_->EnablePrefetch() && file_format_identifier != "blob" && file_format_identifier != "avro") { PAIMON_ASSIGN_OR_RAISE( @@ -185,13 +190,22 @@ Result> AbstractSplitRead::CreateFieldMappingRe auto read_schema = DataField::ConvertDataFieldsToArrowSchema( field_mapping->non_partition_info.non_partition_data_schema); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_reader, - CreateFileBatchReader(file_meta, data_file_path, reader_builder)); + PAIMON_ASSIGN_OR_RAISE(std::string file_format_identifier, file_meta->FileFormat()); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr file_reader, + CreateFileBatchReader(file_format_identifier, data_file_path, reader_builder)); + std::set skip_map_selected_keys_filter_field_ids; + if (file_format_identifier != "blob") { + std::pair, std::set> shared_shredding_result; + PAIMON_ASSIGN_OR_RAISE(shared_shredding_result, ApplySharedShreddingReaderIfNeeded( + std::move(file_reader), read_schema)); + file_reader = std::move(shared_shredding_result.first); + skip_map_selected_keys_filter_field_ids = std::move(shared_shredding_result.second); + } if (NeedCompleteRowTrackingFields(options_.RowTrackingEnabled(), read_schema)) { file_reader = std::make_unique( std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number, pool_); } - const auto& predicate = field_mapping->non_partition_info.non_partition_filter; auto all_data_schema = DataField::ConvertDataFieldsToArrowSchema(data_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr final_reader, @@ -203,13 +217,68 @@ Result> AbstractSplitRead::CreateFieldMappingRe return std::unique_ptr(); } - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr mapping_reader, - FieldMappingReader::Create(field_mapping_builder->GetReadFieldCount(), - std::move(final_reader), partition, - std::move(field_mapping), pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr mapping_reader, + FieldMappingReader::Create(field_mapping_builder->GetReadFieldCount(), + std::move(final_reader), partition, std::move(field_mapping), + std::move(skip_map_selected_keys_filter_field_ids), pool_)); return mapping_reader; } +Result, std::set>> +AbstractSplitRead::ApplySharedShreddingReaderIfNeeded( + std::unique_ptr&& file_reader, + const std::shared_ptr& read_schema) const { + std::set handled_shared_shredding_field_ids; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, + file_reader->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_arrow_schema, + arrow::ImportSchema(file_schema.get())); + std::map + shared_shredding_name_to_context; + for (const auto& read_field : read_schema->fields()) { + const auto& field_name = read_field->name(); + auto file_field = file_arrow_schema->GetFieldByName(field_name); + if (!file_field) { + // may exists field _ROW_ID in read schema + continue; + } + std::shared_ptr metadata = + std::const_pointer_cast(file_field->metadata()); + if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata)) { + // not a map shared shredding field + continue; + } + // get meta + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingFieldMeta meta, + MapSharedShreddingUtils::DeserializeMetadata( + metadata, MapSharedShreddingDefine::kDefaultDictCompression)); + // get selected_keys + PAIMON_ASSIGN_OR_RAISE(std::vector selected_keys, + NestedProjectionUtils::GetMapSelectedKeys(read_field)); + if (selected_keys.empty()) { + // select all keys + selected_keys.reserve(meta.name_to_id.size()); + for (const auto& [key_name, _] : meta.name_to_id) { + selected_keys.push_back(key_name); + } + } + // get map type + auto map_type = arrow::internal::checked_pointer_cast(read_field->type()); + shared_shredding_name_to_context.emplace( + field_name, + MapSharedShreddingFileReader::SharedShreddingContext(meta, selected_keys, map_type)); + PAIMON_ASSIGN_OR_RAISE(int32_t field_id, + NestedProjectionUtils::GetPaimonFieldId(read_field)); + handled_shared_shredding_field_ids.insert(field_id); + } + if (!shared_shredding_name_to_context.empty()) { + file_reader = std::make_unique( + std::move(file_reader), std::move(shared_shredding_name_to_context), pool_); + } + return std::make_pair(std::move(file_reader), std::move(handled_shared_shredding_field_ids)); +} + Result> AbstractSplitRead::ProjectFieldsForRowTrackingAndDataEvolution( const std::shared_ptr& data_schema, const std::optional>& write_cols) { diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index 276b5277..dfbe6891 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -20,7 +20,9 @@ #include #include +#include #include +#include #include #include "arrow/type_fwd.h" @@ -98,7 +100,7 @@ class AbstractSplitRead : public SplitRead { const std::string& format_identifier) const; Result> CreateFileBatchReader( - const std::shared_ptr& file_meta, const std::string& data_file_path, + const std::string& file_format_identifier, const std::string& data_file_path, const ReaderBuilder* reader_builder) const; // return nullptr if data file is skipped by index or dv @@ -109,6 +111,10 @@ class AbstractSplitRead : public SplitRead { const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const; + Result, std::set>> + ApplySharedShreddingReaderIfNeeded(std::unique_ptr&& file_reader, + const std::shared_ptr& read_schema) const; + static bool NeedCompleteRowTrackingFields(bool row_tracking_enabled, const std::shared_ptr& read_schema); diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index 16216216..31ca2f9c 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -58,7 +58,7 @@ struct DeletionFile; /// splits)->(BlobViewResolvingBatchReader)->(CompleteIndexScoreBatchReader)-> /// CompleteRowKindBatchReader->(PredicateBatchReader) /// ->ConcatBatchReader across files->DataEvolutionFileReader->(ConcatBatchReader across blob files) -/// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader) +/// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader)->(MapSharedShreddingFileReader) /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader /// /// diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h index 89145f80..45bcddca 100644 --- a/src/paimon/core/operation/merge_file_split_read.h +++ b/src/paimon/core/operation/merge_file_split_read.h @@ -73,8 +73,8 @@ class MergeFunctionWrapper; /// ->ConcatBatchReader across no overlapped /// files->KeyValueProjectionReader/AsyncKeyValueProjectionReader /// ->DropDeleteReader->SortMergeReader->ConcatKeyValueRecordReader->KeyValueDataFileRecordReader -/// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(DelegatingPrefetchReader) -/// ->(PrefetchFileBatchReader)->FormatReader +/// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(MapSharedShreddingFileReader) +/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader class MergeFileSplitRead : public AbstractSplitRead { public: static Result> Create( diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index 3aa1ada6..ba5035f6 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -55,7 +55,7 @@ struct DeletionFile; /// splits)->CompleteRowKindBatchReader->(PredicateBatchReader) /// ->ConcatBatchReader across /// files->FieldMappingReader->(ApplyBitmapIndexBatchReader)->(CompleteRowTrackingFieldsBatchReader) -/// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader +/// ->(MapSharedShreddingFileReader)->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader class RawFileSplitRead : public AbstractSplitRead { public: diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index 70fb4bce..8f73daa4 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -46,29 +46,37 @@ std::shared_ptr NestedProjectionUtils::FindFieldByName( return nullptr; } -int32_t NestedProjectionUtils::GetPaimonFieldId(const std::shared_ptr& field) { - if (!field || !field->HasMetadata() || !field->metadata()) { - return -1; +Result NestedProjectionUtils::GetPaimonFieldId( + const std::shared_ptr& field) { + if (!field->HasMetadata() || !field->metadata()) { + return Status::Invalid(fmt::format( + "GetPaimonFieldId failed, do not exist metadata in field {}", field->name())); } auto result = field->metadata()->Get(DataField::FIELD_ID); if (!result.ok()) { - return -1; + return Status::Invalid( + fmt::format("GetPaimonFieldId failed, cannot find field_id in metadata in field {}", + field->name())); } std::optional field_id = StringUtils::StringToValue(result.ValueUnsafe()); - return field_id.value_or(-1); + if (!field_id) { + return Status::Invalid( + fmt::format("GetPaimonFieldId failed, cannot find convert field_id {} to int32", + result.ValueUnsafe())); + } + return field_id.value(); } -std::shared_ptr NestedProjectionUtils::FindFieldByPaimonId( +Result> NestedProjectionUtils::FindFieldByPaimonId( const std::shared_ptr& struct_type, int32_t field_id) { - if (!struct_type || struct_type->id() != arrow::Type::STRUCT) { - return nullptr; - } for (const auto& child : struct_type->fields()) { - if (GetPaimonFieldId(child) == field_id) { + PAIMON_ASSIGN_OR_RAISE(int32_t paimon_field_id, GetPaimonFieldId(child)); + if (paimon_field_id == field_id) { return child; } } - return nullptr; + return Status::Invalid( + fmt::format("cannot find field {} in struct type {}", field_id, struct_type->ToString())); } Result NestedProjectionUtils::HasNestedSubfieldProjectionType( @@ -148,21 +156,9 @@ Result>> NestedProjectionUtils::P case arrow::Type::STRUCT: { arrow::FieldVector pruned_fields; for (const auto& read_child : read_type->fields()) { - int32_t read_child_id = GetPaimonFieldId(read_child); - if (read_child_id < 0) { - return Status::Invalid(fmt::format( - "PruneDataType requires paimon.id for nested struct field '{}', but it " - "is missing or invalid", - read_child->name())); - } - std::shared_ptr data_child = - FindFieldByPaimonId(data_type, read_child_id); - if (!data_child) { - return Status::Invalid(fmt::format( - "PruneDataType does not support schema evolution inside struct: nested " - "field '{}' (id={}) does not exist in data type {}", - read_child->name(), read_child_id, data_type->ToString())); - } + PAIMON_ASSIGN_OR_RAISE(int32_t read_child_id, GetPaimonFieldId(read_child)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_child, + FindFieldByPaimonId(data_type, read_child_id)); if (read_child->name() != data_child->name()) { return Status::Invalid(fmt::format( "PruneDataType does not support schema evolution inside struct: nested " @@ -190,7 +186,6 @@ Result>> NestedProjectionUtils::P } return std::optional>(arrow::struct_(pruned_fields)); } - case arrow::Type::LIST: { // Keep behavior aligned with format readers: partial projection inside // LIST is unsupported and must fail fast. @@ -199,7 +194,6 @@ Result>> NestedProjectionUtils::P "vs target {}", data_type->ToString(), read_type->ToString())); } - case arrow::Type::MAP: { // Keep behavior aligned with format readers: partial projection inside // MAP is unsupported and must fail fast. @@ -207,7 +201,6 @@ Result>> NestedProjectionUtils::P "PruneDataType does not support partial projection inside map: src {} vs target {}", data_type->ToString(), read_type->ToString())); } - default: // Atomic type: return data_type as-is (type evolution is handled // separately by CastExecutor). @@ -240,28 +233,20 @@ Result NestedProjectionUtils::HasNestedSubfieldProjection( } // Map selected-keys support - Result> NestedProjectionUtils::GetMapSelectedKeys( const std::shared_ptr& field) { std::vector result; - if (!field || !field->HasMetadata() || !field->metadata()) { + if (!field->HasMetadata() || !field->metadata()) { return result; } auto get_result = field->metadata()->Get(DataField::MAP_SELECTED_KEYS); if (!get_result.ok()) { return result; } - std::string value = get_result.ValueUnsafe(); - if (value.empty()) { - // Metadata is explicitly present but empty: select the empty-string key. - result.push_back(""); - return result; - } - - auto tokens = StringUtils::Split(value, ",", /*ignore_empty=*/false); + auto tokens = StringUtils::Split(get_result.ValueUnsafe(), ",", /*ignore_empty=*/false); std::unordered_set deduplicated; deduplicated.reserve(tokens.size()); - for (auto& token : tokens) { + for (const auto& token : tokens) { if (!deduplicated.insert(token).second) { return Status::Invalid(fmt::format("Duplicate selected key '{}' in {} metadata", token, DataField::MAP_SELECTED_KEYS)); diff --git a/src/paimon/core/utils/nested_projection_utils.h b/src/paimon/core/utils/nested_projection_utils.h index 0677ea75..983d7376 100644 --- a/src/paimon/core/utils/nested_projection_utils.h +++ b/src/paimon/core/utils/nested_projection_utils.h @@ -43,12 +43,13 @@ class PAIMON_EXPORT NestedProjectionUtils { const std::string& name); /// Extract the paimon field ID from an Arrow field's metadata ("paimon.id"). - /// Returns -1 if the metadata key is not present. - static int32_t GetPaimonFieldId(const std::shared_ptr& field); + /// @return The paimon.id in metadata, or return bad status if the metadata key is not present + /// or convert error. + static Result GetPaimonFieldId(const std::shared_ptr& field); /// Find a child field in a STRUCT DataType by paimon field ID. - /// Returns nullptr if no child has the given ID. - static std::shared_ptr FindFieldByPaimonId( + /// @return The specific arrow field, or return bad status if no child has the given ID. + static Result> FindFieldByPaimonId( const std::shared_ptr& struct_type, int32_t field_id); /// Recursively prune `data_type` so that only the sub-fields requested by @@ -58,21 +59,20 @@ class PAIMON_EXPORT NestedProjectionUtils { /// Supported nesting: STRUCT, LIST (element recurse), MAP (key/value recurse). /// For atomic types, `data_type` is returned as-is. /// - /// Returns std::nullopt when all sub-fields of a STRUCT are pruned away + /// @return std::nullopt when all sub-fields of a STRUCT are pruned away /// (caller should skip this field entirely, mirroring Java's null return). static Result>> PruneDataType( const std::shared_ptr& read_type, const std::shared_ptr& data_type); - /// Returns true if `read_schema` requests a nested sub-field projection against + /// @return true if `read_schema` requests a nested sub-field projection against /// `file_schema` (same top-level field, but nested STRUCT/LIST/MAP subtree is pruned). static Result HasNestedSubfieldProjection( const std::shared_ptr& file_schema, const std::shared_ptr& read_schema); /// Parse the "paimon.map.selected-keys" metadata from an Arrow field. - /// Returns an empty vector if the field is null, has no metadata, or the metadata key - /// is absent. + /// @return an empty vector if has no metadata, or the specific metadata key is absent. /// The metadata value is a comma-separated string, e.g. "key1,key2". /// Empty tokens are preserved ("" means selecting empty-string keys), and duplicate /// selected keys are rejected as invalid. @@ -83,7 +83,7 @@ class PAIMON_EXPORT NestedProjectionUtils { /// Supports string keys and dictionary keys. /// The output map entry order follows /// `selected_keys` order, and duplicate selected keys are rejected. - /// Returns the original array unchanged if `selected_keys` is empty. + /// @return the original array unchanged if `selected_keys` is empty. static Result> FilterMapArrayBySelectedKeys( const std::shared_ptr& map_array, const std::vector& selected_keys, arrow::MemoryPool* pool); diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index 14dd8858..26b45015 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -44,16 +44,14 @@ static std::shared_ptr MakeField(const std::string& name, TEST(NestedProjectionUtilsTest, GetPaimonFieldIdPresent) { auto field = MakeField("col", arrow::int32(), 42); - ASSERT_EQ(NestedProjectionUtils::GetPaimonFieldId(field), 42); + ASSERT_OK_AND_ASSIGN(int32_t field_id, NestedProjectionUtils::GetPaimonFieldId(field)); + ASSERT_EQ(field_id, 42); } TEST(NestedProjectionUtilsTest, GetPaimonFieldIdMissing) { auto field = arrow::field("col", arrow::int32()); - ASSERT_EQ(NestedProjectionUtils::GetPaimonFieldId(field), -1); -} - -TEST(NestedProjectionUtilsTest, GetPaimonFieldIdNullptr) { - ASSERT_EQ(NestedProjectionUtils::GetPaimonFieldId(nullptr), -1); + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::GetPaimonFieldId(field), + "do not exist metadata in field"); } // ============== FindFieldByPaimonId ============== @@ -61,18 +59,16 @@ TEST(NestedProjectionUtilsTest, GetPaimonFieldIdNullptr) { TEST(NestedProjectionUtilsTest, FindFieldByPaimonIdFound) { auto struct_type = arrow::struct_({MakeField("x", arrow::int32(), 1), MakeField("y", arrow::utf8(), 2)}); - auto found = NestedProjectionUtils::FindFieldByPaimonId(struct_type, 2); + ASSERT_OK_AND_ASSIGN(std::shared_ptr found, + NestedProjectionUtils::FindFieldByPaimonId(struct_type, 2)); ASSERT_NE(found, nullptr); ASSERT_EQ(found->name(), "y"); } TEST(NestedProjectionUtilsTest, FindFieldByPaimonIdNotFound) { auto struct_type = arrow::struct_({MakeField("x", arrow::int32(), 1)}); - ASSERT_EQ(NestedProjectionUtils::FindFieldByPaimonId(struct_type, 99), nullptr); -} - -TEST(NestedProjectionUtilsTest, FindFieldByPaimonIdNonStruct) { - ASSERT_EQ(NestedProjectionUtils::FindFieldByPaimonId(arrow::int32(), 1), nullptr); + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::FindFieldByPaimonId(struct_type, 99), + "cannot find field 99"); } // ============== PruneDataType ============== @@ -116,7 +112,7 @@ TEST(NestedProjectionUtilsTest, PruneDataTypeStructAllFieldsPruned) { auto read_type = arrow::struct_({MakeField("y", arrow::int32(), 99)}); ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), - "does not support schema evolution inside struct"); + "cannot find field 99 in struct type"); } TEST(NestedProjectionUtilsTest, PruneDataTypeNestedStruct) { @@ -260,7 +256,6 @@ TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionMissingTopLevelFieldR } // ============== GetMapSelectedKeys ============== - TEST(NestedProjectionUtilsTest, GetMapSelectedKeysPresent) { auto metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"key1,key2,key3"}); @@ -316,11 +311,6 @@ TEST(NestedProjectionUtilsTest, GetMapSelectedKeysDuplicateKey) { "Duplicate selected key 'a'"); } -TEST(NestedProjectionUtilsTest, GetMapSelectedKeysNullptr) { - ASSERT_OK_AND_ASSIGN(auto keys, NestedProjectionUtils::GetMapSelectedKeys(nullptr)); - ASSERT_TRUE(keys.empty()); -} - // ============== FilterMapArrayBySelectedKeys ============== class NestedProjectionUtilsMapArrayTest : public ::testing::Test { diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index 41daa8b8..2d99dee4 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -52,8 +52,8 @@ Result> LuceneGlobalIndexReader::Create DataInputStream data_input_stream(paimon_input); PAIMON_ASSIGN_OR_RAISE(int32_t version, data_input_stream.ReadValue()); if (version != kVersion) { - return Status::Invalid(fmt::format("LuceneGlobalIndex not support version {}"), - kVersion); + return Status::Invalid( + fmt::format("LuceneGlobalIndex not support version {}", kVersion)); } PAIMON_ASSIGN_OR_RAISE(int32_t num_files, data_input_stream.ReadValue()); for (int32_t i = 0; i < num_files; i++) { diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index d8ffd614..f0e6f2da 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -16,6 +16,7 @@ * limitations under the License. */ +#include #include #include #include @@ -43,8 +44,10 @@ #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/data/blob_descriptor.h" +#include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/reader/reader_utils.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" @@ -170,6 +173,7 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter } } } + Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { // commit @@ -181,6 +185,25 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter return file_store_commit->Commit(commit_msgs); } + Status WriteNextSchema(const std::string& table_path, const std::vector& fields, + int32_t highest_field_id, + const std::map& options) const { + SchemaManager schema_manager(dir_->GetFileSystem(), table_path); + PAIMON_ASSIGN_OR_RAISE(auto latest_schema_opt, schema_manager.Latest()); + if (!latest_schema_opt) { + return Status::Invalid("table schema does not exist"); + } + auto next_schema = std::make_shared(*latest_schema_opt.value()); + next_schema->id_ = latest_schema_opt.value()->Id() + 1; + next_schema->fields_ = fields; + next_schema->highest_field_id_ = highest_field_id; + next_schema->options_ = options; + PAIMON_ASSIGN_OR_RAISE(std::string schema_content, next_schema->ToJsonString()); + std::string schema_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(next_schema->Id())); + return dir_->GetFileSystem()->AtomicStore(schema_path, schema_content); + } + /// Scan table and return the plan (without reading data). Result> ScanTable(const std::string& table_path, const std::shared_ptr& predicate = nullptr, @@ -2257,6 +2280,253 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { } } +// The shared-shredding map is read from one main data file while the blob payload is read from a +// separate blob file with the same row-id range. +TEST_P(BlobTableInteTest, TestSharedShreddingWithBlobDataEvolution) { + if (GetParam() != "parquet" && GetParam() != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + BlobUtils::ToArrowField("payload"), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto map_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[1]}), + R"([ + [1, [["a", 10], ["z", 11]]], + [2, [["a", 20]]], + [3, null] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto map_msgs, WriteArray(table_path, {}, {"id", "tags"}, {map_array})); + ASSERT_OK(Commit(table_path, map_msgs)); + + auto blob_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[2]}), + R"([ + [1, "payload-1"], + [2, "payload-2"], + [3, "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto blob_msgs, + WriteArray(table_path, {}, {"id", "payload"}, {blob_array})); + SetFirstRowId(0, blob_msgs); + ASSERT_OK(Commit(table_path, blob_msgs)); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["z", 11]], "payload-1"], + [2, [["a", 20]], "payload-2"], + [3, null, "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields)->field_names(), expected)); +} + +// Two independent shared-shredding map columns are written into different main files. +TEST_P(BlobTableInteTest, TestMultipleSharedShreddingMapsWithBlobDataEvolution) { + if (GetParam() != "parquet" && GetParam() != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("f0", arrow::map(arrow::utf8(), arrow::int64())), + arrow::field("f1", arrow::map(arrow::utf8(), arrow::utf8())), + BlobUtils::ToArrowField("payload"), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.f0.map.storage-layout", "shared-shredding"}, + {"fields.f0.map.shared-shredding.max-columns", "1"}, + {"fields.f1.map.storage-layout", "shared-shredding"}, + {"fields.f1.map.shared-shredding.max-columns", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto f0_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[1]}), + R"([ + [1, [["a", 10], ["z", 11]]], + [2, [["a", 20]]], + [3, null] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto f0_msgs, WriteArray(table_path, {}, {"id", "f0"}, {f0_array})); + ASSERT_OK(Commit(table_path, f0_msgs)); + + auto f1_blob_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[2], fields[3]}), + R"([ + [[["b", "red"], ["y", "blue"]], "payload-1"], + [[["b", "green"]], "payload-2"], + [[], "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto f1_blob_msgs, + WriteArray(table_path, {}, {"f1", "payload"}, {f1_blob_array})); + SetFirstRowId(0, f1_blob_msgs); + ASSERT_OK(Commit(table_path, f1_blob_msgs)); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["z", 11]], [["b", "red"], ["y", "blue"]], "payload-1"], + [2, [["a", 20]], [["b", "green"]], "payload-2"], + [3, null, [], "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields)->field_names(), expected)); +} + +// A newer partial data file rewrites only the shared-shredding map for the same row-id range. +TEST_P(BlobTableInteTest, TestSharedShreddingMapOverrideWithBlobDataEvolution) { + if (GetParam() != "parquet" && GetParam() != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + BlobUtils::ToArrowField("payload"), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto old_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["z", 11]], "payload-1"], + [2, [["a", 20]], "payload-2"], + [3, [["a", 30]], "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN( + auto old_msgs, + WriteArray(table_path, {}, arrow::schema(fields)->field_names(), {old_array})); + ASSERT_OK(Commit(table_path, old_msgs)); + + auto new_map_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[1]}), + R"([ + [[["a", 100], ["z", 101]]], + [null], + [[]] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto new_map_msgs, WriteArray(table_path, {}, {"tags"}, {new_map_array})); + SetFirstRowId(0, new_map_msgs); + ASSERT_OK(Commit(table_path, new_map_msgs)); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", 100], ["z", 101]], "payload-1"], + [2, null, "payload-2"], + [3, [], "payload-3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields)->field_names(), expected)); +} + +TEST_P(BlobTableInteTest, TestOrcMapStorageLayoutEvolutionWithBlobDataEvolution) { + if (GetParam() != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + BlobUtils::ToArrowField("payload"), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, "orc"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "default"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"orc.dictionary-key-size-threshold", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options_v0); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto array_v0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", "red"], ["z", "blue"]], "payload-1"], + [2, [["a", "red"], ["z", "green"]], "payload-2"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN( + auto msgs_v0, WriteArray(table_path, {}, arrow::schema(fields)->field_names(), {array_v0})); + ASSERT_OK(Commit(table_path, msgs_v0)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "shared-shredding"; + options_v1["fields.tags.map.shared-shredding.max-columns"] = "1"; + ASSERT_OK(WriteNextSchema( + table_path, {DataField(0, fields[0]), DataField(1, fields[1]), DataField(2, fields[2])}, + /*highest_field_id=*/2, options_v1)); + + auto array_v1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [3, [["a", "red"], ["z", "yellow"]], "payload-3"], + [4, [["a", "red"]], "payload-4"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN( + auto msgs_v1, WriteArray(table_path, {}, arrow::schema(fields)->field_names(), {array_v1})); + ASSERT_OK(Commit(table_path, msgs_v1)); + + auto expected = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + R"([ + [1, [["a", "red"], ["z", "blue"]], "payload-1"], + [2, [["a", "red"], ["z", "green"]], "payload-2"], + [3, [["a", "red"], ["z", "yellow"]], "payload-3"], + [4, [["a", "red"]], "payload-4"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields)->field_names(), expected)); +} + TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { if (GetParam() == "lance") { return; diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp index c18eb752..19a08fb9 100644 --- a/test/inte/nested_column_pruning_inte_test.cpp +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -1643,7 +1643,16 @@ TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { ASSERT_NOK_WITH_MSG(create_reader_result, "partial projection inside list"); } +std::vector GetTestValuesForNestedColumnPruningInteTest() { + std::vector values; + values.emplace_back("parquet"); +#ifdef PAIMON_ENABLE_ORC + values.emplace_back("orc"); +#endif + return values; +} + INSTANTIATE_TEST_SUITE_P(FileFormats, NestedColumnPruningInteTest, - ::testing::Values("parquet", "orc")); + ::testing::ValuesIn(GetTestValuesForNestedColumnPruningInteTest())); } // namespace paimon::test diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index e85c2344..7315cc16 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -26,12 +26,15 @@ #include #include "arrow/api.h" +#include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/schema/schema_manager.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" #include "paimon/predicate/literal.h" @@ -71,6 +74,77 @@ class WriteAndReadInteTest return new_options; } + Status WriteNextSchema(const std::vector& fields, int32_t highest_field_id, + const std::map& options) const { + auto file_system = dir_->GetFileSystem(); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + SchemaManager schema_manager(file_system, table_path); + PAIMON_ASSIGN_OR_RAISE(auto latest_schema_opt, schema_manager.Latest()); + if (!latest_schema_opt) { + return Status::Invalid("table schema does not exist"); + } + auto next_schema = std::make_shared(*latest_schema_opt.value()); + next_schema->id_ = latest_schema_opt.value()->Id() + 1; + next_schema->fields_ = fields; + next_schema->highest_field_id_ = highest_field_id; + next_schema->options_ = options; + PAIMON_ASSIGN_OR_RAISE(std::string schema_content, next_schema->ToJsonString()); + std::string schema_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(next_schema->Id())); + return file_system->AtomicStore(schema_path, schema_content); + } + + Result ReadAndCheckProjectedResult(const std::map& options, + const std::vector& read_fields, + const std::shared_ptr& expected_type, + const std::string& expected_data) const { + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadFieldNames(read_fields); + PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + auto expected, arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data)); + return std::make_shared(expected)->Equals(actual); + } + + /// Read with a custom Arrow read schema (supports nested column pruning and + /// paimon.map.selected-keys metadata for shared-shredding partial key recall). + Result ReadAndCheckWithReadSchema(const std::map& options, + const std::shared_ptr& read_schema, + const std::shared_ptr& expected_type, + const std::string& expected_data) const { + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + + auto c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_schema.get())); + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + auto expected, arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data)); + return std::make_shared(expected)->Equals(actual); + } + + Result> InnerScan( + const std::map& options) const { + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options).AddOption(Options::SCAN_MODE, + StartupMode::LatestFull().ToString()); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + return table_scan->CreatePlan(); + } + private: std::string test_dir_; std::unique_ptr dir_; @@ -1182,6 +1256,1003 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) { ASSERT_EQ(CacheKind::DATA_FILE_FOOTER, cache->LastKind()); } +TEST_P(WriteAndReadInteTest, TestAppendSharedShreddingMap) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + std::string data = R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30], ["a", 40], ["b", 50]]], + [3, null], + [4, [["d", 60], ["a", null], ["c", 70]]] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + std::string expected_data = R"([ + [0, 1, [["a", 10], ["b", 20]]], + [0, 2, [["a", 40], ["b", 50], ["c", 30]]], + [0, 3, null], + [0, 4, [["a", null], ["c", 70], ["d", 60]]] + ])"; + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(data_type, data_splits, expected_data)); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingWithSchemaEvolution) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields_v0 = { + arrow::field("f0", map_type), + arrow::field("f1", map_type), + arrow::field("k1", arrow::utf8()), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.f0.map.storage-layout", "shared-shredding"}, + {"fields.f0.map.shared-shredding.max-columns", "1"}, + {"fields.f1.map.storage-layout", "shared-shredding"}, + {"fields.f1.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields_v0), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), + R"([ + [[["a", 10], ["z", 11]], [["b", 20]], "old-1"], + [[["a", 12]], [["b", 21], ["y", 22]], "old-2"] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_v1 = { + arrow::field("f0", map_type), + arrow::field("f1", map_type), + arrow::field("k2", arrow::utf8()), + arrow::field("f2", map_type), + }; + std::map options_v1 = options_v0; + options_v1["fields.f2.map.storage-layout"] = "shared-shredding"; + options_v1["fields.f2.map.shared-shredding.max-columns"] = "1"; + std::vector data_fields_v1 = { + DataField(0, fields_v1[0]), + DataField(1, fields_v1[1]), + DataField(2, fields_v1[2]), + DataField(3, fields_v1[3]), + }; + ASSERT_OK(WriteNextSchema(data_fields_v1, /*highest_field_id=*/3, options_v1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v1), + R"([ + [[["a", 30], ["z", 31]], [["b", 40]], "new-1", [["c", 50], ["x", 51]]], + [[["a", 32]], [["b", 41]], "new-2", [["c", 52]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("f0", map_type), + arrow::field("f2", map_type), + arrow::field("k2", arrow::utf8()), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckProjectedResult(options_v1, {"f0", "f2", "k2"}, expected_type, + R"([ + [0, [["a", 10], ["z", 11]], null, "old-1"], + [0, [["a", 12]], null, "old-2"], + [0, [["a", 30], ["z", 31]], [["c", 50], ["x", 51]], "new-1"], + [0, [["a", 32]], [["c", 52]], "new-2"] + ])")); + ASSERT_TRUE(success); +} + +// Verify storage-layout evolution: default->shared-shredding. +TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShredding) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([[1, [["a", 10], ["z", 11]]], [2, null]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "shared-shredding"; + options_v1["fields.tags.map.shared-shredding.max-columns"] = "1"; + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[3, [["a", 30], ["z", 31]]], [4, [["a", 40]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", 10], ["z", 11]]], + [0, 2, null], + [0, 3, [["a", 30], ["z", 31]]], + [0, 4, [["a", 40]]] + ])")); + ASSERT_TRUE(success); +} + +// Verify storage-layout evolution: shared-shredding->default. +TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefault) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[1, [["a", 10], ["z", 11]]], [2, [["a", 20]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "default"; + options_v1.erase("fields.tags.map.shared-shredding.max-columns"); + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([[3, [["a", 30], ["z", 31]]], [4, null]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", 10], ["z", 11]]], + [0, 2, [["a", 20]]], + [0, 3, [["a", 30], ["z", 31]]], + [0, 4, null] + ])")); + ASSERT_TRUE(success); +} + +// Nested map values through both selected physical columns and overflow. +TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto value_type = arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::int32()), + }); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), value_type)), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", ["alice", 10]], ["z", ["zoe", 11]]]], + [2, [["a", ["amy", null]]]], + [3, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", ["alice", 10]], ["z", ["zoe", 11]]]], + [0, 2, [["a", ["amy", null]]]], + [0, 3, null] + ])")); + ASSERT_TRUE(success); +} + +// Keep ORC lazy dictionary decoding enabled across a default -> shared-shredding schema change. +// The test inspects every user-visible batch directly, because ReadResultCollector would otherwise +// decode dictionary arrays and hide a type mismatch between old and new files. +TEST_P(WriteAndReadInteTest, TestOrcDictionaryLazyDecodingWithSharedShredding) { + auto [file_format, file_system] = GetParam(); + if (file_format != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"}, + {"orc.read.enable-lazy-decoding", "true"}, {"orc.dictionary-key-size-threshold", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", "red"], ["z", "blue"]]], + [2, [["a", "red"], ["z", "green"]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options; + options_v1["fields.tags.map.storage-layout"] = "shared-shredding"; + options_v1["fields.tags.map.shared-shredding.max-columns"] = "1"; + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + + helper.reset(); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(helper, TestHelper::Create(table_path, options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", "red"], ["z", "yellow"]]], + [4, [["a", "red"]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", "red"], ["z", "blue"]]], + [0, 2, [["a", "red"], ["z", "green"]]], + [0, 3, [["a", "red"], ["z", "yellow"]]], + [0, 4, [["a", "red"]]] + ])")); + ASSERT_TRUE(success); +} + +// Verify shared-shredding in the PK read path. +TEST_P(WriteAndReadInteTest, TestPkSharedShreddingMap) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("pk", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN(auto batch_0, TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[1, [["a", 10], ["z", 11]]], [2, [["b", 20]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto batch_1, TestHelper::MakeRecordBatch( + arrow::struct_(fields), + R"([[1, [["a", 100], ["z", 101]]], [3, [["c", 30]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", 100], ["z", 101]]], + [0, 2, [["b", 20]]], + [0, 3, [["c", 30]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithOverflow) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + // max-columns=2, so keys are allocated in insertion order: + // Row 1: {a:1, b:2, c:3, d:4} -> physical cols: a, b; overflow: c, d + // Row 2: {a:10, b:20} -> physical cols: a, b; no overflow + // Row 3: null + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 1], ["b", 2], ["c", 3], ["d", 4]]], + [2, [["a", 10], ["b", 20]]], + [3, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + // Sub-case 1: read only key "a" (in physical column) + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["a", 1]]], + [0, 2, [["a", 10]]], + [0, 3, null] + ])")); + ASSERT_TRUE(success); + } + + // Sub-case 2: read only key "c" (in overflow) + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"c"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["c", 3]]], + [0, 2, []], + [0, 3, null] + ])")); + ASSERT_TRUE(success); + } + + // Sub-case 3: read keys "c" and "a" (cross physical + overflow), order follows user request + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"c,a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["c", 3], ["a", 1]]], + [0, 2, [["a", 10]]], + [0, 3, null] + ])")); + ASSERT_TRUE(success); + } +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithNullOrMissingKey) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + // Row 1: tags=null + // Row 2: tags={b:2} (no key "a") + // Row 3: tags={a:30, b:40} + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, null], + [2, [["b", 2]]], + [3, [["a", 30], ["b", 40]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + // Sub-case 1: row with null tags, request key "a" -> tags should be null + // Sub-case 2: row without key "a", request key "a" -> key "a" maps to null value + // Sub-case 3: row with key "a", request key "a" -> normal value + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, null], + [0, 2, []], + [0, 3, [["a", 30]]] + ])")); + ASSERT_TRUE(success); + } + + // Sub-case 4: request a key that was never written ("nonexistent") -> all rows have null value + { + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"nonexistent"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, null], + [0, 2, []], + [0, 3, []] + ])")); + ASSERT_TRUE(success); + } +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallMultipleColumns) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + arrow::field("metrics", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "3"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 1], ["b", 2]], [["x", 100], ["y", 200]]], + [2, [["a", 10], ["c", 30]], [["x", 1000], ["z", 3000]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + // Sub-case 1: partial key recall on both columns independently + { + auto tags_meta = arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"a"}); + auto metrics_meta = arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"x"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(tags_meta), + arrow::field("metrics", map_type)->WithMetadata(metrics_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + arrow::field("metrics", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["a", 1]], [["x", 100]]], + [0, 2, [["a", 10]], [["x", 1000]]] + ])")); + ASSERT_TRUE(success); + } + + // Sub-case 2: partial key recall on tags + full recall on metrics + { + auto tags_meta = arrow::KeyValueMetadata::Make({"paimon.map.selected-keys"}, {"a,b"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(tags_meta), + arrow::field("metrics", map_type), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + arrow::field("metrics", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [["a", 1], ["b", 2]], [["x", 100], ["y", 200]]], + [0, 2, [["a", 10]], [["x", 1000], ["z", 3000]]] + ])")); + ASSERT_TRUE(success); + } +} + +TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShreddingPartialKeyRecall) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, {"fields.tags.map.storage-layout", "default"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["b", 20]]], + [2, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "shared-shredding"; + options_v1["fields.tags.map.shared-shredding.max-columns"] = "1"; + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", 30], ["z", 31]]], + [4, [["z", 41]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + auto selected_keys_meta = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options_v1, read_schema, expected_type, + R"([ + [0, 1, [["a", 10]]], + [0, 2, null], + [0, 3, [["a", 30]]], + [0, 4, []] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefaultPartialKeyRecall) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["z", 11]]], + [2, [["z", 21]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "default"; + options_v1.erase("fields.tags.map.shared-shredding.max-columns"); + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options_v1, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", 30], ["z", 31]]], + [4, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + auto selected_keys_meta = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool success, + ReadAndCheckWithReadSchema(options_v1, read_schema, expected_type, + R"([ + [0, 1, [["a", 10]]], + [0, 2, []], + [0, 3, [["a", 30]]], + [0, 4, null] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingDuplicateSelectedKeys) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, TestHelper::MakeRecordBatch( + arrow::struct_(fields), R"([[1, [["a", 10], ["b", 20]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_NOK_WITH_MSG(ReadAndCheckWithReadSchema(options, read_schema, expected_type, "[]"), + "Duplicate selected key 'a'"); +} + +TEST_P(WriteAndReadInteTest, TestSharedShreddingAllNullMapColumn) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, null], + [2, null], + [3, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, null], + [0, 2, null], + [0, 3, null] + ])")); + ASSERT_TRUE(success); +} + INSTANTIATE_TEST_SUITE_P(FileFormatAndFileSystem, WriteAndReadInteTest, ::testing::ValuesIn(GetTestValuesForWriteAndReadInteTest())); From a3f1516cedf50835f6853ad26bfa81b708313ef3 Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Tue, 30 Jun 2026 02:22:09 -0700 Subject: [PATCH 075/138] feat: add CreateReader api with field_name & index_type From c24aa3d8c60b608fe32983777fb505113f36c566 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Tue, 30 Jun 2026 20:47:35 +0800 Subject: [PATCH 076/138] feat(shredding): add shared-shredding map placement policies --- include/paimon/defs.h | 6 + src/paimon/CMakeLists.txt | 5 +- ..._map_shared_shredding_column_allocator.cpp | 110 +++++++++ ...ru_map_shared_shredding_column_allocator.h | 52 +++++ ...shared_shredding_column_allocator_test.cpp | 77 +++++++ .../map_shared_shredding_batch_converter.cpp | 89 +++++--- .../map_shared_shredding_batch_converter.h | 62 ++--- ..._shared_shredding_batch_converter_test.cpp | 214 +++++++++++++----- .../map_shared_shredding_column_allocator.cpp | 27 +-- .../map_shared_shredding_column_allocator.h | 27 ++- ...shared_shredding_column_allocator_test.cpp | 113 --------- .../map_shared_shredding_file_reader_test.cpp | 9 +- .../shredding/map_shared_shredding_utils.cpp | 2 +- .../shredding/map_shared_shredding_utils.h | 66 +++--- .../map_shared_shredding_utils_test.cpp | 30 +-- ...in_map_shared_shredding_column_allocator.h | 58 +++++ ...shared_shredding_column_allocator_test.cpp | 131 +++++++++++ ...al_map_shared_shredding_column_allocator.h | 45 ++++ ...shared_shredding_column_allocator_test.cpp | 60 +++++ src/paimon/common/defs.cpp | 2 + .../core/append/append_only_writer_test.cpp | 135 ++++++++--- src/paimon/core/core_options.cpp | 18 ++ src/paimon/core/core_options.h | 3 + src/paimon/core/core_options_test.cpp | 44 +++- ...edding_append_data_file_writer_factory.cpp | 16 +- ...ing_key_value_data_file_writer_factory.cpp | 16 +- .../core/mergetree/merge_tree_writer_test.cpp | 19 +- .../append_only_file_store_write_test.cpp | 24 +- .../key_value_file_store_write_test.cpp | 6 +- ...shared_shredding_column_placement_policy.h | 55 +++++ .../postpone/postpone_bucket_writer_test.cpp | 12 +- src/paimon/core/schema/schema_validation.cpp | 2 + .../core/schema/schema_validation_test.cpp | 15 ++ 33 files changed, 1151 insertions(+), 399 deletions(-) create mode 100644 src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.cpp create mode 100644 src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h create mode 100644 src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp delete mode 100644 src/paimon/common/data/shredding/map_shared_shredding_column_allocator_test.cpp create mode 100644 src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h create mode 100644 src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp create mode 100644 src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h create mode 100644 src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp create mode 100644 src/paimon/core/options/map_shared_shredding_column_placement_policy.h diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 91910ee5..1d9271ae 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -383,6 +383,12 @@ struct PAIMON_EXPORT Options { /// map.storage-layout = shared-shredding. Rows with more fields than K_max spill to /// __overflow. Default value is 256. Each column can have its own max-columns setting. static const char MAP_SHARED_SHREDDING_MAX_COLUMNS[]; + /// "map.shared-shredding.column-placement-policy" - Suffix for per-column shared-shredding + /// physical column placement policy. + /// Used as `fields..map.shared-shredding.column-placement-policy`. + /// Values: "plain", "sequential" and "lru". Default value is "lru". + /// Only effective when map.storage-layout = shared-shredding. + static const char MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY[]; /// "blob-as-descriptor" - Read blob field using blob descriptor rather than blob /// bytes. Default value is "false". diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index d2e3787c..18b09e67 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -144,6 +144,7 @@ set(PAIMON_COMMON_SRCS common/data/shredding/map_shared_shredding_context.cpp common/data/shredding/map_shared_shredding_batch_converter.cpp common/data/shredding/map_shared_shredding_column_allocator.cpp + common/data/shredding/lru_map_shared_shredding_column_allocator.cpp common/data/shredding/map_shared_shredding_file_reader.cpp common/utils/delta_varint_compressor.cpp common/utils/fields_comparator.cpp @@ -549,7 +550,9 @@ if(PAIMON_BUILD_TESTS) common/utils/generic_lru_cache_test.cpp common/data/shredding/map_shared_shredding_utils_test.cpp common/data/shredding/map_shared_shredding_batch_converter_test.cpp - common/data/shredding/map_shared_shredding_column_allocator_test.cpp + common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp + common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp + common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp common/data/shredding/map_shared_shredding_field_dict_test.cpp common/data/shredding/map_shared_shredding_context_test.cpp common/data/shredding/map_shared_shredding_file_reader_test.cpp diff --git a/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.cpp b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.cpp new file mode 100644 index 00000000..ae5228ed --- /dev/null +++ b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.cpp @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h" + +#include +#include + +namespace paimon { + +LruMapSharedShreddingColumnAllocator::LruMapSharedShreddingColumnAllocator(int32_t num_columns) + : MapSharedShreddingColumnAllocator(num_columns), + col_field_(num_columns, -1), + last_used_(num_columns, 0) {} + +int32_t LruMapSharedShreddingColumnAllocator::SelectColumn( + const std::vector& candidates, + const std::vector& planned_col_to_field) const { + int32_t selected_col = candidates.front(); + int64_t selected_last_used = std::numeric_limits::max(); + for (int32_t col : candidates) { + if (planned_col_to_field[col] == -1) { + return col; + } + if (last_used_[col] < selected_last_used) { + selected_col = col; + selected_last_used = last_used_[col]; + } + } + return selected_col; +} + +void LruMapSharedShreddingColumnAllocator::UpdateLastUsed(const RowAllocation& allocation) { + bool touched = false; + for (int32_t col = 0; col < num_columns_; ++col) { + if (allocation.col_to_field[col] != -1) { + last_used_[col] = lru_clock_; + touched = true; + } + } + if (touched) { + ++lru_clock_; + } +} + +RowAllocation LruMapSharedShreddingColumnAllocator::AllocateRow( + const std::vector& field_ids) { + std::vector sorted_field_ids = field_ids; + std::sort(sorted_field_ids.begin(), sorted_field_ids.end()); + + RowAllocation allocation; + allocation.col_to_field.assign(num_columns_, -1); + std::vector next_col_to_field = col_field_; + std::vector used_cols(num_columns_, false); + std::vector unassigned; + + for (int32_t field_id : sorted_field_ids) { + auto it = std::find(col_field_.begin(), col_field_.end(), field_id); + if (it != col_field_.end()) { + int32_t col = static_cast(it - col_field_.begin()); + used_cols[col] = true; + allocation.col_to_field[col] = field_id; + } else { + unassigned.push_back(field_id); + } + } + + for (int32_t field_id : unassigned) { + std::vector candidates; + candidates.reserve(num_columns_); + for (int32_t col = 0; col < num_columns_; ++col) { + if (!used_cols[col]) { + candidates.push_back(col); + } + } + + if (candidates.empty()) { + allocation.overflow_fields.push_back(field_id); + continue; + } + + int32_t col = SelectColumn(candidates, next_col_to_field); + used_cols[col] = true; + allocation.col_to_field[col] = field_id; + next_col_to_field[col] = field_id; + } + + UpdateLastUsed(allocation); + col_field_ = next_col_to_field; + CommitRow(allocation, sorted_field_ids); + return allocation; +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h new file mode 100644 index 00000000..8c2bb1a4 --- /dev/null +++ b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" + +namespace paimon { + +/// Allocator that keeps current column assignments across rows and evicts least-recently-used +/// columns. +/// +/// Keys that are still assigned to physical columns keep those columns when they appear again. +/// New keys, including keys that were previously evicted, use empty columns first; if no empty +/// column is available, the allocator replaces the least-recently-used physical column. +class LruMapSharedShreddingColumnAllocator : public MapSharedShreddingColumnAllocator { + public: + /// @param num_columns Number of available physical columns. + explicit LruMapSharedShreddingColumnAllocator(int32_t num_columns); + + RowAllocation AllocateRow(const std::vector& field_ids) override; + + private: + int32_t SelectColumn(const std::vector& candidates, + const std::vector& planned_col_to_field) const; + void UpdateLastUsed(const RowAllocation& allocation); + + int64_t lru_clock_ = 0; + std::vector col_field_; + std::vector last_used_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp new file mode 100644 index 00000000..a04f886e --- /dev/null +++ b/src/paimon/common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { +namespace { + +void ExpectAllocation(const RowAllocation& allocation, const std::vector& col_to_field, + const std::vector& overflow_fields) { + ASSERT_EQ(col_to_field, allocation.col_to_field); + ASSERT_EQ(overflow_fields, allocation.overflow_fields); +} + +} // namespace + +TEST(LruMapSharedShreddingColumnAllocatorTest, AllocatesWithHitRetainEvictAndOverflow) { + LruMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation row0 = allocator.AllocateRow({0, 1, 2}); + ExpectAllocation(row0, {0, 1, 2}, {}); + + RowAllocation row1 = allocator.AllocateRow({0, 1}); + ExpectAllocation(row1, {0, 1, -1}, {}); + + RowAllocation row2 = allocator.AllocateRow({3, 4, 5}); + ExpectAllocation(row2, {4, 5, 3}, {}); + + RowAllocation row3 = allocator.AllocateRow({0, 3, 4, 5}); + ExpectAllocation(row3, {4, 5, 3}, {0}); + + ASSERT_EQ(4, allocator.GetMaxRowWidth()); + + const auto& field_to_columns = allocator.GetFieldToColumns(); + ASSERT_EQ((std::set{0}), field_to_columns.at(0)); + ASSERT_EQ((std::set{1}), field_to_columns.at(1)); + ASSERT_EQ((std::set{2}), field_to_columns.at(2)); + ASSERT_EQ((std::set{2}), field_to_columns.at(3)); + ASSERT_EQ((std::set{0}), field_to_columns.at(4)); + ASSERT_EQ((std::set{1}), field_to_columns.at(5)); + ASSERT_EQ((std::set{0}), allocator.GetOverflowFieldSet()); +} + +TEST(LruMapSharedShreddingColumnAllocatorTest, HandlesEmptyRows) { + LruMapSharedShreddingColumnAllocator allocator(2); + + RowAllocation empty_row = allocator.AllocateRow({}); + ExpectAllocation(empty_row, {-1, -1}, {}); + ASSERT_EQ(0, allocator.GetMaxRowWidth()); + + RowAllocation row = allocator.AllocateRow({7}); + ExpectAllocation(row, {7, -1}, {}); + ASSERT_EQ(1, allocator.GetMaxRowWidth()); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp index 1bc89a65..73960fe1 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp @@ -19,7 +19,7 @@ #include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" -#include +#include #include #include @@ -28,11 +28,15 @@ #include "arrow/c/bridge.h" #include "arrow/type.h" #include "fmt/format.h" +#include "paimon/common/data/shredding/lru_map_shared_shredding_column_allocator.h" #include "paimon/common/data/shredding/map_shared_shredding_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h" +#include "paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/core/core_options.h" namespace paimon { /// Checks that a dynamic_cast result is not null, returning Status::Invalid on failure. #define PAIMON_CHECK_NOT_NULL(ptr, msg) \ @@ -42,32 +46,34 @@ namespace paimon { } \ } while (false) -Result -MapSharedShreddingBatchConverter::CreateConverter( - const std::shared_ptr& logical_schema, - const std::shared_ptr& context, - const std::shared_ptr& pool) { - ConverterBundle bundle; - if (!context) { - return bundle; +namespace { + +Result> CreateMapSharedShreddingColumnAllocator( + int32_t num_columns, MapSharedShreddingColumnPlacementPolicy placement_policy) { + switch (placement_policy) { + case MapSharedShreddingColumnPlacementPolicy::PLAIN: + return std::make_unique(num_columns); + case MapSharedShreddingColumnPlacementPolicy::SEQUENTIAL: + return std::make_unique(num_columns); + case MapSharedShreddingColumnPlacementPolicy::LRU: + return std::make_unique(num_columns); } - - std::map field_to_k = context->ComputeNextK(); - PAIMON_ASSIGN_OR_RAISE(bundle.physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, field_to_k)); - bundle.converter = std::make_shared( - logical_schema, bundle.physical_schema, field_to_k, pool); - return bundle; + return Status::Invalid("unknown shared-shredding column placement policy"); } -MapSharedShreddingBatchConverter::MapSharedShreddingBatchConverter( +} // namespace + +Result> MapSharedShreddingBatchConverter::Create( const std::shared_ptr& logical_schema, - const std::shared_ptr& physical_schema, - const std::map& field_to_num_columns, - const std::shared_ptr& pool) - : logical_schema_(logical_schema), - physical_schema_(physical_schema), - pool_(GetArrowPool(pool)) { + const std::shared_ptr& context, const CoreOptions& options, + const std::shared_ptr& pool) { + std::map field_to_num_columns = context->ComputeNextK(); + std::shared_ptr physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, field_to_num_columns); + std::vector contexts; + std::vector shredding_field_names; + contexts.reserve(field_to_num_columns.size()); + shredding_field_names.reserve(field_to_num_columns.size()); // Iterate in schema field order (not map order) so that shredding_field_names_ // matches the order in which shredding columns appear in the schema. // This is critical for the sequential matching logic in Convert(). @@ -75,10 +81,33 @@ MapSharedShreddingBatchConverter::MapSharedShreddingBatchConverter( const std::string& name = logical_schema->field(i)->name(); auto it = field_to_num_columns.find(name); if (it != field_to_num_columns.end()) { - contexts_.emplace_back(name, it->second); - shredding_field_names_.push_back(name); + int32_t num_columns = it->second; + PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingColumnPlacementPolicy placement_policy, + options.GetMapSharedShreddingColumnPlacementPolicy(name)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr allocator, + CreateMapSharedShreddingColumnAllocator(num_columns, placement_policy)); + contexts.emplace_back(name, num_columns, std::move(allocator)); + shredding_field_names.push_back(name); } } + return std::shared_ptr( + new MapSharedShreddingBatchConverter(logical_schema, physical_schema, std::move(contexts), + std::move(shredding_field_names), pool)); +} + +MapSharedShreddingBatchConverter::MapSharedShreddingBatchConverter( + const std::shared_ptr& logical_schema, + const std::shared_ptr& physical_schema, std::vector&& contexts, + std::vector&& shredding_field_names, const std::shared_ptr& pool) + : logical_schema_(logical_schema), + physical_schema_(physical_schema), + contexts_(std::move(contexts)), + shredding_field_names_(std::move(shredding_field_names)), + pool_(GetArrowPool(pool)) {} + +const std::shared_ptr& MapSharedShreddingBatchConverter::GetPhysicalSchema() const { + return physical_schema_; } Result> MapSharedShreddingBatchConverter::Convert( @@ -197,7 +226,7 @@ Result> MapSharedShreddingBatchConverter::ConvertO &field_id_to_value_index); // Allocate columns - RowAllocation allocation = context->allocator.AllocateRow(field_ids); + RowAllocation allocation = context->allocator->AllocateRow(field_ids); // Fill sub-columns PAIMON_RETURN_NOT_OK(AppendFieldMapping(allocation, num_cols, field_mapping_builder, @@ -296,13 +325,13 @@ Result MapSharedShreddingBatchConverter::BuildField MapSharedShreddingFieldMeta meta; meta.name_to_id = context.dict.GetNameToId(); // Convert set -> vector for field_to_columns - for (const auto& [field_id, col_set] : context.allocator.GetFieldToColumns()) { + for (const auto& [field_id, col_set] : context.allocator->GetFieldToColumns()) { meta.field_to_columns[field_id] = std::vector(col_set.begin(), col_set.end()); } - meta.overflow_field_set = context.allocator.GetOverflowFieldSet(); - meta.num_columns = context.allocator.GetNumColumns(); - meta.max_row_width = context.allocator.GetMaxRowWidth(); + meta.overflow_field_set = context.allocator->GetOverflowFieldSet(); + meta.num_columns = context.num_columns; + meta.max_row_width = context.allocator->GetMaxRowWidth(); return meta; } } diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h index 8ec30007..ea49f2ec 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h @@ -20,7 +20,6 @@ #pragma once #include -#include #include #include #include @@ -39,6 +38,7 @@ struct ArrowArray; namespace paimon { +class CoreOptions; class MapSharedShreddingContext; /// Converts logical batches containing MAP columns into physical batches @@ -49,42 +49,20 @@ class MapSharedShreddingContext; /// Each shared-shredding column has its own FieldDict and ColumnAllocator. class MapSharedShreddingBatchConverter { public: - /// Per-column context for one shared-shredding MAP column. - struct ColumnContext { - std::string field_name; - int32_t num_columns; // K - MapSharedShreddingFieldDict dict; - MapSharedShreddingColumnAllocator allocator; - - ColumnContext(const std::string& _field_name, int32_t _num_columns) - : field_name(_field_name), num_columns(_num_columns), allocator(_num_columns) {} - }; - - struct ConverterBundle { - std::shared_ptr converter; - std::shared_ptr physical_schema; - }; - - /// Creates a converter + physical schema for one file write cycle. + /// Creates a converter for one file write cycle. /// Computes per-file K from context, builds physical schema, and constructs the converter. /// @param logical_schema The original schema with MAP columns. /// @param context The cross-file shared context for K adaptation. + /// @param options CoreOptions used to read each column's placement policy. /// @param pool Paimon memory pool for Arrow allocations. - /// @return A struct containing the converter and physical schema. - static Result CreateConverter( + /// @return The converter. + static Result> Create( const std::shared_ptr& logical_schema, - const std::shared_ptr& context, + const std::shared_ptr& context, const CoreOptions& options, const std::shared_ptr& pool); - /// Constructs a converter. - /// @param logical_schema The original schema with MAP columns. - /// @param physical_schema The physical schema (MAP columns replaced with STRUCT). - /// @param field_to_num_columns Map from field name to K. - /// @param pool Paimon memory pool for Arrow allocations. - MapSharedShreddingBatchConverter(const std::shared_ptr& logical_schema, - const std::shared_ptr& physical_schema, - const std::map& field_to_num_columns, - const std::shared_ptr& pool); + /// Returns the physical schema produced for this converter. + const std::shared_ptr& GetPhysicalSchema() const; /// Converts a logical batch to a physical batch. /// @param logical_batch Input ArrowArray (C ABI) with logical schema. Consumed on success. @@ -99,6 +77,30 @@ class MapSharedShreddingBatchConverter { const std::vector& GetShreddingColumnNames() const; private: + /// Per-column context for one shared-shredding MAP column. + struct ColumnContext { + std::string field_name; + int32_t num_columns; // K + MapSharedShreddingFieldDict dict; + std::unique_ptr allocator; + + ColumnContext(const std::string& field_name, int32_t num_columns, + std::unique_ptr&& allocator) + : field_name(field_name), num_columns(num_columns), allocator(std::move(allocator)) {} + }; + + /// Constructs a converter. + /// @param logical_schema The original schema with MAP columns. + /// @param physical_schema The physical schema (MAP columns replaced with STRUCT). + /// @param contexts Per-shredding-column conversion contexts. + /// @param shredding_field_names Shared-shredding field names in schema order. + /// @param pool Paimon memory pool for Arrow allocations. + MapSharedShreddingBatchConverter(const std::shared_ptr& logical_schema, + const std::shared_ptr& physical_schema, + std::vector&& contexts, + std::vector&& shredding_field_names, + const std::shared_ptr& pool); + /// Converts one MAP column to physical STRUCT for all rows. /// @param physical_struct_type The physical struct type from physical_schema for this column. Result> ConvertOneColumn( diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp index 643dad42..db63fd7c 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp @@ -19,6 +19,7 @@ #include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include #include #include #include @@ -28,8 +29,10 @@ #include "arrow/ipc/json_simple.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/core/core_options.h" #include "paimon/memory/memory_pool.h" #include "paimon/testing/utils/testharness.h" @@ -60,6 +63,15 @@ class MapSharedShreddingBatchConverterTest : public ::testing::Test { << expected->ToString() << "\nActual:\n" << actual->ToString(); } + + Result MakeCoreOptions(const std::map& field_to_policy) { + std::map options; + for (const auto& [field_name, policy] : field_to_policy) { + options.emplace( + "fields." + field_name + ".map.shared-shredding.column-placement-policy", policy); + } + return CoreOptions::FromMap(options); + } }; TEST_F(MapSharedShreddingBatchConverterTest, BasicConversion) { @@ -68,11 +80,12 @@ TEST_F(MapSharedShreddingBatchConverterTest, BasicConversion) { arrow::field("id", arrow::int32()), arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), }); - std::map field_to_num_columns = {{"tags", 3}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, field_to_num_columns)); - MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, - field_to_num_columns, pool_); + auto context = + std::make_shared(std::map{{"tags", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); auto logical_type = arrow::struct_(logical_schema->fields()); auto physical_type = arrow::struct_(physical_schema->fields()); @@ -84,7 +97,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, BasicConversion) { [100, [["a", 1], ["b", 2]]], [200, [["b", 3], ["c", 4], ["a", 5]]] ])", - physical_type, &converter); + physical_type, converter.get()); // Expected physical: [id, [mapping, col0, col1, col2, overflow]] // Row0: a=fid0->col0, b=fid1->col1, col2 unused // Row1: b=fid1->col0, c=fid2->col1, a=fid0->col2 @@ -97,7 +110,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, BasicConversion) { AssertArrayEquals(expected, actual); // Verify GetShreddingColumnNames - ASSERT_EQ(std::vector({"tags"}), converter.GetShreddingColumnNames()); + ASSERT_EQ(std::vector({"tags"}), converter->GetShreddingColumnNames()); // Verify BuildFieldMeta: a=0,b=1,c=2, K=3, max_row_width=3, no overflow MapSharedShreddingFieldMeta expected_meta; @@ -105,7 +118,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, BasicConversion) { expected_meta.field_to_columns = {{0, {0, 2}}, {1, {0, 1}}, {2, {1}}}; expected_meta.num_columns = 3; expected_meta.max_row_width = 3; - ASSERT_EQ(expected_meta, converter.BuildFieldMeta("tags").value()); + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value()); } TEST_F(MapSharedShreddingBatchConverterTest, NestedValueStruct) { @@ -118,11 +131,12 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueStruct) { arrow::field("id", arrow::int32()), arrow::field("props", arrow::map(arrow::utf8(), value_type)), }); - std::map field_to_num_columns = {{"props", 2}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, field_to_num_columns)); - MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, - field_to_num_columns, pool_); + auto context = + std::make_shared(std::map{{"props", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"props", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); auto logical_type = arrow::struct_(logical_schema->fields()); auto physical_type = arrow::struct_(physical_schema->fields()); @@ -136,7 +150,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueStruct) { [3, [["a", [null, null]], ["c", [5, 5.5]], ["b", [6, 6.5]]]], [4, null] ])", - physical_type, &converter); + physical_type, converter.get()); auto expected = ArrayFromJSON(physical_type, R"([ [1, [[0, 1], [1, 1.5], [null, 2.5], null]], @@ -149,7 +163,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueStruct) { AssertArrayEquals(expected, actual); // Verify GetShreddingColumnNames - ASSERT_EQ(std::vector({"props"}), converter.GetShreddingColumnNames()); + ASSERT_EQ(std::vector({"props"}), converter->GetShreddingColumnNames()); // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, b overflowed in row2 MapSharedShreddingFieldMeta expected_meta; @@ -158,7 +172,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueStruct) { expected_meta.overflow_field_set = {1}; expected_meta.num_columns = 2; expected_meta.max_row_width = 3; - ASSERT_EQ(expected_meta, converter.BuildFieldMeta("props").value()); + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("props").value()); } TEST_F(MapSharedShreddingBatchConverterTest, NestedValueList) { @@ -167,11 +181,12 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueList) { arrow::field("id", arrow::int32()), arrow::field("tags", arrow::map(arrow::utf8(), arrow::list(arrow::int32()))), }); - std::map field_to_num_columns = {{"tags", 2}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, field_to_num_columns)); - MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, - field_to_num_columns, pool_); + auto context = + std::make_shared(std::map{{"tags", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); auto logical_type = arrow::struct_(logical_schema->fields()); auto physical_type = arrow::struct_(physical_schema->fields()); @@ -185,7 +200,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueList) { [3, [["c", [5, 6, 7]]]], [4, [["b", [8]], ["a", [9, 10]], ["c", [null]]]] ])", - physical_type, &converter); + physical_type, converter.get()); auto expected = ArrayFromJSON(physical_type, R"([ [1, [[0, 1], [1, null, 2], [3], null]], @@ -198,7 +213,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueList) { AssertArrayEquals(expected, actual); // Verify GetShreddingColumnNames - ASSERT_EQ(std::vector({"tags"}), converter.GetShreddingColumnNames()); + ASSERT_EQ(std::vector({"tags"}), converter->GetShreddingColumnNames()); // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, c overflowed in row3 MapSharedShreddingFieldMeta expected_meta; @@ -207,7 +222,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueList) { expected_meta.overflow_field_set = {2}; expected_meta.num_columns = 2; expected_meta.max_row_width = 3; - ASSERT_EQ(expected_meta, converter.BuildFieldMeta("tags").value()); + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value()); } TEST_F(MapSharedShreddingBatchConverterTest, NestedValueMap) { @@ -217,11 +232,12 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueMap) { arrow::field("id", arrow::int32()), arrow::field("nested", arrow::map(arrow::utf8(), inner_map_type)), }); - std::map field_to_num_columns = {{"nested", 2}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, field_to_num_columns)); - MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, - field_to_num_columns, pool_); + auto context = + std::make_shared(std::map{{"nested", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"nested", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); auto logical_type = arrow::struct_(logical_schema->fields()); auto physical_type = arrow::struct_(physical_schema->fields()); @@ -235,7 +251,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueMap) { [3, null], [4, [["a", [["m", 7]]], ["b", [["n", 8]]], ["c", [["o", 9]]]]] ])", - physical_type, &converter); + physical_type, converter.get()); auto expected = ArrayFromJSON(physical_type, R"([ [1, [[0, 1], [["x", 1], ["y", null]], [["z", 3]], null]], @@ -248,7 +264,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueMap) { AssertArrayEquals(expected, actual); // Verify GetShreddingColumnNames - ASSERT_EQ(std::vector({"nested"}), converter.GetShreddingColumnNames()); + ASSERT_EQ(std::vector({"nested"}), converter->GetShreddingColumnNames()); // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, c overflowed in row3 MapSharedShreddingFieldMeta expected_meta; @@ -257,7 +273,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedValueMap) { expected_meta.overflow_field_set = {2}; expected_meta.num_columns = 2; expected_meta.max_row_width = 3; - ASSERT_EQ(expected_meta, converter.BuildFieldMeta("nested").value()); + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("nested").value()); } TEST_F(MapSharedShreddingBatchConverterTest, NestedComplex) { @@ -271,11 +287,12 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedComplex) { arrow::field("id", arrow::int32()), arrow::field("data", arrow::map(arrow::utf8(), value_type)), }); - std::map field_to_num_columns = {{"data", 2}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, field_to_num_columns)); - MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, - field_to_num_columns, pool_); + auto context = + std::make_shared(std::map{{"data", 2}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"data", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); auto logical_type = arrow::struct_(logical_schema->fields()); auto physical_type = arrow::struct_(physical_schema->fields()); @@ -289,7 +306,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedComplex) { [3, [["a", [30, [null, "t4"], []]], ["b", [null, [], [["q", 5]]]], ["c", [40, ["t5"], [["r", 6]]]]]], [4, null] ])", - physical_type, &converter); + physical_type, converter.get()); auto expected = ArrayFromJSON(physical_type, R"([ [1, [[0, 1], [10, ["t1", "t2"], [["x", 1]]], [20, ["t3"], [["y", 2], ["z", 3]]], null]], @@ -302,7 +319,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedComplex) { AssertArrayEquals(expected, actual); // Verify GetShreddingColumnNames - ASSERT_EQ(std::vector({"data"}), converter.GetShreddingColumnNames()); + ASSERT_EQ(std::vector({"data"}), converter->GetShreddingColumnNames()); // Verify BuildFieldMeta: a=0,b=1,c=2; K=2, max_row_width=3, c overflowed in row2 MapSharedShreddingFieldMeta expected_meta; @@ -311,7 +328,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, NestedComplex) { expected_meta.overflow_field_set = {2}; expected_meta.num_columns = 2; expected_meta.max_row_width = 3; - ASSERT_EQ(expected_meta, converter.BuildFieldMeta("data").value()); + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("data").value()); } TEST_F(MapSharedShreddingBatchConverterTest, MultipleMapFields) { @@ -322,12 +339,13 @@ TEST_F(MapSharedShreddingBatchConverterTest, MultipleMapFields) { arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), arrow::field("attrs", arrow::map(arrow::utf8(), arrow::float64())), }); - std::map field_to_num_columns = {{"tags", 2}, {"attrs", 3}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, field_to_num_columns)); - - MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, - field_to_num_columns, pool_); + auto context = std::make_shared( + std::map{{"tags", 2}, {"attrs", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeCoreOptions({{"tags", "plain"}, {"attrs", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); auto logical_type = arrow::struct_(logical_schema->fields()); auto physical_type = arrow::struct_(physical_schema->fields()); @@ -345,7 +363,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, MultipleMapFields) { [2, [["c", 30], ["a", 40], ["b", 50]], [["z", 3.3]]], [3, null, [["x", 4.4], ["y", 5.5], ["z", 6.6], ["w", 7.7]]] ])", - physical_type, &converter); + physical_type, converter.get()); auto expected = ArrayFromJSON(physical_type, R"([ [1, [[0, 1], 10, 20, null], [[0, 1, -1], 1.1, 2.2, null, null]], @@ -357,7 +375,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, MultipleMapFields) { AssertArrayEquals(expected, actual); // Verify GetShreddingColumnNames returns both columns in order - ASSERT_EQ(std::vector({"tags", "attrs"}), converter.GetShreddingColumnNames()); + ASSERT_EQ(std::vector({"tags", "attrs"}), converter->GetShreddingColumnNames()); // Verify BuildFieldMeta for tags: a=0,b=1,c=2; K=2, max_row_width=3 MapSharedShreddingFieldMeta tags_meta; @@ -366,7 +384,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, MultipleMapFields) { tags_meta.overflow_field_set = {1}; tags_meta.num_columns = 2; tags_meta.max_row_width = 3; - ASSERT_EQ(tags_meta, converter.BuildFieldMeta("tags").value()); + ASSERT_EQ(tags_meta, converter->BuildFieldMeta("tags").value()); // Verify BuildFieldMeta for attrs: x=0,y=1,z=2,w=3; K=3, max_row_width=4 MapSharedShreddingFieldMeta attrs_meta; @@ -375,7 +393,7 @@ TEST_F(MapSharedShreddingBatchConverterTest, MultipleMapFields) { attrs_meta.overflow_field_set = {3}; attrs_meta.num_columns = 3; attrs_meta.max_row_width = 4; - ASSERT_EQ(attrs_meta, converter.BuildFieldMeta("attrs").value()); + ASSERT_EQ(attrs_meta, converter->BuildFieldMeta("attrs").value()); } TEST_F(MapSharedShreddingBatchConverterTest, BuildFieldMetaInvalidFieldName) { @@ -385,20 +403,100 @@ TEST_F(MapSharedShreddingBatchConverterTest, BuildFieldMetaInvalidFieldName) { arrow::field("id", arrow::int32()), arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), }); - std::map field_to_num_columns = {{"tags", 3}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, field_to_num_columns)); - MapSharedShreddingBatchConverter converter(logical_schema, physical_schema, - field_to_num_columns, pool_); + auto context = + std::make_shared(std::map{{"tags", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "plain"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); // Valid case: "tags" exists - ASSERT_OK_AND_ASSIGN([[maybe_unused]] auto meta, converter.BuildFieldMeta("tags")); + ASSERT_OK_AND_ASSIGN([[maybe_unused]] auto meta, converter->BuildFieldMeta("tags")); // Invalid case: "id" is not a shredding field - ASSERT_NOK_WITH_MSG(converter.BuildFieldMeta("id"), "cannot find field_name 'id'"); + ASSERT_NOK_WITH_MSG(converter->BuildFieldMeta("id"), "cannot find field_name 'id'"); // Invalid case: nonexistent field name - ASSERT_NOK_WITH_MSG(converter.BuildFieldMeta("nonexistent"), + ASSERT_NOK_WITH_MSG(converter->BuildFieldMeta("nonexistent"), "cannot find field_name 'nonexistent'"); } + +TEST_F(MapSharedShreddingBatchConverterTest, SequentialPlacementUsesSmallestColumn) { + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto context = + std::make_shared(std::map{{"tags", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "sequential"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + auto actual = RunConvert(logical_type, R"([ + [100, [["a", 1], ["b", 2]]], + [200, [["b", 3], ["c", 4], ["a", 5]]] + ])", + physical_type, converter.get()); + + auto expected = ArrayFromJSON(physical_type, R"([ + [100, [[0, 1, -1], 1, 2, null, null]], + [200, [[0, 1, 2], 5, 3, 4, null]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 3; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value()); +} + +TEST_F(MapSharedShreddingBatchConverterTest, LruPlacementPreservesResidentColumns) { + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto context = + std::make_shared(std::map{{"tags", 3}}); + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"tags", "lru"}})); + ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create( + logical_schema, context, options, pool_)); + auto physical_schema = converter->GetPhysicalSchema(); + + auto logical_type = arrow::struct_(logical_schema->fields()); + auto physical_type = arrow::struct_(physical_schema->fields()); + + auto actual = RunConvert(logical_type, R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]], + [2, [["a", 40], ["b", 50]]], + [3, [["d", 60], ["e", 70], ["f", 80]]], + [4, [["a", 90], ["d", 100], ["e", 110], ["f", 120]]] + ])", + physical_type, converter.get()); + + auto expected = ArrayFromJSON(physical_type, R"([ + [1, [[0, 1, 2], 10, 20, 30, null]], + [2, [[0, 1, -1], 40, 50, null, null]], + [3, [[4, 5, 3], 70, 80, 60, null]], + [4, [[4, 5, 3], 110, 120, 100, [[0, 90]]]] + ])") + .ValueOrDie(); + + AssertArrayEquals(expected, actual); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}, {"e", 4}, {"f", 5}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}, {3, {2}}, {4, {0}}, {5, {1}}}; + expected_meta.overflow_field_set = {0}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 4; + ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value()); +} + } // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp index b0bedfe5..42b7b467 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.cpp @@ -26,27 +26,20 @@ namespace paimon { MapSharedShreddingColumnAllocator::MapSharedShreddingColumnAllocator(int32_t num_columns) : num_columns_(num_columns) {} -RowAllocation MapSharedShreddingColumnAllocator::AllocateRow( - const std::vector& field_ids) { +void MapSharedShreddingColumnAllocator::CommitRow(const RowAllocation& allocation, + const std::vector& field_ids) { max_row_width_ = std::max(max_row_width_, static_cast(field_ids.size())); - RowAllocation result; - result.col_to_field.assign(num_columns_, -1); - int32_t assign_limit = std::min(static_cast(field_ids.size()), num_columns_); - - for (int32_t i = 0; i < assign_limit; ++i) { - int32_t field_id = field_ids[i]; - result.col_to_field[i] = field_id; - field_to_columns_[field_id].insert(i); + for (int32_t col = 0; col < num_columns_; ++col) { + int32_t field_id = allocation.col_to_field[col]; + if (field_id != -1) { + field_to_columns_[field_id].insert(col); + } } - for (int32_t i = assign_limit; i < static_cast(field_ids.size()); ++i) { - int32_t field_id = field_ids[i]; - result.overflow_fields.push_back(field_id); + for (int32_t field_id : allocation.overflow_fields) { overflow_field_set_.insert(field_id); } - - return result; } const std::map>& MapSharedShreddingColumnAllocator::GetFieldToColumns() @@ -62,8 +55,4 @@ int32_t MapSharedShreddingColumnAllocator::GetMaxRowWidth() const { return max_row_width_; } -int32_t MapSharedShreddingColumnAllocator::GetNumColumns() const { - return num_columns_; -} - } // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h index 1e115a77..f9c18bd9 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator.h @@ -19,11 +19,9 @@ #pragma once -#include #include #include #include -#include #include namespace paimon { @@ -38,21 +36,16 @@ struct RowAllocation { std::vector overflow_fields; }; -/// Allocates MAP field ids to K physical columns on a per-row basis, +/// Allocates shared-shredding MAP field ids to K physical columns on a per-row basis, /// and accumulates field-level metadata (field_to_columns, overflow_field_set, max_row_width). -/// -/// This is a trivial implementation: each row simply assigns columns 0..min(N,K)-1 -/// in order, with no LRU eviction. -/// TODO(jinli.zjw): support LRU class MapSharedShreddingColumnAllocator { public: - /// @param num_columns Number of physical columns K for this shared-shredding MAP column. - explicit MapSharedShreddingColumnAllocator(int32_t num_columns); + virtual ~MapSharedShreddingColumnAllocator() = default; /// Allocates physical columns for one row's field ids. - /// @param field_ids The field ids present in this row (order matters for fake impl). + /// @param field_ids Field ids present in this row. /// @return Allocation result with column assignments and overflow list. - RowAllocation AllocateRow(const std::vector& field_ids); + virtual RowAllocation AllocateRow(const std::vector& field_ids) = 0; /// Returns accumulated field_id -> set of column indices (for MapSharedShreddingFileMeta). const std::map>& GetFieldToColumns() const; @@ -63,12 +56,18 @@ class MapSharedShreddingColumnAllocator { /// Returns the maximum row width observed so far. int32_t GetMaxRowWidth() const; - /// Returns the number of physical columns K. - int32_t GetNumColumns() const; + protected: + /// @param num_columns Number of physical columns K for this shared-shredding MAP column. + explicit MapSharedShreddingColumnAllocator(int32_t num_columns); + + /// Commits a planned row allocation and updates accumulated metadata. + /// @param allocation Allocation materialized for the current row. + /// @param field_ids Field ids after allocator-specific preparation. + void CommitRow(const RowAllocation& allocation, const std::vector& field_ids); - private: int32_t num_columns_; + private: // ---- Accumulated field-level metadata ---- std::map> field_to_columns_; std::set overflow_field_set_; diff --git a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_column_allocator_test.cpp deleted file mode 100644 index b83c56e3..00000000 --- a/src/paimon/common/data/shredding/map_shared_shredding_column_allocator_test.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" - -#include "gtest/gtest.h" - -namespace paimon { - -TEST(MapSharedShreddingColumnAllocatorTest, BasicAllocation) { - MapSharedShreddingColumnAllocator allocator(3); - - // 2 fields, K=3 -> all fit, no overflow - auto result = allocator.AllocateRow({10, 20}); - ASSERT_EQ(std::vector({10, 20, -1}), result.col_to_field); - ASSERT_TRUE(result.overflow_fields.empty()); -} - -TEST(MapSharedShreddingColumnAllocatorTest, ExactlyKFields) { - MapSharedShreddingColumnAllocator allocator(3); - - auto result = allocator.AllocateRow({0, 1, 2}); - ASSERT_EQ(std::vector({0, 1, 2}), result.col_to_field); - ASSERT_TRUE(result.overflow_fields.empty()); -} - -TEST(MapSharedShreddingColumnAllocatorTest, OverflowWhenExceedK) { - MapSharedShreddingColumnAllocator allocator(2); - - // 4 fields, K=2 -> first 2 assigned, last 2 overflow - auto result = allocator.AllocateRow({10, 20, 30, 40}); - ASSERT_EQ(std::vector({10, 20}), result.col_to_field); - ASSERT_EQ(std::vector({30, 40}), result.overflow_fields); -} - -TEST(MapSharedShreddingColumnAllocatorTest, EmptyRow) { - MapSharedShreddingColumnAllocator allocator(3); - - auto result = allocator.AllocateRow({}); - ASSERT_EQ(std::vector({-1, -1, -1}), result.col_to_field); - ASSERT_TRUE(result.overflow_fields.empty()); -} - -TEST(MapSharedShreddingColumnAllocatorTest, MaxRowWidthTracked) { - MapSharedShreddingColumnAllocator allocator(3); - - allocator.AllocateRow({1, 2}); - ASSERT_EQ(2, allocator.GetMaxRowWidth()); - - allocator.AllocateRow({1, 2, 3, 4, 5}); - ASSERT_EQ(5, allocator.GetMaxRowWidth()); - - allocator.AllocateRow({1}); - ASSERT_EQ(5, allocator.GetMaxRowWidth()); -} - -TEST(MapSharedShreddingColumnAllocatorTest, FieldToColumnsAccumulated) { - MapSharedShreddingColumnAllocator allocator(3); - - allocator.AllocateRow({10, 20, 30}); - allocator.AllocateRow({20, 40}); - - auto field_to_cols = allocator.GetFieldToColumns(); - // field 10 -> {0} - ASSERT_EQ(std::set({0}), field_to_cols.at(10)); - // field 20 -> {1, 0} (col 1 in row 0, col 0 in row 1) - ASSERT_EQ(std::set({0, 1}), field_to_cols.at(20)); - // field 30 -> {2} - ASSERT_EQ(std::set({2}), field_to_cols.at(30)); - // field 40 -> {1} - ASSERT_EQ(std::set({1}), field_to_cols.at(40)); -} - -TEST(MapSharedShreddingColumnAllocatorTest, OverflowFieldSetAccumulated) { - MapSharedShreddingColumnAllocator allocator(2); - - allocator.AllocateRow({1, 2, 3}); // 3 overflows - allocator.AllocateRow({4, 5, 6, 7}); // 6, 7 overflow - - auto overflow_set = allocator.GetOverflowFieldSet(); - ASSERT_EQ(std::set({3, 6, 7}), overflow_set); -} - -TEST(MapSharedShreddingColumnAllocatorTest, GetNumColumns) { - MapSharedShreddingColumnAllocator allocator(5); - ASSERT_EQ(5, allocator.GetNumColumns()); -} - -TEST(MapSharedShreddingColumnAllocatorTest, SingleColumnAllocator) { - MapSharedShreddingColumnAllocator allocator(1); - - auto result = allocator.AllocateRow({10, 20, 30}); - ASSERT_EQ(std::vector({10}), result.col_to_field); - ASSERT_EQ(std::vector({20, 30}), result.overflow_fields); -} - -} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp index 7169ffa8..2f0b3269 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp @@ -73,8 +73,8 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test { std::shared_ptr PhysicalSchemaWithMetadata( const MapSharedShreddingFieldMeta& meta) const { std::map field_to_num_columns = {{"tags", 2}}; - EXPECT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema_, field_to_num_columns)); + auto physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema_, field_to_num_columns); auto metadata = std::make_shared(); EXPECT_OK(MapSharedShreddingUtils::SerializeMetadata( meta, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get())); @@ -240,6 +240,7 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test { {Options::MANIFEST_FORMAT, "mock_format"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "2"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }; }; @@ -460,8 +461,8 @@ TEST_F(MapSharedShreddingFileReaderTest, TestListValue) { meta.max_row_width = 3; std::map field_to_num_columns = {{"tags", 2}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, field_to_num_columns)); + auto physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, field_to_num_columns); auto metadata = std::make_shared(); ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata( meta, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get())); diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp index 518ed32d..5a04f342 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp @@ -128,7 +128,7 @@ std::shared_ptr MapSharedShreddingUtils::InnerBuildSpecificPhys return arrow::struct_(std::move(struct_fields)); } -Result> MapSharedShreddingUtils::LogicalToPhysicalSchema( +std::shared_ptr MapSharedShreddingUtils::LogicalToPhysicalSchema( const std::shared_ptr& logical_schema, const std::map& field_to_num_columns) { arrow::FieldVector physical_fields; diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.h b/src/paimon/common/data/shredding/map_shared_shredding_utils.h index a017c8dd..eef19098 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.h @@ -49,13 +49,6 @@ class MapSharedShreddingUtils { MapSharedShreddingUtils() = delete; ~MapSharedShreddingUtils() = delete; - /// Returns the physical column indices for the given field name from the shredding meta. - /// @param meta The shredding field meta parsed from file footer. - /// @param name The field name to look up. - /// @return Vector of physical column indices assigned to this field, - /// or Status::Invalid if the field name or field id is not found. - static Result> GetPhysicalColumnIndices( - const MapSharedShreddingFieldMeta& meta, const std::string& name); // ---- Column detection ---- /// Checks whether a given arrow field is MAP (the type prerequisite for shredding). @@ -63,15 +56,6 @@ class MapSharedShreddingUtils { /// @return true if the type is MAP. static bool IsShreddingKeyMap(const std::shared_ptr& arrow_type); - /// Finds all shredding MAP field names in a schema by checking per-column config - /// via CoreOptions. - /// @param schema The logical Arrow schema. - /// @param options CoreOptions containing per-column configuration. - /// @return Vector of field names whose map.storage-layout is "shared-shredding", or error - /// if validation fails. - static Result> DetectShreddingColumns( - const std::shared_ptr& schema, const CoreOptions& options); - /// Creates a MapSharedShreddingContext for the given schema and options. /// Returns nullptr if no shredding MAP columns are detected. /// @param schema The logical Arrow schema. @@ -87,7 +71,7 @@ class MapSharedShreddingUtils { /// @param field_to_num_columns Map from field name to its physical column count K. /// Each shredding column can have its own width. /// @return The physical schema for file writing. - static Result> LogicalToPhysicalSchema( + static std::shared_ptr LogicalToPhysicalSchema( const std::shared_ptr& logical_schema, const std::map& field_to_num_columns); @@ -100,23 +84,8 @@ class MapSharedShreddingUtils { const std::shared_ptr& value_type, const std::set& physical_col_ids, bool value_nullable, bool include_overflow); - /// Builds field_to_num_columns map from DetectShreddingColumns result and CoreOptions. - /// @param shredding_field_names Field names returned by DetectShreddingColumns. - /// @param options CoreOptions containing per-column max-columns config. - /// @return Map from field name to K (max physical columns for that field). - static Result> BuildColumnToNumColumns( - const std::vector& shredding_field_names, const CoreOptions& options); - // ---- Metadata serialization ---- - /// Serializes shredding metadata and appends entries to an existing KeyValueMetadata. - /// @param field_meta The field-level shredding metadata to serialize. - /// @param compression Compression codec name for field_dict compression. - /// @param[out] metadata The KeyValueMetadata to append entries to. - static Status SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta, - const std::string& compression, - arrow::KeyValueMetadata* metadata); - /// Deserializes shredding metadata from file footer KeyValueMetadata (per field). /// @param metadata The KeyValueMetadata from file footer. /// @param compression Compression codec name. @@ -148,6 +117,39 @@ class MapSharedShreddingUtils { const std::shared_ptr& physical_schema); private: + /// Returns the physical column indices for the given field name from the shredding meta. + /// @param meta The shredding field meta parsed from file footer. + /// @param name The field name to look up. + /// @return Vector of physical column indices assigned to this field, + /// or Status::Invalid if the field name or field id is not found. + static Result> GetPhysicalColumnIndices( + const MapSharedShreddingFieldMeta& meta, const std::string& name); + + /// Finds all shredding MAP field names in a schema by checking per-column config + /// via CoreOptions. + /// @param schema The logical Arrow schema. + /// @param options CoreOptions containing per-column configuration. + /// @return Vector of field names whose map.storage-layout is "shared-shredding", or error + /// if validation fails. + static Result> DetectShreddingColumns( + const std::shared_ptr& schema, const CoreOptions& options); + + /// Builds shared-shredding max column counts from DetectShreddingColumns result and + /// CoreOptions. + /// @param shredding_field_names Field names returned by DetectShreddingColumns. + /// @param options CoreOptions containing per-column shared-shredding config. + /// @return Map from field name to its configured maximum physical width. + static Result> BuildColumnToNumColumns( + const std::vector& shredding_field_names, const CoreOptions& options); + + /// Serializes shredding metadata and appends entries to an existing KeyValueMetadata. + /// @param field_meta The field-level shredding metadata to serialize. + /// @param compression Compression codec name for field_dict compression. + /// @param[out] metadata The KeyValueMetadata to append entries to. + static Status SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta, + const std::string& compression, + arrow::KeyValueMetadata* metadata); + /// Builds the physical Arrow type for one shredding MAP column. /// @param value_type The value type of the original MAP. /// @param num_columns Number of physical columns K. diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp index 79a21fdf..a7dddfa4 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp @@ -91,8 +91,8 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaBasic) { }); std::map field_to_num_columns = {{"tags", 4}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - schema, field_to_num_columns)); + auto physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, field_to_num_columns); // Build expected schema for comparison auto expected_struct = arrow::struct_({ @@ -119,8 +119,8 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNestedValue) { auto schema = arrow::schema({arrow::field("data", map_type)}); std::map field_to_num_columns = {{"data", 2}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - schema, field_to_num_columns)); + auto physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, field_to_num_columns); auto expected_struct = arrow::struct_({ arrow::field("__field_mapping", arrow::list(arrow::int32()), true), @@ -138,8 +138,7 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNullable) { auto schema_nullable = arrow::schema({arrow::field("m", nullable_map)}); std::map col_map = {{"m", 2}}; - ASSERT_OK_AND_ASSIGN( - auto physical, MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_nullable, col_map)); + auto physical = MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_nullable, col_map); auto struct_type = physical->field(0)->type(); ASSERT_TRUE(struct_type->field(0)->nullable()); ASSERT_TRUE(struct_type->field(1)->nullable()); @@ -149,8 +148,7 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNullable) { auto non_nullable_map = arrow::map(arrow::utf8(), arrow::field("item", arrow::int64(), false)); auto schema_non_nullable = arrow::schema({arrow::field("m", non_nullable_map)}); - ASSERT_OK_AND_ASSIGN(auto physical2, MapSharedShreddingUtils::LogicalToPhysicalSchema( - schema_non_nullable, col_map)); + auto physical2 = MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_non_nullable, col_map); auto struct_type2 = physical2->field(0)->type(); ASSERT_FALSE(struct_type2->field(1)->nullable()); ASSERT_FALSE(struct_type2->field(2)->nullable()); @@ -164,8 +162,7 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaPreservesFieldMetadata) auto schema = arrow::schema({arrow::field("m", map_type, false, metadata)}); std::map col_map = {{"m", 2}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, - MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, col_map)); + auto physical_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, col_map); ASSERT_FALSE(physical_schema->field(0)->nullable()); ASSERT_TRUE(physical_schema->field(0)->metadata()->Equals(*metadata)); @@ -178,8 +175,7 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNoShreddingColumns) { }); std::map empty_map; - ASSERT_OK_AND_ASSIGN(auto physical_schema, - MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, empty_map)); + auto physical_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, empty_map); ASSERT_TRUE(physical_schema->Equals(schema)); } @@ -218,12 +214,6 @@ TEST(MapSharedShreddingUtilsTest, BuildSpecificPhysicalStructTypeWithoutOverflow // ---- BuildColumnToNumColumns ---- TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumns) { - auto schema = arrow::schema({ - arrow::field("id", arrow::int32()), - arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), - arrow::field("metrics", arrow::map(arrow::utf8(), arrow::float64())), - }); - ASSERT_OK_AND_ASSIGN( CoreOptions options, CoreOptions::FromMap({{"fields.tags.map.shared-shredding.max-columns", "128"}, @@ -239,10 +229,6 @@ TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumns) { } TEST(MapSharedShreddingUtilsTest, BuildColumnToNumColumnsDefault) { - auto schema = arrow::schema({ - arrow::field("tags", arrow::map(arrow::utf8(), arrow::utf8())), - }); - // No explicit max-columns config -> default 256 ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); std::vector shredding_field_names = {"tags"}; diff --git a/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h b/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h new file mode 100644 index 00000000..d8057442 --- /dev/null +++ b/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" + +namespace paimon { + +/// Allocator that keeps the input field order and maps it to columns 0..K-1. +class PlainMapSharedShreddingColumnAllocator : public MapSharedShreddingColumnAllocator { + public: + explicit PlainMapSharedShreddingColumnAllocator(int32_t num_columns) + : MapSharedShreddingColumnAllocator(num_columns) {} + + RowAllocation AllocateRow(const std::vector& field_ids) override { + RowAllocation allocation = AllocateLeadingColumns(field_ids); + CommitRow(allocation, field_ids); + return allocation; + } + + protected: + RowAllocation AllocateLeadingColumns(const std::vector& field_ids) const { + RowAllocation allocation; + allocation.col_to_field.assign(num_columns_, -1); + for (size_t i = 0; i < field_ids.size(); ++i) { + int32_t field_id = field_ids[i]; + if (i < static_cast(num_columns_)) { + allocation.col_to_field[static_cast(i)] = field_id; + } else { + allocation.overflow_fields.push_back(field_id); + } + } + return allocation; + } +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp b/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp new file mode 100644 index 00000000..e7197392 --- /dev/null +++ b/src/paimon/common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { +namespace { + +void ExpectAllocation(const RowAllocation& allocation, const std::vector& col_to_field, + const std::vector& overflow_fields) { + ASSERT_EQ(col_to_field, allocation.col_to_field); + ASSERT_EQ(overflow_fields, allocation.overflow_fields); +} + +} // namespace + +TEST(PlainMapSharedShreddingColumnAllocatorTest, BasicAllocation) { + PlainMapSharedShreddingColumnAllocator allocator(3); + // 2 fields, K=3 -> all fit, no overflow + RowAllocation result = allocator.AllocateRow({10, 20}); + ExpectAllocation(result, {10, 20, -1}, {}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, ExactlyKFields) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation result = allocator.AllocateRow({0, 1, 2}); + ExpectAllocation(result, {0, 1, 2}, {}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, OverflowWhenExceedK) { + PlainMapSharedShreddingColumnAllocator allocator(2); + + RowAllocation result = allocator.AllocateRow({10, 20, 30, 40}); + ExpectAllocation(result, {10, 20}, {30, 40}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, EmptyRow) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation result = allocator.AllocateRow({}); + ExpectAllocation(result, {-1, -1, -1}, {}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, MaxRowWidthTracked) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + allocator.AllocateRow({1, 2}); + ASSERT_EQ(2, allocator.GetMaxRowWidth()); + + allocator.AllocateRow({1, 2, 3, 4, 5}); + ASSERT_EQ(5, allocator.GetMaxRowWidth()); + + allocator.AllocateRow({1}); + ASSERT_EQ(5, allocator.GetMaxRowWidth()); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, FieldToColumnsAccumulated) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + allocator.AllocateRow({10, 20, 30}); + allocator.AllocateRow({20, 40}); + + const auto& field_to_cols = allocator.GetFieldToColumns(); + // field 10 -> {0} + ASSERT_EQ(std::set({0}), field_to_cols.at(10)); + // field 20 -> {1, 0} (col 1 in row 0, col 0 in row 1) + ASSERT_EQ(std::set({0, 1}), field_to_cols.at(20)); + // field 30 -> {2} + ASSERT_EQ(std::set({2}), field_to_cols.at(30)); + // field 40 -> {1} + ASSERT_EQ(std::set({1}), field_to_cols.at(40)); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, OverflowFieldSetAccumulated) { + PlainMapSharedShreddingColumnAllocator allocator(2); + + allocator.AllocateRow({1, 2, 3}); // 3 overflows + allocator.AllocateRow({4, 5, 6, 7}); // 6, 7 overflow + + ASSERT_EQ((std::set{3, 6, 7}), allocator.GetOverflowFieldSet()); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, SingleColumnAllocator) { + PlainMapSharedShreddingColumnAllocator allocator(1); + + RowAllocation result = allocator.AllocateRow({10, 20, 30}); + ExpectAllocation(result, {10}, {20, 30}); +} + +TEST(PlainMapSharedShreddingColumnAllocatorTest, UsesInputOrder) { + PlainMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation row0 = allocator.AllocateRow({2, 0, 1}); + ExpectAllocation(row0, {2, 0, 1}, {}); + + RowAllocation row1 = allocator.AllocateRow({4, 3, 5, 6}); + ExpectAllocation(row1, {4, 3, 5}, {6}); + + const auto& field_to_columns = allocator.GetFieldToColumns(); + ASSERT_EQ((std::set{1}), field_to_columns.at(0)); + ASSERT_EQ((std::set{2}), field_to_columns.at(1)); + ASSERT_EQ((std::set{0}), field_to_columns.at(2)); + ASSERT_EQ((std::set{1}), field_to_columns.at(3)); + ASSERT_EQ((std::set{0}), field_to_columns.at(4)); + ASSERT_EQ((std::set{2}), field_to_columns.at(5)); + ASSERT_EQ((std::set{6}), allocator.GetOverflowFieldSet()); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h b/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h new file mode 100644 index 00000000..e26feede --- /dev/null +++ b/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/shredding/plain_map_shared_shredding_column_allocator.h" + +namespace paimon { + +/// Allocator that sorts input fields and maps them to columns 0..K-1. +class SequentialMapSharedShreddingColumnAllocator : public PlainMapSharedShreddingColumnAllocator { + public: + explicit SequentialMapSharedShreddingColumnAllocator(int32_t num_columns) + : PlainMapSharedShreddingColumnAllocator(num_columns) {} + + RowAllocation AllocateRow(const std::vector& field_ids) override { + std::vector sorted_field_ids = field_ids; + std::sort(sorted_field_ids.begin(), sorted_field_ids.end()); + RowAllocation allocation = AllocateLeadingColumns(sorted_field_ids); + CommitRow(allocation, sorted_field_ids); + return allocation; + } +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp b/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp new file mode 100644 index 00000000..f68ef0d7 --- /dev/null +++ b/src/paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator_test.cpp @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/sequential_map_shared_shredding_column_allocator.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { +namespace { + +void ExpectAllocation(const RowAllocation& allocation, const std::vector& col_to_field, + const std::vector& overflow_fields) { + ASSERT_EQ(col_to_field, allocation.col_to_field); + ASSERT_EQ(overflow_fields, allocation.overflow_fields); +} + +} // namespace + +TEST(SequentialMapSharedShreddingColumnAllocatorTest, SortsAndUsesLeadingColumns) { + SequentialMapSharedShreddingColumnAllocator allocator(3); + + RowAllocation row0 = allocator.AllocateRow({1, 2}); + ExpectAllocation(row0, {1, 2, -1}, {}); + + RowAllocation row1 = allocator.AllocateRow({2, 3}); + ExpectAllocation(row1, {2, 3, -1}, {}); + + RowAllocation row2 = allocator.AllocateRow({7, 4, 6, 5}); + ExpectAllocation(row2, {4, 5, 6}, {7}); + + const auto& field_to_columns = allocator.GetFieldToColumns(); + ASSERT_EQ((std::set{0}), field_to_columns.at(1)); + ASSERT_EQ((std::set{0, 1}), field_to_columns.at(2)); + ASSERT_EQ((std::set{1}), field_to_columns.at(3)); + ASSERT_EQ((std::set{0}), field_to_columns.at(4)); + ASSERT_EQ((std::set{1}), field_to_columns.at(5)); + ASSERT_EQ((std::set{2}), field_to_columns.at(6)); + ASSERT_EQ((std::set{7}), allocator.GetOverflowFieldSet()); +} + +} // namespace paimon::test diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 70339ecc..b361497b 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -96,6 +96,8 @@ const char Options::DATA_EVOLUTION_ENABLED[] = "data-evolution.enabled"; const char Options::PARTITION_GENERATE_LEGACY_NAME[] = "partition.legacy-name"; const char Options::MAP_STORAGE_LAYOUT[] = "map.storage-layout"; const char Options::MAP_SHARED_SHREDDING_MAX_COLUMNS[] = "map.shared-shredding.max-columns"; +const char Options::MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY[] = + "map.shared-shredding.column-placement-policy"; const char Options::BLOB_AS_DESCRIPTOR[] = "blob-as-descriptor"; const char Options::BLOB_FIELD[] = "blob-field"; const char Options::BLOB_DESCRIPTOR_FIELD[] = "blob-descriptor-field"; diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp index 1e574e5b..be53042e 100644 --- a/src/paimon/core/append/append_only_writer_test.cpp +++ b/src/paimon/core/append/append_only_writer_test.cpp @@ -896,6 +896,7 @@ TEST_F(AppendOnlyWriterTest, TestSharedShreddingMapRejectsAvroFormatOnCommit) { {Options::MANIFEST_FORMAT, "avro"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -931,6 +932,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) { {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -971,9 +973,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) { // Check shared-shredding map metadata: a=0, b=1, c=2; K=3, max_row_width=3, no overflow. std::map column_to_k = {{"tags", 3}}; - ASSERT_OK_AND_ASSIGN( - auto expected_physical_schema, - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); + auto expected_physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k); MapSharedShreddingFieldMeta expected_meta; expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; @@ -1003,6 +1004,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllEmptyFirstFile) { {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -1034,8 +1036,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllEmptyFirstFile) { path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map first_file_k = {{"tags", 3}}; - ASSERT_OK_AND_ASSIGN(auto first_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, first_file_k)); + auto first_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, first_file_k); MapSharedShreddingFieldMeta empty_meta; empty_meta.num_columns = 3; empty_meta.max_row_width = 0; @@ -1060,6 +1062,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -1089,8 +1092,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF path_factory->ToPath(null_inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map first_file_k = {{"tags", 3}}; - ASSERT_OK_AND_ASSIGN(auto first_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, first_file_k)); + auto first_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, first_file_k); MapSharedShreddingFieldMeta empty_meta; empty_meta.num_columns = 3; empty_meta.max_row_width = 0; @@ -1121,8 +1124,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF // Previous file observed max_row_width=0, but the next file must still keep at least one // physical value column so shared-shredding never produces a K=0 schema. std::map second_file_k = {{"tags", 1}}; - ASSERT_OK_AND_ASSIGN(auto second_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, second_file_k)); + auto second_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, second_file_k); empty_meta.num_columns = 1; CheckShreddingFileSchema(empty_file_path, format, second_schema, /*field_index=*/1, empty_meta, options.GetFileCompression()); @@ -1179,6 +1182,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithOverflow) { {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "2"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -1214,9 +1218,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithOverflow) { path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k = {{"tags", 2}}; - ASSERT_OK_AND_ASSIGN( - auto expected_physical_schema, - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); + auto expected_physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k); std::string compression = options.GetFileCompression(); // Verify metadata: a=0,b=1,c=2,d=3,e=4,f=5; K=2, max_row_width=4 @@ -1243,6 +1246,72 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithOverflow) { CheckFileContent(data_file_path, format, expected_array); } +TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithLruPlacement) { + std::string format = GetFormat(); + auto options = CreateOptions({ + {Options::FILE_FORMAT, format}, + {Options::MANIFEST_FORMAT, format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "lru"}, + {Options::WRITE_ONLY, "true"}, + }); + + auto logical_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }); + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), format, options); + + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, logical_schema, + /*write_cols=*/std::nullopt, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + auto batch = CreateBatch(logical_schema, R"([ + [1, [["a", 10], ["b", 20], ["c", 30]]], + [2, [["a", 40], ["b", 50]]], + [3, [["d", 60]]], + [4, [["a", 70], ["b", 80], ["c", 90], ["d", 100]]] + ])"); + ASSERT_OK(writer->Write(std::move(batch))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + ASSERT_EQ(1, inc.GetNewFilesIncrement().NewFiles().size()); + std::string data_file_path = + path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); + + std::map column_to_k = {{"tags", 3}}; + auto expected_physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k); + + MapSharedShreddingFieldMeta expected_meta; + expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}}; + expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}, {3, {2}}}; + expected_meta.overflow_field_set = {2}; + expected_meta.num_columns = 3; + expected_meta.max_row_width = 4; + CheckShreddingFileSchema(data_file_path, format, expected_physical_schema, /*field_index=*/1, + expected_meta, options.GetFileCompression()); + + auto physical_type = arrow::struct_(expected_physical_schema->fields()); + std::shared_ptr expected_array; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([ + [1, [[0, 1, 2], 10, 20, 30, null]], + [2, [[0, 1, -1], 40, 50, null, null]], + [3, [[-1, -1, 3], null, null, 60, null]], + [4, [[0, 1, 3], 70, 80, 100, [[2, 90]]]] + ])"}, + &expected_array) + .ok()); + CheckFileContent(data_file_path, format, expected_array); +} + TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFiles) { std::string format = GetFormat(); auto options = CreateOptions({ @@ -1250,6 +1319,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -1282,8 +1352,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil // File 1 should have K=10 (first file uses K_max). std::map column_to_k_file1 = {{"tags", 10}}; - ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, column_to_k_file1)); + auto phys_schema1 = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k_file1); // Verify file1 physical schema has 10 columns. auto struct_type1 = std::static_pointer_cast(phys_schema1->field(1)->type()); ASSERT_EQ(12, struct_type1->num_fields()); // mapping + 10 cols + overflow @@ -1311,8 +1381,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil // File 2 should have K=3 (adapted from file1's max_row_width=3). std::map column_to_k_file2 = {{"tags", 3}}; - ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, column_to_k_file2)); + auto phys_schema2 = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k_file2); auto struct_type2 = std::static_pointer_cast(phys_schema2->field(1)->type()); ASSERT_EQ(5, struct_type2->num_fields()); // mapping + 3 cols + overflow @@ -1350,8 +1420,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil // File 3 should have K=5 (window max grew from file2's max_row_width=5). std::map column_to_k_file3 = {{"tags", 5}}; - ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, column_to_k_file3)); + auto phys_schema3 = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k_file3); auto struct_type3 = std::static_pointer_cast(phys_schema3->field(1)->type()); ASSERT_EQ(7, struct_type3->num_fields()); // mapping + 5 cols + overflow @@ -1383,6 +1453,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapUsesInitialContextFo {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -1413,8 +1484,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapUsesInitialContextFo path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k = {{"tags", 2}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, column_to_k)); + auto physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k); MapSharedShreddingFieldMeta expected_meta; expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; expected_meta.field_to_columns = {{0, {0}}, {1, {1}}}; @@ -1444,8 +1515,10 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "8"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {"fields.attrs.map.storage-layout", "shared-shredding"}, {"fields.attrs.map.shared-shredding.max-columns", "4"}, + {"fields.attrs.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -1481,8 +1554,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA // Verify file1: tags K=8, attrs K=4 (first file uses K_max). std::map col_to_k_file1 = {{"tags", 8}, {"attrs", 4}}; - ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, col_to_k_file1)); + auto phys_schema1 = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, col_to_k_file1); MapSharedShreddingFieldMeta meta1_tags; meta1_tags.name_to_id = {{"a", 0}, {"b", 1}}; @@ -1513,8 +1586,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA path_factory->ToPath(inc2.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map col_to_k_file2 = {{"tags", 2}, {"attrs", 1}}; - ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, col_to_k_file2)); + auto phys_schema2 = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, col_to_k_file2); MapSharedShreddingFieldMeta meta2_tags; meta2_tags.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}}; @@ -1547,8 +1620,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA path_factory->ToPath(inc3.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map col_to_k_file3 = {{"tags", 3}, {"attrs", 3}}; - ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, col_to_k_file3)); + auto phys_schema3 = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, col_to_k_file3); MapSharedShreddingFieldMeta meta3_tags; meta3_tags.name_to_id = {{"f", 0}, {"g", 1}}; @@ -1577,6 +1650,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapDataFileMetaInfo) { {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -1629,8 +1703,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapDataFileMetaInfo) { // Verify the written file has correct shared-shredding map content. std::string file_path = path_factory->ToPath(actual_meta->file_name); std::map col_to_k = {{"tags", 3}}; - ASSERT_OK_AND_ASSIGN(auto phys_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, col_to_k)); + auto phys_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, col_to_k); auto physical_type = arrow::struct_(phys_schema->fields()); std::shared_ptr expected_array; @@ -1656,6 +1729,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapWithBlobSeparation) {Options::MANIFEST_FORMAT, format}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, }); @@ -1716,9 +1790,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapWithBlobSeparation) arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), }); std::map col_to_k = {{"tags", 3}}; - ASSERT_OK_AND_ASSIGN( - auto expected_physical_schema, - MapSharedShreddingUtils::LogicalToPhysicalSchema(main_logical_schema, col_to_k)); + auto expected_physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(main_logical_schema, col_to_k); MapSharedShreddingFieldMeta expected_meta; expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 736506d8..7ea5edd4 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -1215,6 +1215,24 @@ Result CoreOptions::GetMapSharedShreddingMaxColumns(const std::string& return max_columns; } +Result +CoreOptions::GetMapSharedShreddingColumnPlacementPolicy(const std::string& field_name) const { + std::string key = std::string(Options::FIELDS_PREFIX) + "." + field_name + "." + + std::string(Options::MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY); + PAIMON_ASSIGN_OR_RAISE(std::string policy_str, OptionsUtils::GetValueFromMap( + impl_->raw_options, key, "lru")); + std::string lower = StringUtils::ToLowerCase(policy_str); + if (lower == "plain") { + return MapSharedShreddingColumnPlacementPolicy::PLAIN; + } else if (lower == "sequential") { + return MapSharedShreddingColumnPlacementPolicy::SEQUENTIAL; + } else if (lower == "lru") { + return MapSharedShreddingColumnPlacementPolicy::LRU; + } + return Status::Invalid( + fmt::format("invalid map.shared-shredding.column-placement-policy: {}", policy_str)); +} + bool CoreOptions::DeletionVectorsEnabled() const { return impl_->deletion_vectors_enabled; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index 12969e49..fb41957a 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -32,6 +32,7 @@ #include "paimon/core/options/external_path_strategy.h" #include "paimon/core/options/lookup_compact_mode.h" #include "paimon/core/options/lookup_strategy.h" +#include "paimon/core/options/map_shared_shredding_column_placement_policy.h" #include "paimon/core/options/map_storage_layout.h" #include "paimon/core/options/merge_engine.h" #include "paimon/core/options/sort_engine.h" @@ -125,6 +126,8 @@ class PAIMON_EXPORT CoreOptions { Result GetMapStorageLayout(const std::string& field_name) const; Result GetMapSharedShreddingMaxColumns(const std::string& field_name) const; + Result GetMapSharedShreddingColumnPlacementPolicy( + const std::string& field_name) const; bool DeletionVectorsEnabled() const; bool DeletionVectorsBitmap64() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 402b6b48..acf8df3d 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -93,6 +93,8 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_FALSE(core_options.FieldCollectAggDistinct("f1").value()); ASSERT_EQ(MapStorageLayout::DEFAULT, core_options.GetMapStorageLayout("any_col").value()); ASSERT_EQ(256, core_options.GetMapSharedShreddingMaxColumns("any_col").value()); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + core_options.GetMapSharedShreddingColumnPlacementPolicy("any_col").value()); ASSERT_FALSE(core_options.DeletionVectorsEnabled()); ASSERT_FALSE(core_options.DeletionVectorsBitmap64()); ASSERT_EQ(2 * 1024 * 1024, core_options.DeletionVectorTargetFileSize()); @@ -269,7 +271,8 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}, {Options::BUCKET_FUNCTION_TYPE, "mod"}, {"fields.metrics.map.storage-layout", "shared-shredding"}, - {"fields.metrics.map.shared-shredding.max-columns", "128"}}; + {"fields.metrics.map.shared-shredding.max-columns", "128"}, + {"fields.metrics.map.shared-shredding.column-placement-policy", "lru"}}; ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap(options)); auto fs = core_options.GetFileSystem(); @@ -415,6 +418,8 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(MapStorageLayout::SHARED_SHREDDING, core_options.GetMapStorageLayout("metrics").value()); ASSERT_EQ(128, core_options.GetMapSharedShreddingMaxColumns("metrics").value()); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + core_options.GetMapSharedShreddingColumnPlacementPolicy("metrics").value()); } TEST(CoreOptionsTest, TestInvalidCase) { @@ -932,10 +937,14 @@ TEST(CoreOptionsTest, TestMapStorageLayout) { ASSERT_EQ(MapStorageLayout::SHARED_SHREDDING, options.GetMapStorageLayout("ext_map").value()); ASSERT_EQ(64, options.GetMapSharedShreddingMaxColumns("ext_map").value()); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + options.GetMapSharedShreddingColumnPlacementPolicy("ext_map").value()); ASSERT_EQ(MapStorageLayout::DEFAULT, options.GetMapStorageLayout("normal_map").value()); // Unconfigured column falls back to default ASSERT_EQ(MapStorageLayout::DEFAULT, options.GetMapStorageLayout("other").value()); ASSERT_EQ(256, options.GetMapSharedShreddingMaxColumns("other").value()); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + options.GetMapSharedShreddingColumnPlacementPolicy("other").value()); } // Test case-insensitive layout value { @@ -967,6 +976,39 @@ TEST(CoreOptionsTest, TestMapStorageLayout) { ASSERT_NOK_WITH_MSG(options.GetMapSharedShreddingMaxColumns("col"), "options map.shared-shredding.max-columns must > 0"); } + // Test placement policy values + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap( + {{"fields.col.map.shared-shredding.column-placement-policy", "PLAIN"}})); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::PLAIN, + options.GetMapSharedShreddingColumnPlacementPolicy("col").value()); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap( + {{"fields.col.map.shared-shredding.column-placement-policy", "sequential"}})); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::SEQUENTIAL, + options.GetMapSharedShreddingColumnPlacementPolicy("col").value()); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap( + {{"fields.col.map.shared-shredding.column-placement-policy", "LRU"}})); + ASSERT_EQ(MapSharedShreddingColumnPlacementPolicy::LRU, + options.GetMapSharedShreddingColumnPlacementPolicy("col").value()); + } + { + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap( + {{"fields.col.map.shared-shredding.column-placement-policy", "invalid"}})); + ASSERT_NOK_WITH_MSG(options.GetMapSharedShreddingColumnPlacementPolicy("col"), + "invalid map.shared-shredding.column-placement-policy: invalid"); + } } } // namespace paimon::test diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp index d319e08f..ceb9ca23 100644 --- a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp +++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp @@ -47,15 +47,13 @@ ShreddingAppendDataFileWriterFactory::ShreddingAppendDataFileWriterFactory( Result>>> ShreddingAppendDataFileWriterFactory::CreateWriter() const { - PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, - MapSharedShreddingBatchConverter::CreateConverter( - write_schema_, shredding_context_, pool_)); - if (!bundle.converter || !bundle.physical_schema) { - return Status::Invalid( - "Shared-shredding append writer requires a converter and physical schema."); + if (!shredding_context_) { + return Status::Invalid("Shared-shredding append writer requires a shredding context."); } - std::shared_ptr file_schema = bundle.physical_schema; - auto converter = bundle.converter; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, + MapSharedShreddingBatchConverter::Create( + write_schema_, shredding_context_, options_, pool_)); + std::shared_ptr file_schema = converter->GetPhysicalSchema(); std::function batch_converter = [converter](::ArrowArray* input, ::ArrowArray* output) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowArray> physical, converter->Convert(input)); @@ -72,7 +70,7 @@ ShreddingAppendDataFileWriterFactory::CreateWriter() const { PAIMON_RETURN_NOT_OK( writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( - bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, shredding_context_, + converter, MapSharedShreddingDefine::kDefaultDictCompression, shredding_context_, file_schema)); return std::unique_ptr>>( std::move(writer)); diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp index 412f3b4c..ce502fb6 100644 --- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp +++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp @@ -48,15 +48,13 @@ ShreddingKeyValueDataFileWriterFactory::ShreddingKeyValueDataFileWriterFactory( Result>>> ShreddingKeyValueDataFileWriterFactory::CreateWriter() const { - PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingBatchConverter::ConverterBundle bundle, - MapSharedShreddingBatchConverter::CreateConverter( - write_schema_, shredding_context_, pool_)); - if (!bundle.converter || !bundle.physical_schema) { - return Status::Invalid( - "Shared-shredding key-value writer requires a converter and physical schema."); + if (!shredding_context_) { + return Status::Invalid("Shared-shredding key-value writer requires a shredding context."); } - std::shared_ptr file_schema = bundle.physical_schema; - auto converter = bundle.converter; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, + MapSharedShreddingBatchConverter::Create( + write_schema_, shredding_context_, options_, pool_)); + std::shared_ptr file_schema = converter->GetPhysicalSchema(); std::function batch_converter = [converter](KeyValueBatch key_value_batch, ::ArrowArray* array) -> Status { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowArray> physical, @@ -75,7 +73,7 @@ ShreddingKeyValueDataFileWriterFactory::CreateWriter() const { PAIMON_RETURN_NOT_OK( writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( - bundle.converter, MapSharedShreddingDefine::kDefaultDictCompression, shredding_context_, + converter, MapSharedShreddingDefine::kDefaultDictCompression, shredding_context_, file_schema)); return std::unique_ptr>>( std::move(writer)); diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp index 685e6569..d2a5cc51 100644 --- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp @@ -385,6 +385,7 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) { {Options::FILE_FORMAT, "orc"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, })); @@ -449,8 +450,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) { options.GetFileSystem()->GetFileStatus(expected_data_file_path)); std::map column_to_k = {{"tags", 3}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - write_schema, column_to_k)); + auto physical_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, column_to_k); auto physical_type = arrow::struct_(physical_schema->fields()); std::shared_ptr expected_array; @@ -493,8 +494,10 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) {Options::FILE_FORMAT, "orc"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "8"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {"fields.attrs.map.storage-layout", "shared-shredding"}, {"fields.attrs.map.shared-shredding.max-columns", "4"}, + {"fields.attrs.map.shared-shredding.column-placement-policy", "plain"}, {Options::WRITE_ONLY, "true"}, })); @@ -540,8 +543,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) path_factory->ToPath(commit_increment1.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k_file1 = {{"tags", 8}, {"attrs", 4}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( - write_schema, column_to_k_file1)); + auto physical_schema1 = + MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, column_to_k_file1); MapSharedShreddingFieldMeta tags_meta1; tags_meta1.name_to_id = {{"a", 0}, {"b", 1}}; tags_meta1.field_to_columns = {{0, {0}}, {1, {1}}}; @@ -568,8 +571,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) path_factory->ToPath(commit_increment2.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k_file2 = {{"tags", 2}, {"attrs", 1}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( - write_schema, column_to_k_file2)); + auto physical_schema2 = + MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, column_to_k_file2); MapSharedShreddingFieldMeta tags_meta2; tags_meta2.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}}; tags_meta2.field_to_columns = {{0, {0}}, {1, {1}}}; @@ -598,8 +601,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) path_factory->ToPath(commit_increment3.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k_file3 = {{"tags", 3}, {"attrs", 3}}; - ASSERT_OK_AND_ASSIGN(auto physical_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( - write_schema, column_to_k_file3)); + auto physical_schema3 = + MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, column_to_k_file3); MapSharedShreddingFieldMeta tags_meta3; tags_meta3.name_to_id = {{"f", 0}, {"g", 1}}; tags_meta3.field_to_columns = {{0, {0}}, {1, {1}}}; diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index daf1fba2..1a0eb92c 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -289,6 +289,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNex {"file.format", "parquet"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {"write-only", "true"}, {"bucket", "1"}, {"bucket-key", "id"}, @@ -323,9 +324,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNex ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); - ASSERT_OK_AND_ASSIGN( - auto expected_second_schema, - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 2}})); + auto expected_second_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 2}}); ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false)); ASSERT_EQ(2, second_meta.num_columns); ASSERT_EQ(3, second_meta.max_row_width); @@ -347,6 +347,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreIgnoresAvroFileWi shredding_options["file.format"] = "parquet"; shredding_options["fields.tags.map.storage-layout"] = "shared-shredding"; shredding_options["fields.tags.map.shared-shredding.max-columns"] = "10"; + shredding_options["fields.tags.map.shared-shredding.column-placement-policy"] = "plain"; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -367,8 +368,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreIgnoresAvroFileWi ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); - ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, {{"tags", 10}})); + auto expected_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 10}}); ASSERT_TRUE(second_file_schema->Equals(*expected_schema, /*check_metadata=*/false)); ASSERT_EQ(10, second_meta.num_columns); ASSERT_EQ(3, second_meta.max_row_width); @@ -379,8 +380,10 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreMultipleMapColumn {"file.format", "parquet"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {"fields.attrs.map.storage-layout", "shared-shredding"}, {"fields.attrs.map.shared-shredding.max-columns", "10"}, + {"fields.attrs.map.shared-shredding.column-placement-policy", "plain"}, {"write-only", "true"}, {"bucket", "1"}, {"bucket-key", "id"}, @@ -426,8 +429,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreMultipleMapColumn auto tags_meta = ShreddingMeta(full_file_schema, /*field_index=*/1); auto attrs_meta = ShreddingMeta(full_file_schema, /*field_index=*/2); - ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, {{"tags", 2}, {"attrs", 4}})); + auto expected_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 2}, {"attrs", 4}}); ASSERT_TRUE(full_file_schema->Equals(*expected_schema, /*check_metadata=*/false)); ASSERT_EQ(2, tags_meta.num_columns); ASSERT_EQ(3, tags_meta.max_row_width); @@ -440,8 +443,10 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreUsesDefaultForMis {"file.format", "parquet"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {"fields.attrs.map.storage-layout", "shared-shredding"}, {"fields.attrs.map.shared-shredding.max-columns", "10"}, + {"fields.attrs.map.shared-shredding.column-placement-policy", "plain"}, {"write-only", "true"}, {"bucket", "1"}, {"bucket-key", "id"}, @@ -476,8 +481,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreUsesDefaultForMis auto tags_meta = ShreddingMeta(full_file_schema, /*field_index=*/1); auto attrs_meta = ShreddingMeta(full_file_schema, /*field_index=*/2); - ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, {{"tags", 2}, {"attrs", 10}})); + auto expected_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 2}, {"attrs", 10}}); ASSERT_TRUE(full_file_schema->Equals(*expected_schema, /*check_metadata=*/false)); ASSERT_EQ(2, tags_meta.num_columns); ASSERT_EQ(3, tags_meta.max_row_width); @@ -490,6 +495,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingPartialWriteSkipsMissing {"file.format", "parquet"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {"write-only", "true"}, {"bucket", "1"}, {"bucket-key", "id"}, diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index e37a8289..f939fb46 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -319,6 +319,7 @@ TEST_F(KeyValueFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNextW {"file.format", "parquet"}, {"fields.tags.map.storage-layout", "shared-shredding"}, {"fields.tags.map.shared-shredding.max-columns", "10"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {"write-only", "true"}, {"bucket", "1"}, {"enable-pk-commit-in-inte-test", ""}, @@ -353,9 +354,8 @@ TEST_F(KeyValueFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNextW ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/3); - ASSERT_OK_AND_ASSIGN( - auto expected_second_schema, - MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 2}})); + auto expected_second_schema = + MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 2}}); ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false)); ASSERT_EQ(2, second_meta.num_columns); ASSERT_EQ(3, second_meta.max_row_width); diff --git a/src/paimon/core/options/map_shared_shredding_column_placement_policy.h b/src/paimon/core/options/map_shared_shredding_column_placement_policy.h new file mode 100644 index 00000000..040c5b32 --- /dev/null +++ b/src/paimon/core/options/map_shared_shredding_column_placement_policy.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +namespace paimon { + +/// Specifies how shared-shredding MAP fields choose physical columns. +enum class MapSharedShreddingColumnPlacementPolicy { + /// Keep the key order from each input MAP row and place the first K keys into columns 0..K-1. + /// Use this when the input order is meaningful and should directly decide column placement. + /// Example: + /// K=2 + /// row [c, a, b] -> ordered [c, a, b] -> columns [c, a], overflow [b] + /// row [a, b, c] -> ordered [a, b, c] -> columns [a, b], overflow [c] + PLAIN = 0, + /// Use the shared-shredding metadata order before placing the first K keys into columns 0..K-1. + /// Use this when each row should choose leading-column keys by a deterministic metadata + /// order instead of the input MAP entry order, and no cross-row history should be used. + /// Example: + /// K=2, metadata order [b, c, a] + /// row [c, a, b] -> ordered [b, c, a] -> columns [b, c], overflow [a] + /// row [c, a] -> ordered [c, a] -> columns [c, a], overflow [] + /// This does not reserve key-to-column mappings across rows. + SEQUENTIAL = 1, + /// Reuse columns for recently seen keys when possible; otherwise choose an empty column first, + /// then the least-recently-used physical column. + /// Use this when hot keys are likely to appear repeatedly across nearby rows and should stay + /// in physical columns when possible. + /// Example: + /// K=3 + /// row [a, b, d] -> ordered [a, b, d] -> columns [a, b, d], overflow [] + /// recently used columns are now [a, b, d] + /// row [d, c, a, b] -> ordered [a, b, c, d] -> columns [a, b, d], overflow [c] + /// If a key has already been evicted, it is treated like a new key when it appears again. + LRU = 2 +}; + +} // namespace paimon diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp index 31814581..641f2831 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp @@ -324,9 +324,11 @@ TEST_F(PostponeBucketWriterTest, TestSharedShreddingMap) { arrow::field("tags", arrow::map(arrow::utf8(), arrow::int32()))}; ASSERT_OK_AND_ASSIGN( CoreOptions options, - CoreOptions::FromMap({{Options::FILE_FORMAT, file_format}, - {"fields.tags.map.storage-layout", "shared-shredding"}, - {"fields.tags.map.shared-shredding.max-columns", "3"}})); + CoreOptions::FromMap( + {{Options::FILE_FORMAT, file_format}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "3"}, + {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}})); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); @@ -366,8 +368,8 @@ TEST_F(PostponeBucketWriterTest, TestSharedShreddingMap) { arrow::FieldVector write_fields = {arrow::field("_SEQUENCE_NUMBER", arrow::int64()), arrow::field("_VALUE_KIND", arrow::int8())}; write_fields.insert(write_fields.end(), fields.begin(), fields.end()); - ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( - arrow::schema(write_fields), {{"tags", 3}})); + auto expected_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema( + arrow::schema(write_fields), {{"tags", 3}}); MapSharedShreddingFieldMeta expected_meta; expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index eb43c9c1..07a8c42b 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -567,6 +567,8 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema, } // Validate max-columns config PAIMON_RETURN_NOT_OK(options.GetMapSharedShreddingMaxColumns(field_name)); + // Validate placement policy config + PAIMON_RETURN_NOT_OK(options.GetMapSharedShreddingColumnPlacementPolicy(field_name)); } return Status::OK(); } diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 4cb52f41..fccf6729 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -940,6 +940,21 @@ TEST(SchemaValidationTest, TestMapStorageLayout) { ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "options map.shared-shredding.max-columns must > 0"); } + // Invalid: shared-shredding with invalid placement policy + { + arrow::FieldVector fields = {f0, f1, f2}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.f2.map.storage-layout", "shared-shredding"}, + {"fields.f2.map.shared-shredding.column-placement-policy", "invalid"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "invalid map.shared-shredding.column-placement-policy: invalid"); + } } } // namespace paimon::test From f75a27b94837ea79db34afc8f3355b8292e91c99 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Wed, 1 Jul 2026 09:51:00 +0800 Subject: [PATCH 077/138] feat(types): support CHAR/VARCHAR/BINARY/VARBINARY in data type json parser --- docs/source/user_guide/data_types.rst | 18 +++- .../common/types/data_type_json_parser.cpp | 16 +++- .../types/data_type_json_parser_test.cpp | 16 ++++ test/inte/write_and_read_inte_test.cpp | 95 +++++++++++++++++++ 4 files changed, 139 insertions(+), 6 deletions(-) diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index de794d83..0c60e41e 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -41,24 +41,29 @@ and `Arrow DataTypes #include #include +#include #include #include #include @@ -468,6 +469,12 @@ Result> TokenParser::ParseTypeWithNullability(b Result> TokenParser::ParseTypeByKeyword(bool* is_blob) { PAIMON_RETURN_NOT_OK(NextToken(TokenType::KEYWORD)); switch (TokenAsKeyword()) { + case Keyword::CHAR: + case Keyword::VARCHAR: + return ParseStringType(); + case Keyword::BINARY: + case Keyword::VARBINARY: + return ParseStringType(); case Keyword::BYTES: return arrow::binary(); case Keyword::BLOB: { @@ -511,9 +518,14 @@ Result TokenParser::ParseStringLength() { if (HasNextToken({TokenType::BEGIN_PARAMETER})) { PAIMON_RETURN_NOT_OK(NextToken(TokenType::BEGIN_PARAMETER)); PAIMON_RETURN_NOT_OK(NextToken(TokenType::LITERAL_INT)); - auto length = TokenAsInt(); + int64_t length = std::stoll(GetToken().value); + if (length < 1 || length > std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("length must be between 1 and {} (both inclusive), but was {}", + std::numeric_limits::max(), length)); + } PAIMON_RETURN_NOT_OK(NextToken(TokenType::END_PARAMETER)); - return length; + return static_cast(length); } // implicit length return -1; diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index 25b2ef59..fc0cdbfa 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -114,6 +114,16 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { {"TIMESTAMP_LTZ(9)", arrow::timestamp(arrow::TimeUnit::NANO, timezone)}, {"BYTES", arrow::binary()}, {"STRING", arrow::utf8()}, + {"CHAR", arrow::utf8()}, + {"CHAR(10)", arrow::utf8()}, + {"VARCHAR", arrow::utf8()}, + {"VARCHAR(10)", arrow::utf8()}, + {"BINARY", arrow::binary()}, + {"BINARY(10)", arrow::binary()}, + {"VARBINARY", arrow::binary()}, + {"VARBINARY(10)", arrow::binary()}, + {"CHAR(1)", arrow::utf8()}, + {"VARCHAR(2147483647)", arrow::utf8()}, }; for (const auto& test_case : test_cases) { @@ -134,6 +144,12 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { rapidjson::Value value("VARCHAR(test)", invalid_doc.GetAllocator()); ASSERT_NOK(DataTypeJsonParser::ParseType("field_name", value)); } + for (const char* invalid_type : {"VARCHAR(0)", "VARBINARY(0)", "VARCHAR(2147483648)"}) { + rapidjson::Document invalid_doc; + rapidjson::Value value(invalid_type, invalid_doc.GetAllocator()); + ASSERT_NOK_WITH_MSG(DataTypeJsonParser::ParseType("field_name", value), + "length must be between 1 and 2147483647"); + } { rapidjson::Document invalid_doc; rapidjson::Value value("TIMESTAMP(4)", invalid_doc.GetAllocator()); diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 7315cc16..c564ab66 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -51,6 +51,9 @@ #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" namespace paimon::test { // This is a sdk end-to-end test demo that supports write, commit, scan, and read operations. @@ -94,6 +97,38 @@ class WriteAndReadInteTest return file_system->AtomicStore(schema_path, schema_content); } + Status WriteNextSchemaWithRawFieldTypes(const std::string& fields_json, + int32_t highest_field_id) const { + auto file_system = dir_->GetFileSystem(); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + SchemaManager schema_manager(file_system, table_path); + PAIMON_ASSIGN_OR_RAISE(auto latest_schema_opt, schema_manager.Latest()); + if (!latest_schema_opt) { + return Status::Invalid("table schema does not exist"); + } + auto next_schema = std::make_shared(*latest_schema_opt.value()); + next_schema->id_ = latest_schema_opt.value()->Id() + 1; + next_schema->highest_field_id_ = highest_field_id; + PAIMON_ASSIGN_OR_RAISE(std::string schema_content, next_schema->ToJsonString()); + + rapidjson::Document schema_doc; + schema_doc.Parse(schema_content.c_str()); + rapidjson::Document fields_doc; + fields_doc.Parse(fields_json.c_str()); + if (schema_doc.HasParseError() || fields_doc.HasParseError() || + !schema_doc.HasMember("fields")) { + return Status::Invalid("failed to assemble schema json with raw field types"); + } + schema_doc["fields"].CopyFrom(fields_doc, schema_doc.GetAllocator()); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + schema_doc.Accept(writer); + + std::string schema_path = PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(next_schema->Id())); + return file_system->AtomicStore(schema_path, std::string(buffer.GetString())); + } + Result ReadAndCheckProjectedResult(const std::map& options, const std::vector& read_fields, const std::shared_ptr& expected_type, @@ -932,6 +967,66 @@ TEST_P(WriteAndReadInteTest, TestWriteSamePartitionTwiceWithAllBasicTypesForPk) ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestCharVarcharBinaryVarbinaryTypes) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + arrow::FieldVector fields = { + arrow::field("c", arrow::utf8()), + arrow::field("vc", arrow::utf8()), + arrow::field("b", arrow::binary()), + arrow::field("vb", arrow::binary()), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + ASSERT_OK(WriteNextSchemaWithRawFieldTypes(R"json([ + { "id" : 0, "name" : "c", "type" : "CHAR(10)" }, + { "id" : 1, "name" : "vc", "type" : "VARCHAR(20)" }, + { "id" : 2, "name" : "b", "type" : "BINARY(10)" }, + { "id" : 3, "name" : "vb", "type" : "VARBINARY(20)" } + ])json", + /*highest_field_id=*/3)); + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options, + /*is_streaming_mode=*/false)); + + std::string data = R"([ + ["alice", "hello world", "abc", "xyz123"], + [null, null, null, null] + ])"; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + std::string expected_data = R"([ + [0, "alice", "hello world", "abc", "xyz123"], + [0, null, null, null, null] + ])"; + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(data_type, data_splits, expected_data)); + ASSERT_TRUE(success); +} + std::vector> GetTestValuesForWriteAndReadInteTest() { std::vector> values = {{"parquet", "local"}}; // values.emplace_back("parquet", "jindo"); From 5ab8be58bf7441f1ff860905164f36d6b2dd8a26 Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Wed, 1 Jul 2026 17:33:48 +0800 Subject: [PATCH 078/138] fix: align duplicate-key replacement semantics with Java Paimon --- src/paimon/CMakeLists.txt | 1 + .../deletionvectors/deletion_file_writer.cpp | 4 +- .../deletion_file_writer_test.cpp | 33 +++++ .../core/deletionvectors/deletion_vector.cpp | 2 +- .../deletionvectors/deletion_vector_test.cpp | 12 ++ .../manifest/index_manifest_file_handler.cpp | 24 ++-- .../manifest/index_manifest_file_handler.h | 4 +- .../index_manifest_file_handler_test.cpp | 65 +++++++++- .../table/source/snapshot/snapshot_reader.cpp | 4 +- .../source/snapshot/snapshot_reader_test.cpp | 115 ++++++++++++++++++ 10 files changed, 242 insertions(+), 22 deletions(-) create mode 100644 src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 18b09e67..fa52567c 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -741,6 +741,7 @@ if(PAIMON_BUILD_TESTS) core/table/source/data_split_test.cpp core/table/source/deletion_file_test.cpp core/table/source/split_generator_test.cpp + core/table/source/snapshot/snapshot_reader_test.cpp core/table/source/startup_mode_test.cpp core/table/source/table_scan_test.cpp core/table/system/system_table_test.cpp diff --git a/src/paimon/core/deletionvectors/deletion_file_writer.cpp b/src/paimon/core/deletionvectors/deletion_file_writer.cpp index 32f1ce52..e463dea6 100644 --- a/src/paimon/core/deletionvectors/deletion_file_writer.cpp +++ b/src/paimon/core/deletionvectors/deletion_file_writer.cpp @@ -44,8 +44,8 @@ Status DeletionFileWriter::Write(const std::string& key, } DataOutputStream output_stream(out_); PAIMON_ASSIGN_OR_RAISE(int32_t length, deletion_vector->SerializeTo(pool_, &output_stream)); - dv_metas_.insert(key, DeletionVectorMeta(key, static_cast(start), length, - deletion_vector->GetCardinality())); + dv_metas_.insert_or_assign(key, DeletionVectorMeta(key, static_cast(start), length, + deletion_vector->GetCardinality())); return Status::OK(); } diff --git a/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp b/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp index 09673d84..140f17f1 100644 --- a/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp +++ b/src/paimon/core/deletionvectors/deletion_file_writer_test.cpp @@ -97,6 +97,39 @@ TEST(DeletionFileWriterTest, GetResultWithoutCloseShouldFail) { ASSERT_NOK_WITH_MSG(writer->GetResult(), "Deletion file result length -1 out of int32 range"); } +TEST(DeletionFileWriterTest, WriteOverwritesDuplicateDataFileName) { + auto dir = UniqueTestDirectory::Create(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, + FileSystemFactory::Get("local", dir->Str(), {})); + auto path_factory = std::make_shared(dir->Str()); + auto pool = GetDefaultPool(); + + ASSERT_OK_AND_ASSIGN(auto writer, DeletionFileWriter::Create(path_factory, fs, pool)); + + RoaringBitmap32 roaring_1; + roaring_1.Add(1); + auto dv_1 = std::make_shared(roaring_1); + + RoaringBitmap32 roaring_2; + roaring_2.Add(2); + roaring_2.Add(3); + auto dv_2 = std::make_shared(roaring_2); + + ASSERT_OK(writer->Write("data-file-1", dv_1)); + ASSERT_OK(writer->Write("data-file-1", dv_2)); + ASSERT_OK(writer->Close()); + + ASSERT_OK_AND_ASSIGN(auto meta, writer->GetResult()); + const auto& dv_ranges = meta->DvRanges(); + ASSERT_TRUE(dv_ranges.has_value()); + ASSERT_EQ(dv_ranges->size(), 1); + + auto iter = dv_ranges->find("data-file-1"); + ASSERT_NE(iter, dv_ranges->end()); + ASSERT_GT(iter->second.GetOffset(), 1); + ASSERT_EQ(iter->second.GetCardinality(), std::optional(2)); +} + TEST(DeletionFileWriterTest, ExternalPathInResult) { auto dir = UniqueTestDirectory::Create(); ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, diff --git a/src/paimon/core/deletionvectors/deletion_vector.cpp b/src/paimon/core/deletionvectors/deletion_vector.cpp index b34a1a32..f981d19b 100644 --- a/src/paimon/core/deletionvectors/deletion_vector.cpp +++ b/src/paimon/core/deletionvectors/deletion_vector.cpp @@ -72,7 +72,7 @@ std::unordered_map DeletionVector::CreateDeletionFile assert(deletion_files.size() == data_files.size()); for (size_t i = 0; i < deletion_files.size(); i++) { if (deletion_files[i] != std::nullopt) { - deletion_file_map.emplace(data_files[i]->file_name, deletion_files[i].value()); + deletion_file_map.insert_or_assign(data_files[i]->file_name, deletion_files[i].value()); } } return deletion_file_map; diff --git a/src/paimon/core/deletionvectors/deletion_vector_test.cpp b/src/paimon/core/deletionvectors/deletion_vector_test.cpp index 194a5c33..daf2ecf1 100644 --- a/src/paimon/core/deletionvectors/deletion_vector_test.cpp +++ b/src/paimon/core/deletionvectors/deletion_vector_test.cpp @@ -195,6 +195,18 @@ TEST(DeletionVectorTest, CreateDeletionFileMap) { ASSERT_EQ(deletion_file_map.at("file-0.orc"), deletion_file_0); ASSERT_EQ(deletion_file_map.count("file-1.orc"), 0); ASSERT_EQ(deletion_file_map.at("file-2.orc"), deletion_file_2); + + DeletionFile deletion_file_0_new("dv-0-new", /*offset=*/50, /*length=*/60, + /*cardinality=*/7); + std::vector> duplicate_data_files = { + CreateDataFileMeta("file-0.orc"), CreateDataFileMeta("file-0.orc")}; + std::vector> duplicate_deletion_files = {deletion_file_0, + deletion_file_0_new}; + + auto duplicate_deletion_file_map = + DeletionVector::CreateDeletionFileMap(duplicate_data_files, duplicate_deletion_files); + ASSERT_EQ(duplicate_deletion_file_map.size(), 1); + ASSERT_EQ(duplicate_deletion_file_map.at("file-0.orc"), deletion_file_0_new); } } // namespace paimon::test diff --git a/src/paimon/core/manifest/index_manifest_file_handler.cpp b/src/paimon/core/manifest/index_manifest_file_handler.cpp index efa6b610..2fb1ceb4 100644 --- a/src/paimon/core/manifest/index_manifest_file_handler.cpp +++ b/src/paimon/core/manifest/index_manifest_file_handler.cpp @@ -32,33 +32,31 @@ std::vector IndexManifestFileHandler::BucketedCombiner::Comb const std::vector& new_index_files) const { std::unordered_map index_entries; for (const auto& entry : prev_index_files) { - index_entries.emplace( + index_entries.insert_or_assign( BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType()), entry); } - std::unordered_map removed; + std::vector removed; removed.reserve(new_index_files.size()); - std::unordered_map added; + std::vector added; added.reserve(new_index_files.size()); for (const auto& entry : new_index_files) { if (entry.kind == FileKind::Delete()) { - removed.emplace( - BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType()), - entry); + removed.push_back(entry); } else if (entry.kind == FileKind::Add()) { - added.emplace( - BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType()), - entry); + added.push_back(entry); } } // The deleted entry is processed first to avoid overwriting a new entry. for (const auto& entry : removed) { - index_entries.erase(entry.first); + index_entries.erase( + BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType())); } for (const auto& entry : added) { - index_entries.emplace(entry.first, entry.second); + index_entries.insert_or_assign( + BucketIdentifier(entry.partition, entry.bucket, entry.index_file->IndexType()), entry); } std::vector result_entries; @@ -74,7 +72,7 @@ std::vector IndexManifestFileHandler::GlobalFileNameCombiner const std::vector& new_index_files) const { std::map index_entries; for (const auto& entry : prev_index_files) { - index_entries.emplace(entry.index_file->FileName(), entry); + index_entries.insert_or_assign(entry.index_file->FileName(), entry); } std::vector removed; @@ -95,7 +93,7 @@ std::vector IndexManifestFileHandler::GlobalFileNameCombiner index_entries.erase(entry.index_file->FileName()); } for (const auto& entry : added) { - index_entries.emplace(entry.index_file->FileName(), entry); + index_entries.insert_or_assign(entry.index_file->FileName(), entry); } std::vector result_entries; diff --git a/src/paimon/core/manifest/index_manifest_file_handler.h b/src/paimon/core/manifest/index_manifest_file_handler.h index 05dd8576..910b70f5 100644 --- a/src/paimon/core/manifest/index_manifest_file_handler.h +++ b/src/paimon/core/manifest/index_manifest_file_handler.h @@ -48,7 +48,7 @@ class IndexManifestFileHandler { const std::vector& new_index_files) const = 0; }; - /// Combine previous and new global index files by file `BucketIdentifier`. + /// Combine previous and new index files by partition, bucket and index type. class BucketedCombiner : public IndexManifestFileCombiner { public: std::vector Combine( @@ -56,7 +56,7 @@ class IndexManifestFileHandler { const std::vector& new_index_files) const override; }; - /// Combine previous and new global index files by file name. + /// Combine previous and new index files by file name. class GlobalFileNameCombiner : public IndexManifestFileCombiner { public: std::vector Combine( diff --git a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp index 15d1e9c4..2265e18b 100644 --- a/src/paimon/core/manifest/index_manifest_file_handler_test.cpp +++ b/src/paimon/core/manifest/index_manifest_file_handler_test.cpp @@ -76,6 +76,25 @@ class IndexManifestFileHandlerTest : public testing::Test { /*external_path=*/std::nullopt)); } + static IndexManifestEntry MakeDvEntry(const FileKind& kind, const BinaryRow& partition, + int32_t bucket, const std::string& file_name, + const std::vector& data_file_names, + int64_t row_count) { + LinkedHashMap dv_ranges; + int32_t offset = 0; + for (const auto& data_file_name : data_file_names) { + dv_ranges.insert(data_file_name, + DeletionVectorMeta(data_file_name, offset, /*length=*/10, + /*cardinality=*/std::nullopt)); + offset += 10; + } + return IndexManifestEntry( + kind, partition, bucket, + std::make_shared(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, + file_name, /*file_size=*/row_count * 10, row_count, + dv_ranges, /*external_path=*/std::nullopt)); + } + std::shared_ptr pool_; std::unique_ptr dir_; }; @@ -156,13 +175,55 @@ TEST_F(IndexManifestFileHandlerTest, BucketedCombinerUsesPartitionBucketAndIndex ASSERT_TRUE(found_bucket1); } +TEST_F(IndexManifestFileHandlerTest, BucketedCombinerOverwritesDuplicateAddedEntries) { + ASSERT_OK_AND_ASSIGN(auto index_manifest_file, CreateManifestFile(/*bucket_mode=*/2)); + + auto partition = BinaryRowGenerator::GenerateRow({10}, pool_.get()); + std::vector new_entries = { + MakeEntry(FileKind::Add(), partition, /*bucket=*/0, + DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-0-old", 10), + MakeEntry(FileKind::Add(), partition, /*bucket=*/0, + DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-0-new", 20)}; + + ASSERT_OK_AND_ASSIGN(std::string current_manifest, + IndexManifestFileHandler::Write( + /*previous_index_manifest=*/std::nullopt, new_entries, + /*bucket_mode=*/2, index_manifest_file.get())); + + std::vector written_entries; + ASSERT_OK(index_manifest_file->Read(current_manifest, /*filter=*/nullptr, &written_entries)); + ASSERT_EQ(written_entries.size(), 1); + ASSERT_EQ(written_entries[0].index_file->FileName(), "dv-0-new"); + ASSERT_EQ(written_entries[0].index_file->RowCount(), 20); +} + +TEST_F(IndexManifestFileHandlerTest, GlobalCombinerOverwritesDuplicateAddedEntries) { + ASSERT_OK_AND_ASSIGN(auto index_manifest_file, CreateManifestFile(/*bucket_mode=*/4)); + + auto partition = BinaryRow::EmptyRow(); + std::vector new_entries = { + MakeEntry(FileKind::Add(), partition, /*bucket=*/0, /*index_type=*/"BTREE", "global-0", 10), + MakeEntry(FileKind::Add(), partition, /*bucket=*/0, /*index_type=*/"BTREE", "global-0", + 20)}; + + ASSERT_OK_AND_ASSIGN(std::string current_manifest, + IndexManifestFileHandler::Write( + /*previous_index_manifest=*/std::nullopt, new_entries, + /*bucket_mode=*/4, index_manifest_file.get())); + + std::vector written_entries; + ASSERT_OK(index_manifest_file->Read(current_manifest, /*filter=*/nullptr, &written_entries)); + ASSERT_EQ(written_entries.size(), 1); + ASSERT_EQ(written_entries[0].index_file->FileName(), "global-0"); + ASSERT_EQ(written_entries[0].index_file->RowCount(), 20); +} + TEST_F(IndexManifestFileHandlerTest, DvWithBucketUnawareModeReturnsNotImplemented) { ASSERT_OK_AND_ASSIGN(auto index_manifest_file, CreateManifestFile(/*bucket_mode=*/-1)); auto partition = BinaryRow::EmptyRow(); std::vector new_entries = { - MakeEntry(FileKind::Add(), partition, /*bucket=*/0, - DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-0", 1)}; + MakeDvEntry(FileKind::Add(), partition, /*bucket=*/0, "dv-0", {"data-0.orc"}, 1)}; ASSERT_NOK_WITH_MSG(IndexManifestFileHandler::Write( /*previous_index_manifest=*/std::nullopt, new_entries, diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader.cpp index c6b0477e..0e98b5cf 100644 --- a/src/paimon/core/table/source/snapshot/snapshot_reader.cpp +++ b/src/paimon/core/table/source/snapshot/snapshot_reader.cpp @@ -123,8 +123,8 @@ Result>> SnapshotReader::GetDeletionFile if (dv_metas != std::nullopt) { for (const auto& dv_meta_iter : dv_metas.value()) { const auto& dv_meta = dv_meta_iter.second; - data_file_to_index_file_meta.insert( - std::make_pair(dv_meta.GetDataFileName(), index_file_meta)); + data_file_to_index_file_meta.insert_or_assign(dv_meta.GetDataFileName(), + index_file_meta); } } } diff --git a/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp new file mode 100644 index 00000000..cebc644c --- /dev/null +++ b/src/paimon/core/table/source/snapshot/snapshot_reader_test.cpp @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/snapshot/snapshot_reader.h" + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +class SnapshotReaderTest : public testing::Test { + protected: + void SetUp() override { + pool_ = GetDefaultPool(); + dir_ = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_ != nullptr); + } + + std::shared_ptr CreateDataFileMeta(const std::string& file_name) const { + return std::make_shared( + file_name, /*file_size=*/100, /*row_count=*/10, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/0, /*schema_id=*/0, + DataFileMeta::DUMMY_LEVEL, std::vector>{}, Timestamp(0, 0), + std::nullopt, nullptr, FileSource::Append(), std::nullopt, std::nullopt, std::nullopt, + std::nullopt); + } + + std::shared_ptr CreateIndexFileMeta(const std::string& index_file_name, + const std::string& data_file_name, + int64_t offset, int64_t length, + std::optional cardinality) const { + LinkedHashMap dv_ranges; + dv_ranges.insert(data_file_name, + DeletionVectorMeta(data_file_name, offset, length, cardinality)); + return std::make_shared(DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, + index_file_name, /*file_size=*/100, /*row_count=*/10, + dv_ranges, /*external_path=*/std::nullopt); + } + + Result> CreateIndexFileHandler() const { + auto schema = arrow::schema({arrow::field("f0", arrow::int32())}); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + dir_->Str(), schema, /*partition_keys=*/{}, /*default_part_value=*/"", "orc", + /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/true, + /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, + /*index_file_in_data_file_dir=*/false, pool_)); + auto path_factories = std::make_shared(path_factory); + return std::make_unique(std::make_shared(), + std::unique_ptr(), + path_factories, /*dv_bitmap64=*/false, pool_); + } + + std::shared_ptr pool_; + std::unique_ptr dir_; +}; + +TEST_F(SnapshotReaderTest, GetDeletionFilesOverwritesDuplicateDataFileName) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_file_handler, + CreateIndexFileHandler()); + SnapshotReader snapshot_reader(/*scan=*/nullptr, /*path_factory=*/nullptr, + /*split_generator=*/nullptr, std::move(index_file_handler)); + + const std::string data_file_name = "data-0.orc"; + std::vector> data_files = {CreateDataFileMeta(data_file_name)}; + std::vector> index_file_metas = { + CreateIndexFileMeta("index-first", data_file_name, /*offset=*/1, /*length=*/11, + /*cardinality=*/3), + CreateIndexFileMeta("index-second", data_file_name, /*offset=*/2, /*length=*/22, + /*cardinality=*/4)}; + + ASSERT_OK_AND_ASSIGN(std::vector> deletion_files, + snapshot_reader.GetDeletionFiles(BinaryRow::EmptyRow(), /*bucket=*/0, + data_files, index_file_metas)); + + ASSERT_EQ(deletion_files.size(), 1); + ASSERT_TRUE(deletion_files[0].has_value()); + EXPECT_EQ(deletion_files[0]->path, dir_->Str() + "/index/index-second"); + EXPECT_EQ(deletion_files[0]->offset, 2); + EXPECT_EQ(deletion_files[0]->length, 22); + EXPECT_EQ(deletion_files[0]->cardinality, std::optional(4)); +} + +} // namespace paimon::test From 0bbac900617f9f15e205313497f76fc950eb8f45 Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Thu, 2 Jul 2026 08:51:42 +0800 Subject: [PATCH 079/138] fix: handle non-contiguous RowRanges when resolving global row IDs --- include/paimon/reader/file_batch_reader.h | 10 +- .../map_shared_shredding_file_reader.cpp | 5 +- .../map_shared_shredding_file_reader.h | 2 +- .../bitmap/apply_bitmap_index_batch_reader.h | 27 ++- .../reader/delegating_prefetch_reader.h | 4 +- .../prefetch_file_batch_reader_impl.cpp | 72 +++++-- .../reader/prefetch_file_batch_reader_impl.h | 6 +- .../prefetch_file_batch_reader_impl_test.cpp | 178 +++++++++++------ .../apply_deletion_vector_batch_reader.h | 16 +- .../complete_row_tracking_fields_reader.cpp | 9 +- .../io/complete_row_tracking_fields_reader.h | 4 +- src/paimon/core/io/field_mapping_reader.h | 4 +- .../io/key_value_data_file_record_reader.cpp | 9 +- .../io/key_value_data_file_record_reader.h | 7 +- .../format/avro/avro_file_batch_reader.cpp | 3 + .../format/avro/avro_file_batch_reader.h | 17 +- .../avro/avro_file_batch_reader_test.cpp | 22 +-- .../format/blob/blob_file_batch_reader.cpp | 4 +- .../format/blob/blob_file_batch_reader.h | 22 ++- .../blob/blob_file_batch_reader_test.cpp | 14 +- .../format/orc/orc_file_batch_reader.cpp | 9 +- src/paimon/format/orc/orc_file_batch_reader.h | 19 +- .../format/orc/orc_file_batch_reader_test.cpp | 33 ++-- src/paimon/format/orc/orc_reader_wrapper.cpp | 1 + .../parquet/parquet_file_batch_reader.cpp | 61 +++++- .../parquet/parquet_file_batch_reader.h | 28 ++- .../parquet_file_batch_reader_test.cpp | 180 +++++++++++++++++- src/paimon/format/parquet/row_ranges.h | 2 +- .../testing/mock/mock_file_batch_reader.h | 9 +- .../testing/utils/read_result_collector.h | 98 ++++++---- 30 files changed, 663 insertions(+), 212 deletions(-) diff --git a/include/paimon/reader/file_batch_reader.h b/include/paimon/reader/file_batch_reader.h index 38c89f00..87771be3 100644 --- a/include/paimon/reader/file_batch_reader.h +++ b/include/paimon/reader/file_batch_reader.h @@ -49,8 +49,14 @@ class PAIMON_EXPORT FileBatchReader : public BatchReader { using BatchReader::NextBatch; using BatchReader::NextBatchWithBitmap; - /// Get the row number of the first row in the previously read batch. - virtual Result GetPreviousBatchFirstRowNumber() const = 0; + /// Get the file-level row ID for a given batch-relative row index + /// in the previously read batch. + /// + /// @param batch_row_id Zero-based index within the current batch. + /// @return The corresponding file-level row ID, or Status::Invalid + /// if no batch has been read yet, the last batch was EOF, + /// or batch_row_id is out of range. + virtual Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const = 0; /// Get the number of rows in the file. virtual Result GetNumberOfRows() const = 0; diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp index c1933cf1..dcceddc7 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.cpp @@ -394,8 +394,9 @@ void MapSharedShreddingFileReader::Close() { reader_->Close(); } -Result MapSharedShreddingFileReader::GetPreviousBatchFirstRowNumber() const { - return reader_->GetPreviousBatchFirstRowNumber(); +Result MapSharedShreddingFileReader::GetPreviousBatchFileRowId( + uint64_t batch_row_id) const { + return reader_->GetPreviousBatchFileRowId(batch_row_id); } Result MapSharedShreddingFileReader::GetNumberOfRows() const { diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h index 3ab92753..ba608cf0 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader.h @@ -63,7 +63,7 @@ class MapSharedShreddingFileReader : public FileBatchReader { void Close() override; - Result GetPreviousBatchFirstRowNumber() const override; + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override; Result GetNumberOfRows() const override; diff --git a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h index 28957512..4894ad06 100644 --- a/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h +++ b/src/paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h @@ -82,8 +82,8 @@ class ApplyBitmapIndexBatchReader : public FileBatchReader { return Status::Invalid("ApplyBitmapIndexBatchReader does not support SetReadSchema"); } - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return reader_->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { @@ -96,14 +96,23 @@ class ApplyBitmapIndexBatchReader : public FileBatchReader { private: Result Filter(int32_t batch_size) const { - RoaringBitmap32 is_valid; - PAIMON_ASSIGN_OR_RAISE(int32_t start_pos, reader_->GetPreviousBatchFirstRowNumber()); - int32_t length = batch_size; - for (auto iter = bitmap_.EqualOrLarger(start_pos); - iter != bitmap_.End() && *iter < start_pos + length; ++iter) { - is_valid.Add(*iter - start_pos); + RoaringBitmap32 result; + auto bitmap_iter = bitmap_.Begin(); + auto bitmap_end = bitmap_.End(); + + for (int32_t i = 0; i < batch_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(uint64_t file_row_id, reader_->GetPreviousBatchFileRowId(i)); + while (bitmap_iter != bitmap_end && static_cast(*bitmap_iter) < file_row_id) { + ++bitmap_iter; + } + if (bitmap_iter == bitmap_end) { + break; + } + if (static_cast(*bitmap_iter) == file_row_id) { + result.Add(i); + } } - return is_valid; + return result; } private: diff --git a/src/paimon/common/reader/delegating_prefetch_reader.h b/src/paimon/common/reader/delegating_prefetch_reader.h index 6d2b09c9..3cbcb08b 100644 --- a/src/paimon/common/reader/delegating_prefetch_reader.h +++ b/src/paimon/common/reader/delegating_prefetch_reader.h @@ -57,8 +57,8 @@ class DelegatingPrefetchReader : public FileBatchReader { return prefetch_reader_->SetReadSchema(read_schema, predicate, selection_bitmap); } - Result GetPreviousBatchFirstRowNumber() const override { - return GetReader()->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return GetReader()->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp index 95a11d3e..5f3e4c72 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp @@ -42,6 +42,19 @@ class Schema; namespace paimon { +namespace { + +std::pair ComputeBatchSliceByReadRange( + const std::vector& global_row_ids, const std::pair& read_range) { + auto begin_it = + std::lower_bound(global_row_ids.begin(), global_row_ids.end(), read_range.first); + auto end_it = std::lower_bound(global_row_ids.begin(), global_row_ids.end(), read_range.second); + return {static_cast(std::distance(global_row_ids.begin(), begin_it)), + static_cast(std::distance(global_row_ids.begin(), end_it))}; +} + +} // namespace + Result> PrefetchFileBatchReaderImpl::Create( const std::string& data_file_path, const ReaderBuilder* reader_builder, const std::shared_ptr& fs, uint32_t prefetch_max_parallel_num, int32_t batch_size, @@ -267,6 +280,7 @@ Status PrefetchFileBatchReaderImpl::CleanUp() { read_ranges_.clear(); read_ranges_in_group_.clear(); + current_batch_global_row_ids_.clear(); clean_prefetch_queue(); for (size_t i = 0; i < readers_pos_.size(); i++) { readers_pos_[i]->store(0); @@ -411,28 +425,44 @@ Status PrefetchFileBatchReaderImpl::EnsureReaderPosition( Status PrefetchFileBatchReaderImpl::HandleReadResult( size_t reader_idx, const std::pair& read_range, ReadBatchWithBitmap&& read_batch_with_bitmap) { - PAIMON_ASSIGN_OR_RAISE(uint64_t first_row_number, - readers_[reader_idx]->GetPreviousBatchFirstRowNumber()); auto& prefetch_queue = prefetch_queues_[reader_idx]; if (!BatchReader::IsEofBatch(read_batch_with_bitmap)) { auto& [read_batch, bitmap] = read_batch_with_bitmap; auto& [c_array, c_schema] = read_batch; + std::vector global_row_ids; + global_row_ids.reserve(c_array->length); + for (int64_t i = 0; i < c_array->length; ++i) { + PAIMON_ASSIGN_OR_RAISE(uint64_t global_row_id, + readers_[reader_idx]->GetPreviousBatchFileRowId(i)); + global_row_ids.push_back(global_row_id); + } + if (global_row_ids.empty()) { + ReaderUtils::ReleaseReadBatch(std::move(read_batch)); + return Status::OK(); + } + auto [slice_begin, slice_end] = ComputeBatchSliceByReadRange(global_row_ids, read_range); + // slice_begin should always be 0, records before read_range.first have been consumed or + // filtered out. + if (slice_begin != 0) { + return Status::Invalid(fmt::format("Slice begin is {}, which is not 0.", slice_begin)); + } - if (first_row_number >= read_range.second) { - // fully out of range, data before first_row_number has been filtered out - readers_pos_[reader_idx]->store(first_row_number); + if (0 == slice_end) { + // fully out of range, data before global_row_ids has been filtered out + readers_pos_[reader_idx]->store(global_row_ids[0]); ReaderUtils::ReleaseReadBatch(std::move(read_batch)); return Status::OK(); - } else if (first_row_number + c_array->length > read_range.second) { + } else if (slice_end < c_array->length) { // partially out of range, data before read_range.second has been effectively consumed readers_pos_[reader_idx]->store(read_range.second); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr src_array, arrow::ImportArray(c_array.get(), c_schema.get())); - int32_t target_length = read_range.second - first_row_number; - auto array = src_array->Slice(/*offset=*/0, target_length); + auto array = src_array->Slice(0, slice_end); PAIMON_RETURN_NOT_OK_FROM_ARROW( arrow::ExportArray(*array, c_array.get(), c_schema.get())); - bitmap.RemoveRange(target_length, src_array->length()); + bitmap.RemoveRange(slice_end, src_array->length()); + global_row_ids = + std::vector(global_row_ids.begin(), global_row_ids.begin() + slice_end); } else { // all within the range, data before readers_[reader_idx]->GetNextRowToRead() has been // effectively consumed @@ -442,11 +472,12 @@ Status PrefetchFileBatchReaderImpl::HandleReadResult( ReaderUtils::ReleaseReadBatch(std::move(read_batch)); return Status::OK(); } - prefetch_queue->push({read_range, std::move(read_batch_with_bitmap), first_row_number}); + prefetch_queue->push( + {read_range, std::move(read_batch_with_bitmap), std::move(global_row_ids)}); } else { std::pair eof_range; PAIMON_ASSIGN_OR_RAISE(eof_range, EofRange()); - prefetch_queue->push({eof_range, std::move(read_batch_with_bitmap), first_row_number}); + prefetch_queue->push({eof_range, std::move(read_batch_with_bitmap), {}}); readers_pos_[reader_idx]->store(std::numeric_limits::max()); } return Status::OK(); @@ -529,7 +560,7 @@ Result PrefetchFileBatchReaderImpl::NextBatchW std::unique_lock lock(working_mutex_); cv_.notify_one(); } - previous_batch_first_row_num_ = prefetch_batch.value().previous_batch_first_row_num; + current_batch_global_row_ids_ = std::move(prefetch_batch.value().global_row_ids); return std::move(prefetch_batch).value().batch; } } @@ -539,7 +570,7 @@ Result PrefetchFileBatchReaderImpl::NextBatchW assert(false); return Status::Invalid("peek batch not suppose to be nullptr"); } - previous_batch_first_row_num_ = peek_batch->previous_batch_first_row_num; + current_batch_global_row_ids_.clear(); return BatchReader::MakeEofBatchWithBitmap(); } if (value_count == prefetch_queues_.size()) { @@ -573,8 +604,19 @@ Result> PrefetchFileBatchReaderImpl::GetFileSchem return readers_[0]->GetFileSchema(); } -Result PrefetchFileBatchReaderImpl::GetPreviousBatchFirstRowNumber() const { - return previous_batch_first_row_num_; +Result PrefetchFileBatchReaderImpl::GetPreviousBatchFileRowId( + uint64_t batch_row_id) const { + if (current_batch_global_row_ids_.empty()) { + return Status::Invalid( + "Last batch is not read or last batch is empty, cannot get previous batch global row " + "id"); + } + if (batch_row_id >= current_batch_global_row_ids_.size()) { + return Status::Invalid( + fmt::format("batch_row_id {} is out of range, last batch row count is {}", batch_row_id, + current_batch_global_row_ids_.size())); + } + return current_batch_global_row_ids_[batch_row_id]; } Result PrefetchFileBatchReaderImpl::GetNumberOfRows() const { diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h index f2916da3..c54cda49 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h @@ -78,7 +78,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { const std::optional& selection_bitmap) override; Status SeekToRow(uint64_t row_number) override; - Result GetPreviousBatchFirstRowNumber() const override; + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override; Result GetNumberOfRows() const override; uint64_t GetNextRowToRead() const override; void Close() override; @@ -107,7 +107,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { struct PrefetchBatch { std::pair read_range; BatchReader::ReadBatchWithBitmap batch; - uint64_t previous_batch_first_row_num; + std::vector global_row_ids; }; PrefetchFileBatchReaderImpl( @@ -162,7 +162,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { std::unique_ptr background_thread_; Status read_status_; std::atomic is_shutdown_ = false; - uint64_t previous_batch_first_row_num_ = std::numeric_limits::max(); + std::vector current_batch_global_row_ids_; bool need_prefetch_ = false; bool read_ranges_freshed_ = false; const uint32_t prefetch_queue_capacity_; diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp index 3386a1bc..30ecdf65 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl_test.cpp @@ -249,6 +249,28 @@ class PrefetchFileBatchReaderImplTest : public ::testing::Test, std::shared_ptr executor_; }; +static Result, std::vector>> +CollectResultAndRowIds(FileBatchReader* reader) { + arrow::ArrayVector result_array_vector; + std::vector row_ids; + while (true) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader)); + if (batch == nullptr) { + break; + } + PAIMON_ASSIGN_OR_RAISE(uint64_t file_row_id, + reader->GetPreviousBatchFileRowId(batch->chunk(0)->length() - 1)); + row_ids.push_back(file_row_id); + result_array_vector.push_back(batch->chunk(0)); + } + if (result_array_vector.empty()) { + return std::make_pair(std::shared_ptr(), row_ids); + } + auto result_array = std::make_shared(result_array_vector); + return std::make_pair(result_array, row_ids); +} + std::vector PrepareTestParam() { std::vector values = { TestParam{"parquet", PrefetchCacheMode::ALWAYS}, @@ -282,14 +304,13 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestSimple) { /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 101); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); + auto expected_array = std::make_shared(data_array); - ASSERT_TRUE(result_array->Equals(expected_array)); + ASSERT_TRUE(array_and_row_ids.first->Equals(expected_array)); + auto row_ids = array_and_row_ids.second; + ASSERT_EQ(row_ids[row_ids.size() - 1], 100); } } @@ -605,14 +626,12 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestReadWithLargeBatchSize) { prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 101); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); + auto row_ids = array_and_row_ids.second; + ASSERT_EQ(row_ids[row_ids.size() - 1], 100); auto expected_array = std::make_shared(data_array); - ASSERT_TRUE(result_array->Equals(expected_array)); + ASSERT_TRUE(array_and_row_ids.first->Equals(expected_array)); } TEST_F(PrefetchFileBatchReaderImplTest, TestPartialReaderSuccessRead) { @@ -634,12 +653,11 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPartialReaderSuccessRead) { } arrow::ArrayVector result_array_vector; - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch_with_bitmap, reader->NextBatchWithBitmap()); auto& [batch, bitmap] = batch_with_bitmap; ASSERT_EQ(batch.first->length, bitmap.Cardinality()); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 0); + ASSERT_EQ(reader->GetPreviousBatchFileRowId(0).value(), 0); ASSERT_OK_AND_ASSIGN(auto array, ReadResultCollector::GetArray(std::move(batch))); result_array_vector.push_back(array); ASSERT_OK(prefetch_reader->GetReadStatus()); @@ -680,11 +698,9 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestAllReaderFailedWithIOError) { ->SetNextBatchStatus(Status::IOError("mock error")); } - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); auto batch_result = reader->NextBatchWithBitmap(); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_FALSE(batch_result.ok()); ASSERT_TRUE(batch_result.status().IsIOError()); ASSERT_FALSE(prefetch_reader->is_shutdown_); @@ -693,8 +709,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestAllReaderFailedWithIOError) { // call NextBatch again, will still return error status auto batch_result2 = reader->NextBatchWithBitmap(); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_FALSE(batch_result2.ok()); ASSERT_TRUE(batch_result2.status().IsIOError()); } @@ -711,13 +726,11 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithEmptyData) { prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 0); - ASSERT_FALSE(result_array); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); + auto row_ids = array_and_row_ids.second; + ASSERT_EQ(row_ids.size(), 0); + ASSERT_FALSE(array_and_row_ids.first); } TEST_F(PrefetchFileBatchReaderImplTest, TestCallNextBatchAfterReadingEof) { @@ -732,17 +745,16 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestCallNextBatchAfterReadingEof) { prefetch_max_parallel_num * 2, /*enable_adaptive_prefetch_strategy=*/false, executor_, /*initialize_read_ranges=*/true, /*prefetch_cache_mode=*/PrefetchCacheMode::ALWAYS, CacheConfig(), GetDefaultPool())); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 10); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); + auto row_ids = array_and_row_ids.second; + ASSERT_EQ(row_ids[row_ids.size() - 1], 9); auto expected_array = std::make_shared(data_array); - ASSERT_TRUE(result_array->Equals(expected_array)); + ASSERT_TRUE(array_and_row_ids.first->Equals(expected_array)); // continue to call NextBatch() after reading eof ASSERT_OK_AND_ASSIGN(auto batch_with_bitmap, reader->NextBatchWithBitmap()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_TRUE(BatchReader::IsEofBatch(batch_with_bitmap)); } @@ -825,6 +837,7 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestInvalidCase) { TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCompleteFiltering) { auto [file_format, cache_mode] = GetParam(); auto data_array = PrepareArray(90); + int32_t batch_size = 10; PrepareTestData(file_format, data_array, /*stripe_row_count=*/30, /*row_index_stride=*/30); auto schema = arrow::schema(fields_); ASSERT_OK_AND_ASSIGN(auto predicate, @@ -835,22 +848,19 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithPredicatePushdownWithCom FieldType::BIGINT, Literal(70l)), })); - auto reader = - PreparePrefetchReader(file_format, schema.get(), predicate, - /*selection_bitmap=*/std::nullopt, - /*batch_size=*/10, /*prefetch_max_parallel_num=*/3, cache_mode); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 90); + auto reader = PreparePrefetchReader(file_format, schema.get(), predicate, + /*selection_bitmap=*/std::nullopt, + /*batch_size=*/batch_size, /*prefetch_max_parallel_num=*/3, + cache_mode); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); arrow::ArrayVector expected_array_vector; + std::vector expected_row_ids = {9, 19, 29, 69, 79, 89}; expected_array_vector.push_back(data_array->Slice(0, 30)); expected_array_vector.push_back(data_array->Slice(60, 30)); auto expected_array = std::make_shared(expected_array_vector); - ASSERT_TRUE(CheckEqual(expected_array, result_array)); + ASSERT_TRUE(expected_array->Equals(array_and_row_ids.first)); + ASSERT_EQ(expected_row_ids, array_and_row_ids.second); } /// There are three stripes: [0,30), [30,60), [60,90). Each stripe has 3 row groups. @@ -860,6 +870,7 @@ TEST_P(PrefetchFileBatchReaderImplTest, TestPrefetchWithOrcPredicatePushdownWithRowGroupGranularity) { auto [file_format, cache_mode] = GetParam(); auto data_array = PrepareArray(90); + int32_t batch_size = 10; PrepareTestData(file_format, data_array, /*stripe_row_count=*/30, /*row_index_stride=*/10); auto schema = arrow::schema(fields_); @@ -871,23 +882,21 @@ TEST_P(PrefetchFileBatchReaderImplTest, FieldType::BIGINT, Literal(70l)), })); - auto reader = - PreparePrefetchReader(file_format, schema.get(), predicate, - /*selection_bitmap=*/std::nullopt, - /*batch_size=*/10, /*prefetch_max_parallel_num=*/3, cache_mode); + auto reader = PreparePrefetchReader(file_format, schema.get(), predicate, + /*selection_bitmap=*/std::nullopt, + /*batch_size=*/batch_size, /*prefetch_max_parallel_num=*/3, + cache_mode); ASSERT_OK(reader->RefreshReadRanges()); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); - ASSERT_OK_AND_ASSIGN(auto result_array, - ReadResultCollector::CollectResult( - reader.get(), /*max simulated data processing time*/ 100)); - ASSERT_EQ(reader->GetPreviousBatchFirstRowNumber().value(), 90); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(auto array_and_row_ids, CollectResultAndRowIds(reader.get())); arrow::ArrayVector expected_array_vector; + std::vector expected_row_ids = {9, 19, 79, 89}; expected_array_vector.push_back(data_array->Slice(0, 20)); expected_array_vector.push_back(data_array->Slice(70, 20)); auto expected_array = std::make_shared(expected_array_vector); - ASSERT_TRUE(CheckEqual(expected_array, result_array)); + ASSERT_TRUE(expected_array->Equals(array_and_row_ids.first)); + ASSERT_EQ(expected_row_ids, array_and_row_ids.second); } TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithBitmap) { @@ -921,4 +930,55 @@ TEST_F(PrefetchFileBatchReaderImplTest, TestPrefetchWithBitmap) { ASSERT_TRUE(result_chunk_array->Equals(expected_chunk_array)); } +TEST_P(PrefetchFileBatchReaderImplTest, TestRowMapping) { + auto [file_format, cache_mode] = GetParam(); + auto data_array = PrepareArray(90); + PrepareTestData(file_format, data_array, /*stripe_row_count=*/30, /*row_index_stride=*/10); + auto schema = arrow::schema(fields_); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or({ + PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, + Literal(20l), Literal(29l)), + PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, + Literal(70l), Literal(79l)), + })); + + auto reader = + PreparePrefetchReader(file_format, schema.get(), predicate, + /*selection_bitmap=*/std::nullopt, + /*batch_size=*/10, /*prefetch_max_parallel_num=*/3, cache_mode); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader.get())); + for (uint64_t i = 0; i < 10; i++) { + ASSERT_EQ(reader->GetPreviousBatchFileRowId(i).value(), 20 + i); + } + + ASSERT_OK_AND_ASSIGN(batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader.get())); + for (uint64_t i = 0; i < 10; i++) { + ASSERT_EQ(reader->GetPreviousBatchFileRowId(i).value(), 70 + i); + } + + // Set read schema again + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + predicate = PredicateBuilder::Between(/*field_index=*/1, /*field_name=*/"f1", FieldType::BIGINT, + Literal(30l), Literal(49l)); + ASSERT_OK(reader->SetReadSchema(c_schema.get(), predicate, std::nullopt)); + + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN(batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader.get())); + for (uint64_t i = 0; i < 10; i++) { + ASSERT_EQ(reader->GetPreviousBatchFileRowId(i).value(), 30 + i); + } + ASSERT_OK_AND_ASSIGN(batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(reader.get())); + for (uint64_t i = 0; i < 10; i++) { + ASSERT_EQ(reader->GetPreviousBatchFileRowId(i).value(), 40 + i); + } +} + } // namespace paimon::test diff --git a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h index ed20c574..1f43468c 100644 --- a/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h +++ b/src/paimon/core/deletionvectors/apply_deletion_vector_batch_reader.h @@ -84,8 +84,8 @@ class ApplyDeletionVectorBatchReader : public FileBatchReader { return Status::Invalid("ApplyDeletionVectorBatchReader does not support SetReadSchema"); } - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return reader_->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { @@ -98,9 +98,15 @@ class ApplyDeletionVectorBatchReader : public FileBatchReader { private: Result Filter(int32_t batch_size) const { - PAIMON_ASSIGN_OR_RAISE(uint64_t previous_batch_first_row_number, - reader_->GetPreviousBatchFirstRowNumber()); - return deletion_vector_->IsValid(previous_batch_first_row_number, batch_size); + RoaringBitmap32 is_valid; + for (int32_t i = 0; i < batch_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(uint64_t file_row_id, reader_->GetPreviousBatchFileRowId(i)); + PAIMON_ASSIGN_OR_RAISE(bool is_deleted, deletion_vector_->IsDeleted(file_row_id)); + if (!is_deleted) { + is_valid.Add(i); + } + } + return is_valid; } private: diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp index 2d63af11..93e93e9b 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp @@ -88,15 +88,14 @@ CompleteRowTrackingFieldsBatchReader::NextBatchWithBitmap() { std::string row_id_field_name = SpecialFields::RowId().Name(); if (read_schema_->GetFieldIndex(row_id_field_name) != -1) { row_id_array = src_struct_array->GetFieldByName(row_id_field_name); - PAIMON_ASSIGN_OR_RAISE(uint64_t previous_batch_first_row_number, - reader_->GetPreviousBatchFirstRowNumber()); - auto row_id_convert_func = [previous_batch_first_row_number, - this](int32_t idx_in_array) -> Result { + auto row_id_convert_func = [this](int32_t idx_in_array) -> Result { if (first_row_id_ == std::nullopt) { return Status::Invalid( "unexpected: read _ROW_ID special field, but first row id is null in meta"); } - return first_row_id_.value() + previous_batch_first_row_number + idx_in_array; + PAIMON_ASSIGN_OR_RAISE(uint64_t file_row_id, + reader_->GetPreviousBatchFileRowId(idx_in_array)); + return first_row_id_.value() + file_row_id; }; PAIMON_RETURN_NOT_OK(ConvertRowTrackingField(src_struct_array->length(), /*init_value=*/0, row_id_convert_func, &row_id_array)); diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.h b/src/paimon/core/io/complete_row_tracking_fields_reader.h index 9c75c21e..86b970d6 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.h +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.h @@ -62,8 +62,8 @@ class CompleteRowTrackingFieldsBatchReader : public FileBatchReader { reader_->Close(); } - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return reader_->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { diff --git a/src/paimon/core/io/field_mapping_reader.h b/src/paimon/core/io/field_mapping_reader.h index 5c09974f..7a5edc16 100644 --- a/src/paimon/core/io/field_mapping_reader.h +++ b/src/paimon/core/io/field_mapping_reader.h @@ -79,8 +79,8 @@ class FieldMappingReader : public FileBatchReader { return Status::Invalid("FieldMappingReader does not support SetReadSchema"); } - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + return reader_->GetPreviousBatchFileRowId(batch_row_id); } Result GetNumberOfRows() const override { diff --git a/src/paimon/core/io/key_value_data_file_record_reader.cpp b/src/paimon/core/io/key_value_data_file_record_reader.cpp index 85bc553b..18e27679 100644 --- a/src/paimon/core/io/key_value_data_file_record_reader.cpp +++ b/src/paimon/core/io/key_value_data_file_record_reader.cpp @@ -83,15 +83,15 @@ Result KeyValueDataFileRecordReader::Iterator::Next() { Result> KeyValueDataFileRecordReader::Iterator::NextWithFilePos() { PAIMON_ASSIGN_OR_RAISE(KeyValue kv, Next()); - return std::make_pair(previous_batch_first_row_number_ + cursor_ - 1, std::move(kv)); + PAIMON_ASSIGN_OR_RAISE(uint64_t global_row_id, + reader_->reader_->GetPreviousBatchFileRowId(cursor_ - 1)); + return std::make_pair(static_cast(global_row_id), std::move(kv)); } Result> KeyValueDataFileRecordReader::NextBatch() { Reset(); PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, reader_->NextBatchWithBitmap()); - PAIMON_ASSIGN_OR_RAISE(int64_t previous_batch_first_row_number, - reader_->GetPreviousBatchFirstRowNumber()); if (BatchReader::IsEofBatch(batch_with_bitmap)) { // reader eof, just return return std::unique_ptr(); @@ -142,8 +142,7 @@ Result> KeyValueDataFileRecordRe key_ctx_ = std::make_shared(key_fields, pool_); value_ctx_ = std::make_shared(value_fields, pool_); ArrowUtils::TraverseArray(data_batch); - return std::make_unique( - this, previous_batch_first_row_number); + return std::make_unique(this); } void KeyValueDataFileRecordReader::Reset() { diff --git a/src/paimon/core/io/key_value_data_file_record_reader.h b/src/paimon/core/io/key_value_data_file_record_reader.h index 398b290a..1ed08257 100644 --- a/src/paimon/core/io/key_value_data_file_record_reader.h +++ b/src/paimon/core/io/key_value_data_file_record_reader.h @@ -56,16 +56,13 @@ class KeyValueDataFileRecordReader : public KeyValueRecordReader { class Iterator : public KeyValueRecordReader::Iterator { public: - Iterator(KeyValueDataFileRecordReader* reader, int64_t previous_batch_first_row_number) - : previous_batch_first_row_number_(previous_batch_first_row_number), - reader_(reader), - selection_cardinality_(reader->selection_bitmap_.Cardinality()) {} + explicit Iterator(KeyValueDataFileRecordReader* reader) + : reader_(reader), selection_cardinality_(reader->selection_bitmap_.Cardinality()) {} Result HasNext() const override; Result Next() override; Result> NextWithFilePos(); private: - int64_t previous_batch_first_row_number_; mutable int64_t cursor_ = 0; KeyValueDataFileRecordReader* reader_ = nullptr; int64_t selection_cardinality_ = 0; diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp index 74a3b10f..92ac769c 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader.cpp @@ -118,6 +118,7 @@ Result AvroFileBatchReader::NextBatch() { previous_first_row_ = next_row_to_read_; next_row_to_read_ += array_builder_->length(); if (array_builder_->length() == 0) { + previous_batch_row_count_ = 0; return BatchReader::MakeEofBatch(); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, @@ -125,6 +126,7 @@ Result AvroFileBatchReader::NextBatch() { std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); + previous_batch_row_count_ = c_array->length; return make_pair(std::move(c_array), std::move(c_schema)); } catch (const ::avro::Exception& e) { return Status::Invalid(fmt::format("avro reader next batch failed. {}", e.what())); @@ -170,6 +172,7 @@ Status AvroFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, reader_ = std::move(reader); array_builder_ = std::move(array_builder); previous_first_row_ = std::numeric_limits::max(); + previous_batch_row_count_ = 0; next_row_to_read_ = std::numeric_limits::max(); close_ = false; return Status::OK(); diff --git a/src/paimon/format/avro/avro_file_batch_reader.h b/src/paimon/format/avro/avro_file_batch_reader.h index e54a936e..1bb2452f 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.h +++ b/src/paimon/format/avro/avro_file_batch_reader.h @@ -47,8 +47,20 @@ class AvroFileBatchReader : public FileBatchReader { Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) override; - Result GetPreviousBatchFirstRowNumber() const override { - return previous_first_row_; + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + if (previous_batch_row_count_ == 0) { + if (previous_first_row_ == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet."); + } else { + return Status::Invalid("Last batch was EOF."); + } + } + if (batch_row_id >= previous_batch_row_count_) { + return Status::Invalid( + fmt::format("batch_row_id {} is out of range, last batch row count is {}", + batch_row_id, previous_batch_row_count_)); + } + return previous_first_row_ + batch_row_id; } Result GetNumberOfRows() const override; @@ -92,6 +104,7 @@ class AvroFileBatchReader : public FileBatchReader { std::optional> read_fields_projection_; uint64_t previous_first_row_ = std::numeric_limits::max(); uint64_t next_row_to_read_ = std::numeric_limits::max(); + uint64_t previous_batch_row_count_ = 0; mutable std::optional total_rows_ = std::nullopt; const int32_t batch_size_; bool close_ = false; diff --git a/src/paimon/format/avro/avro_file_batch_reader_test.cpp b/src/paimon/format/avro/avro_file_batch_reader_test.cpp index 13fb7656..0a0a858d 100644 --- a/src/paimon/format/avro/avro_file_batch_reader_test.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader_test.cpp @@ -329,7 +329,7 @@ TEST_F(AvroFileBatchReaderTest, TestSetReadSchemaRejectNestedSubFieldProjection) "does not support nested sub-field projection"); } -TEST_F(AvroFileBatchReaderTest, TestGetPreviousBatchFirstRowNumber) { +TEST_F(AvroFileBatchReaderTest, TestGetPreviousBatchFileRowId) { std::string path = paimon::test::GetDataDir() + "/avro/append_simple.db/" "append_simple/bucket-0/" @@ -354,26 +354,25 @@ TEST_F(AvroFileBatchReaderTest, TestGetPreviousBatchFirstRowNumber) { ASSERT_OK_AND_ASSIGN(auto num_rows, reader->GetNumberOfRows()); ASSERT_EQ(4, num_rows); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch1, reader->NextBatch()); ArrowArrayRelease(batch1.first.get()); ArrowSchemaRelease(batch1.second.get()); - ASSERT_EQ(0, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); ASSERT_OK_AND_ASSIGN(auto batch2, reader->NextBatch()); - ASSERT_EQ(1, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(1, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch2.first.get()); ArrowSchemaRelease(batch2.second.get()); ASSERT_OK_AND_ASSIGN(auto batch3, reader->NextBatch()); - ASSERT_EQ(2, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(2, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch3.first.get()); ArrowSchemaRelease(batch3.second.get()); ASSERT_OK_AND_ASSIGN(auto batch4, reader->NextBatch()); - ASSERT_EQ(3, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(3, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch4.first.get()); ArrowSchemaRelease(batch4.second.get()); ASSERT_OK_AND_ASSIGN(auto batch5, reader->NextBatch()); - ASSERT_EQ(4, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_TRUE(BatchReader::IsEofBatch(batch5)); } @@ -399,7 +398,7 @@ TEST_F(AvroFileBatchReaderTest, TestSetReadSchemaResetsReaderToFirstRow) { ASSERT_OK_AND_ASSIGN(auto reader, reader_builder->Build(in)); ASSERT_OK_AND_ASSIGN(auto first_batch, reader->NextBatch()); - ASSERT_EQ(0, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); auto first_array = arrow::ImportArray(first_batch.first.get(), first_batch.second.get()).ValueOrDie(); ASSERT_TRUE(first_array->Equals(src_array->Slice(0, 2))) << first_array->ToString(); @@ -409,11 +408,10 @@ TEST_F(AvroFileBatchReaderTest, TestSetReadSchemaResetsReaderToFirstRow) { ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); ASSERT_OK(reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto projected_batch, reader->NextBatch()); - ASSERT_EQ(0, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); auto projected_array = arrow::ImportArray(projected_batch.first.get(), projected_batch.second.get()).ValueOrDie(); auto expected_projected_array = arrow::ipc::internal::json::ArrayFromJSON( diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index f74386e0..03bc78a2 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -159,7 +159,7 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, target_type_ = arrow::struct_(arrow_schema->fields()); current_pos_ = 0; previous_batch_first_row_number_ = std::numeric_limits::max(); - + previous_batch_row_count_ = 0; return Status::OK(); } @@ -294,6 +294,7 @@ Result BlobFileBatchReader::NextBatch() { } if (current_pos_ >= target_blob_lengths_.size()) { PAIMON_ASSIGN_OR_RAISE(previous_batch_first_row_number_, GetNumberOfRows()); + previous_batch_row_count_ = 0; return BatchReader::MakeEofBatch(); } int32_t left_rows = target_blob_lengths_.size() - current_pos_; @@ -305,6 +306,7 @@ Result BlobFileBatchReader::NextBatch() { PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*blob_array, c_array.get(), c_schema.get())); previous_batch_first_row_number_ = target_blob_row_indexes_[current_pos_]; current_pos_ += rows_to_read; + previous_batch_row_count_ = c_array->length; return make_pair(std::move(c_array), std::move(c_schema)); } diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h index 159d0552..e55e5944 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.h +++ b/src/paimon/format/blob/blob_file_batch_reader.h @@ -26,6 +26,7 @@ #include "arrow/memory_pool.h" #include "arrow/type.h" +#include "fmt/format.h" #include "paimon/common/data/blob_defs.h" #include "paimon/fs/file_system.h" #include "paimon/memory/bytes.h" @@ -99,14 +100,26 @@ class BlobFileBatchReader : public FileBatchReader { Result NextBatch() override; - Result GetPreviousBatchFirstRowNumber() const override { + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { if (all_blob_lengths_.size() != target_blob_lengths_.size()) { - return Status::Invalid( - "Cannot call GetPreviousBatchFirstRowNumber in BlobFileBatchReader because, after " + return Status::NotImplemented( + "Cannot call GetPreviousBatchFileRowId in BlobFileBatchReader because, after " "bitmap pushdown, rows in the array returned by NextBatch are no longer " "contiguous."); } - return previous_batch_first_row_number_; + if (previous_batch_row_count_ == 0) { + if (previous_batch_first_row_number_ == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet."); + } else { + return Status::Invalid("Last batch was EOF."); + } + } + if (batch_row_id >= previous_batch_row_count_) { + return Status::Invalid( + fmt::format("batch_row_id {} is out of range, last batch row count is {}", + batch_row_id, previous_batch_row_count_)); + } + return previous_batch_first_row_number_ + batch_row_id; } Result GetNumberOfRows() const override { @@ -176,6 +189,7 @@ class BlobFileBatchReader : public FileBatchReader { size_t current_pos_ = 0; uint64_t previous_batch_first_row_number_ = std::numeric_limits::max(); + uint64_t previous_batch_row_count_ = 0; bool closed_ = false; }; diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 0704fd5b..61960e5c 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -171,22 +171,21 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) { ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); ASSERT_EQ(3, number_of_rows); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch1, reader->NextBatch()); ArrowArrayRelease(batch1.first.get()); ArrowSchemaRelease(batch1.second.get()); - ASSERT_EQ(0, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value()); ASSERT_OK_AND_ASSIGN(auto batch2, reader->NextBatch()); - ASSERT_EQ(1, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(1, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch2.first.get()); ArrowSchemaRelease(batch2.second.get()); ASSERT_OK_AND_ASSIGN(auto batch3, reader->NextBatch()); - ASSERT_EQ(2, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_EQ(2, reader->GetPreviousBatchFileRowId(0).value()); ArrowArrayRelease(batch3.first.get()); ArrowSchemaRelease(batch3.second.get()); ASSERT_OK_AND_ASSIGN(auto batch4, reader->NextBatch()); - ASSERT_EQ(3, reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_TRUE(BatchReader::IsEofBatch(batch4)); } @@ -257,8 +256,7 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); ASSERT_EQ(0, number_of_rows); - ASSERT_EQ(std::numeric_limits::max(), - reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch, reader->NextBatch()); ASSERT_TRUE(BatchReader::IsEofBatch(batch)); } diff --git a/src/paimon/format/orc/orc_file_batch_reader.cpp b/src/paimon/format/orc/orc_file_batch_reader.cpp index c96b4e5f..cd627eb5 100644 --- a/src/paimon/format/orc/orc_file_batch_reader.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader.cpp @@ -168,6 +168,7 @@ Status OrcFileBatchReader::SetReadSchema(::ArrowSchema* read_schema, options_, &target_column_ids)); target_column_ids_ = target_column_ids; + previous_batch_row_count_ = 0; PAIMON_RETURN_NOT_OK(reader_->SetReadSchema(target_type, row_reader_options)); return Status::OK(); } @@ -181,7 +182,13 @@ Result>> OrcFileBatchReader::PreBuffer } Result OrcFileBatchReader::NextBatch() { - return reader_->Next(); + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader_->Next()); + if (BatchReader::IsEofBatch(batch)) { + previous_batch_row_count_ = 0; + } else { + previous_batch_row_count_ = batch.first->length; + } + return batch; } std::shared_ptr OrcFileBatchReader::GetReaderMetrics() const { diff --git a/src/paimon/format/orc/orc_file_batch_reader.h b/src/paimon/format/orc/orc_file_batch_reader.h index 5ba4726d..85673a93 100644 --- a/src/paimon/format/orc/orc_file_batch_reader.h +++ b/src/paimon/format/orc/orc_file_batch_reader.h @@ -64,8 +64,21 @@ class OrcFileBatchReader : public PrefetchFileBatchReader { // OrcFileBatchReader. Therefore, we need to hold BatchReader when using output ArrowArray. Result NextBatch() override; - Result GetPreviousBatchFirstRowNumber() const override { - return reader_->GetRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + uint64_t previous_first_row = reader_->GetRowNumber(); + if (previous_batch_row_count_ == 0) { + if (previous_first_row == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet."); + } else { + return Status::Invalid("Last batch was EOF."); + } + } + if (batch_row_id >= previous_batch_row_count_) { + return Status::Invalid( + fmt::format("batch_row_id {} is out of range, last batch row count is {}", + batch_row_id, previous_batch_row_count_)); + } + return previous_first_row + batch_row_id; } Result GetNumberOfRows() const override { @@ -122,6 +135,8 @@ class OrcFileBatchReader : public PrefetchFileBatchReader { std::shared_ptr metrics_; std::vector target_column_ids_; std::vector> cache_ranges_; + + uint64_t previous_batch_row_count_ = 0; }; } // namespace paimon::orc diff --git a/src/paimon/format/orc/orc_file_batch_reader_test.cpp b/src/paimon/format/orc/orc_file_batch_reader_test.cpp index 135e7c5a..038c93e2 100644 --- a/src/paimon/format/orc/orc_file_batch_reader_test.cpp +++ b/src/paimon/format/orc/orc_file_batch_reader_test.cpp @@ -495,14 +495,22 @@ TEST_P(OrcFileBatchReaderTest, TestNextBatchSimple) { for (auto batch_size : {1, 2, 3, 5, 8, 10}) { auto orc_batch_reader = PrepareOrcFileBatchReader(file_name, &read_schema, batch_size, natural_read_size); - ASSERT_EQ(std::numeric_limits::max(), - orc_batch_reader->GetPreviousBatchFirstRowNumber().value()); - ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( - orc_batch_reader.get())); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 8); + ASSERT_NOK(orc_batch_reader->GetPreviousBatchFileRowId(0)); + int i = 0; + while (true) { + ASSERT_OK_AND_ASSIGN( + auto result_array, + paimon::test::ReadResultCollector::CollectResultOneBatch(orc_batch_reader.get())); + if (!result_array) { + ASSERT_NOK(orc_batch_reader->GetPreviousBatchFileRowId(0)); + break; + } + ASSERT_EQ(orc_batch_reader->GetPreviousBatchFileRowId(0).value(), i * batch_size); + ASSERT_TRUE(result_array->Equals(std::make_shared( + struct_array_->Slice(i * batch_size, result_array->length())))); + i++; + } orc_batch_reader->Close(); - auto expected_array = std::make_shared(struct_array_); - ASSERT_TRUE(result_array->Equals(expected_array)); // test metrics auto reader_metrics = orc_batch_reader->GetReaderMetrics(); ASSERT_OK_AND_ASSIGN(uint64_t io_count, @@ -770,19 +778,18 @@ TEST_F(OrcFileBatchReaderTest, TestReadNoField) { auto orc_batch_reader = PrepareOrcFileBatchReader(file_name, &read_schema, /*batch_size=*/3, /*natural_read_size=*/10); // read 3 rows - ASSERT_EQ(std::numeric_limits::max(), - orc_batch_reader->GetPreviousBatchFirstRowNumber().value()); + ASSERT_NOK(orc_batch_reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch1, orc_batch_reader->NextBatch()); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 0); + ASSERT_EQ(orc_batch_reader->GetPreviousBatchFileRowId(0).value(), 0); // read 3 rows ASSERT_OK_AND_ASSIGN(auto batch2, orc_batch_reader->NextBatch()); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 3); + ASSERT_EQ(orc_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); // read 2 rows ASSERT_OK_AND_ASSIGN(auto batch3, orc_batch_reader->NextBatch()); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 6); + ASSERT_EQ(orc_batch_reader->GetPreviousBatchFileRowId(0).value(), 6); // read rows with eof ASSERT_OK_AND_ASSIGN(auto batch4, orc_batch_reader->NextBatch()); - ASSERT_EQ(orc_batch_reader->GetPreviousBatchFirstRowNumber().value(), 8); + ASSERT_NOK(orc_batch_reader->GetPreviousBatchFileRowId(0)); ASSERT_TRUE(BatchReader::IsEofBatch(batch4)); orc_batch_reader->Close(); diff --git a/src/paimon/format/orc/orc_reader_wrapper.cpp b/src/paimon/format/orc/orc_reader_wrapper.cpp index 20956063..f1482690 100644 --- a/src/paimon/format/orc/orc_reader_wrapper.cpp +++ b/src/paimon/format/orc/orc_reader_wrapper.cpp @@ -51,6 +51,7 @@ Status OrcReaderWrapper::SetReadSchema(const std::shared_ptr& t try { row_reader_ = reader_->createRowReader(row_reader_options); target_type_ = target_type; + next_row_ = 0; } catch (const std::exception& e) { return Status::Invalid( fmt::format("orc file batch reader create row reader failed for file {}, with {} error", diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 32ec420c..83ae593c 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -41,6 +41,7 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/core/utils/nested_projection_utils.h" @@ -215,11 +216,13 @@ Status ParquetFileBatchReader::SetReadSchema( target_row_groups.emplace_back(/*rg_index=*/rg_id, /*is_partially_matched=*/true, /*ranges=*/it->second); } else { - target_row_groups.emplace_back(/*rg_index=*/rg_id, - /*is_partially_matched=*/false, - /*ranges=*/RowRanges()); + target_row_groups.emplace_back( + /*rg_index=*/rg_id, /*is_partially_matched=*/false, /*ranges=*/ + RowRanges(Range(0, reader_->GetAllRowGroupRanges()[rg_id].second - + reader_->GetAllRowGroupRanges()[rg_id].first - 1))); } } + PAIMON_RETURN_NOT_OK(UpdateAllTargetRowRanges(target_row_groups)); PAIMON_RETURN_NOT_OK(reader_->PrepareForReadingLazy(target_row_groups, column_indices)); } PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetFileBatchReader::SetReadSchema") @@ -345,6 +348,7 @@ Result ParquetFileBatchReader::NextBatch() { try { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, reader_->Next()); if (batch == nullptr) { + row_mapping_.clear(); return BatchReader::MakeEofBatch(); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, @@ -363,6 +367,7 @@ Result ParquetFileBatchReader::NextBatch() { "equal with read schema {}", array->type()->ToString(), read_data_type_->ToString())); } + PAIMON_RETURN_NOT_OK(GenerateRowMapping(array->length())); std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); @@ -528,4 +533,54 @@ Result> ParquetFileBatchReader::ComputeNestedColumnIndices( return indices; } +Status ParquetFileBatchReader::UpdateAllTargetRowRanges( + const std::vector& target_row_groups) { + row_mapping_.clear(); + auto all_row_group_ranges = reader_->GetAllRowGroupRanges(); + RowRanges all_ranges; + for (const auto& target_row_group : target_row_groups) { + for (const auto& range : target_row_group.row_ranges.GetRanges()) { + all_ranges.Add( + Range(range.from + all_row_group_ranges[target_row_group.row_group_index].first, + range.to + all_row_group_ranges[target_row_group.row_group_index].first)); + } + } + all_row_ranges_ = std::move(all_ranges); + return Status::OK(); +} + +Status ParquetFileBatchReader::GenerateRowMapping(int64_t batch_length) { + const std::vector& all_ranges = all_row_ranges_.GetRanges(); + PAIMON_ASSIGN_OR_RAISE(int64_t batch_start_row, reader_->GetPreviousBatchFirstRowNumber()); + + auto cur_range_it = + std::upper_bound(all_ranges.begin(), all_ranges.end(), batch_start_row, + [](int64_t value, const Range& r) { return value < r.from; }); + if (cur_range_it == all_ranges.begin()) { + return Status::Invalid("No range found!"); + } + --cur_range_it; + if (batch_start_row < cur_range_it->from || batch_start_row > cur_range_it->to) { + return Status::Invalid( + fmt::format("Batch start row {} is not in the current range [{}, {}]!", batch_start_row, + cur_range_it->from, cur_range_it->to)); + } + + std::vector row_mapping; + row_mapping.reserve(batch_length); + int64_t global_row = batch_start_row; + for (int64_t i = 0; i < batch_length; ++i) { + if (global_row > cur_range_it->to) { + ++cur_range_it; + if (cur_range_it == all_ranges.end()) { + return Status::Invalid("Batch length exceeds the total row ranges!"); + } + global_row = cur_range_it->from; + } + row_mapping.push_back(global_row); + global_row++; + } + row_mapping_ = std::move(row_mapping); + return Status::OK(); +} } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 592c128e..4ba2aca3 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -18,6 +18,8 @@ #pragma once +#include + #include #include #include @@ -96,9 +98,22 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { Result>> GenReadRanges( bool* need_prefetch) const override; - Result GetPreviousBatchFirstRowNumber() const override { - assert(reader_); - return reader_->GetPreviousBatchFirstRowNumber(); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + if (row_mapping_.empty()) { + PAIMON_ASSIGN_OR_RAISE(uint64_t previous_first_row, + reader_->GetPreviousBatchFirstRowNumber()); + if (previous_first_row == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet."); + } else { + return Status::Invalid("Last batch was EOF."); + } + } + if (batch_row_id >= row_mapping_.size()) { + return Status::Invalid( + fmt::format("batch_row_id {} is out of range, last batch row count is {}", + batch_row_id, row_mapping_.size())); + } + return row_mapping_[batch_row_id]; } Result GetNumberOfRows() const override { @@ -175,6 +190,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { const std::shared_ptr& read_schema, const std::shared_ptr& file_schema); + Status UpdateAllTargetRowRanges(const std::vector& target_row_groups); + // precondition: predicate supposed not be empty Result> FilterRowGroupsByPredicate( const std::shared_ptr& predicate, @@ -191,6 +208,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { const std::map& column_name_to_index, const std::vector& src_row_groups); + Status GenerateRowMapping(int64_t batch_length); + private: std::map options_; // hold the lifecycle of arrow memory pool. @@ -206,6 +225,9 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { uint64_t read_rows_ = 0; uint64_t read_batch_count_ = 0; + + RowRanges all_row_ranges_; + std::vector row_mapping_; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index 61a7caef..417c2aea 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -170,12 +170,15 @@ class ParquetFileBatchReaderTest : public ::testing::Test, void WriteArray(const std::string& file_path, const std::shared_ptr& src_array, const std::shared_ptr& arrow_schema, int64_t write_batch_size, - bool enable_dictionary, int64_t max_row_group_length) const { + bool enable_dictionary, int64_t max_row_group_length, + int64_t max_page_size = 1024 * 1024 * 1024) const { ASSERT_OK_AND_ASSIGN(std::shared_ptr out, fs_->Create(file_path, /*overwrite=*/true)); ::parquet::WriterProperties::Builder builder; builder.write_batch_size(write_batch_size); builder.max_row_group_length(max_row_group_length); + builder.data_pagesize(max_page_size); + builder.enable_write_page_index(); enable_dictionary ? builder.enable_dictionary() : builder.disable_dictionary(); auto writer_properties = builder.build(); ASSERT_OK_AND_ASSIGN(auto format_writer, ParquetFormatWriter::Create( @@ -231,6 +234,17 @@ class ParquetFileBatchReaderTest : public ::testing::Test, std::shared_ptr struct_array_; }; +static std::shared_ptr MakeSequentialIntData(int32_t num_rows) { + arrow::Int32Builder val_builder; + EXPECT_TRUE(val_builder.Reserve(num_rows).ok()); + for (int32_t i = 0; i < num_rows; ++i) { + val_builder.UnsafeAppend(i); + } + auto val_array = val_builder.Finish().ValueOrDie(); + auto field = arrow::field("f0", arrow::int32()); + return arrow::StructArray::Make({val_array}, {field}).ValueOrDie(); +} + TEST_F(ParquetFileBatchReaderTest, TestParquetMetadataCacheReusesSerializedFooter) { WriteArray(file_path_, struct_array_, schema_, /*write_batch_size=*/struct_array_->length(), /*enable_dictionary=*/false, @@ -449,11 +463,8 @@ TEST_F(ParquetFileBatchReaderTest, TestNextBatchSimple) { auto parquet_batch_reader = PrepareParquetFileBatchReader(file_name, schema_, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt, batch_size); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), - std::numeric_limits::max()); ASSERT_OK_AND_ASSIGN(auto result_array, paimon::test::ReadResultCollector::CollectResult( parquet_batch_reader.get())); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 6); parquet_batch_reader->Close(); auto expected_array = std::make_shared(struct_array_); ASSERT_TRUE(result_array->Equals(expected_array)); @@ -814,20 +825,19 @@ TEST_F(ParquetFileBatchReaderTest, TestReadNoField) { PrepareParquetFileBatchReader(file_name, read_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt, /*batch_size=*/2); // read 2 rows - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), - std::numeric_limits::max()); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN(auto batch1, parquet_batch_reader->NextBatch()); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 0); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 0); // read 2 rows ASSERT_OK_AND_ASSIGN(auto batch2, parquet_batch_reader->NextBatch()); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 2); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 2); // read 2 rows ASSERT_OK_AND_ASSIGN(auto batch3, parquet_batch_reader->NextBatch()); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 4); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 4); // read rows with eof ASSERT_OK_AND_ASSIGN(auto batch4, parquet_batch_reader->NextBatch()); - ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFirstRowNumber().value(), 6); ASSERT_TRUE(BatchReader::IsEofBatch(batch4)); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); parquet_batch_reader->Close(); arrow::FieldVector fields; @@ -1015,4 +1025,154 @@ TEST_F(ParquetFileBatchReaderTest, TestAddMetadataPerFieldMetadata) { ASSERT_TRUE(data->Equals(*result_array->chunk(0))) << result_array->ToString(); } +TEST_F(ParquetFileBatchReaderTest, TestRowMappingSimple) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto src_array = MakeSequentialIntData(12); + // data in file rowGroup0:[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + // one row per page + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1, + /*enable_dictionary=*/true, /*max_row_group_length=*/12, /*max_page_size=*/1); + + // 1<=f0<=3 || 5<=f0<=6 + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(1), Literal(3)), + PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(5), Literal(6))})); + + auto parquet_batch_reader = PrepareParquetFileBatchReader( + file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/2); + + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch1, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + auto expected_batch1 = src_array->Slice(1, 2); + ASSERT_TRUE(batch1->chunk(0)->Equals(expected_batch1)) << batch1->ToString(); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 1); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(1).value(), 2); + // out of bound return invalid + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(2)); + + // Not adjacent pages + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch2, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + auto expected_batch2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +[3], +[5] + ])") + .ValueOrDie()); + ASSERT_TRUE(batch2->chunk(0)->Equals(expected_batch2)) << batch2->ToString(); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(1).value(), 5); + + // Only one record read + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch3, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + auto expected_batch3 = src_array->Slice(6, 1); + ASSERT_TRUE(batch3->chunk(0)->Equals(expected_batch3)) << batch3->ToString(); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 6); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(1)); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr eof_batch, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(nullptr, eof_batch); + // previous batch is eof, return invalid. + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); +} + +TEST_F(ParquetFileBatchReaderTest, TestRowMappingFullyAndPartially) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto src_array = MakeSequentialIntData(12); + // data in file RowGroup0:[0, 1, 2] | RowGroup1:[3, 4, 5] | RowGroup2:[6, 7, 8] | RowGroup3:[9, + // 10, 11] one row per page + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1, + /*enable_dictionary=*/true, /*max_row_group_length=*/3, /*max_page_size=*/1); + + // 3<=f0<=5 || f0==6 || f0==8 + // RowGroup 1 is fully matched, RowGroup 2 is partially matched, RowGroup 0 and RowGroup 3 are + // not matched. + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(3), Literal(5)), + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(6)), + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(8))})); + + auto parquet_batch_reader = PrepareParquetFileBatchReader( + file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3); + + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch1, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(2).value(), 5); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch2, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 6); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(1).value(), 8); +} + +TEST_F(ParquetFileBatchReaderTest, TestRowMappingSetReadSchemaTwice) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto src_array = MakeSequentialIntData(12); + // data in file RowGroup0:[0, 1, 2] | RowGroup1:[3, 4, 5] | RowGroup2:[6, 7, 8] | RowGroup3:[9, + // 10, 11] one row per page + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1, + /*enable_dictionary=*/true, /*max_row_group_length=*/3, /*max_page_size=*/1); + + // 1<=f0<=3 || 6<=f0<=7 + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(1), Literal(3)), + PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(6), Literal(7))})); + + auto parquet_batch_reader = PrepareParquetFileBatchReader( + file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3); + + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch1, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 1); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(1).value(), 2); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch2, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(1)); + + ASSERT_OK_AND_ASSIGN( + predicate, + PredicateBuilder::Or({PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(3), Literal(5))})); + + std::unique_ptr c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*arrow_schema, c_schema.get()).ok()); + ASSERT_OK( + parquet_batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/predicate, std::nullopt)); + ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr batch3, + paimon::test::ReadResultCollector::CollectResultOneBatch(parquet_batch_reader.get())); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(0).value(), 3); + ASSERT_EQ(parquet_batch_reader->GetPreviousBatchFileRowId(2).value(), 5); +} + } // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/row_ranges.h b/src/paimon/format/parquet/row_ranges.h index 05edec20..9a8547fe 100644 --- a/src/paimon/format/parquet/row_ranges.h +++ b/src/paimon/format/parquet/row_ranges.h @@ -112,7 +112,7 @@ class RowRanges { struct TargetRowGroup { int32_t row_group_index{-1}; bool is_partially_matched{false}; - // page-filtered row ranges, only valid if is_partially_matched is true. + RowRanges row_ranges; // Whether this row group has been excluded by ApplyReadRanges. // When true, this row group is logically skipped during iteration diff --git a/src/paimon/testing/mock/mock_file_batch_reader.h b/src/paimon/testing/mock/mock_file_batch_reader.h index b50e053a..f05a2347 100644 --- a/src/paimon/testing/mock/mock_file_batch_reader.h +++ b/src/paimon/testing/mock/mock_file_batch_reader.h @@ -158,8 +158,11 @@ class MockFileBatchReader : public PrefetchFileBatchReader { return metrics; } - Result GetPreviousBatchFirstRowNumber() const override { - return ToReaderRowNumber(previous_batch_first_row_num_); + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override { + if (previous_batch_first_row_num_ == std::numeric_limits::max()) { + return Status::Invalid("No batch has been read yet"); + } + return previous_batch_first_row_num_ + batch_row_id; } Result GetNumberOfRows() const override { @@ -193,7 +196,7 @@ class MockFileBatchReader : public PrefetchFileBatchReader { int32_t batch_size_ = 0; int32_t current_pos_ = 0; int32_t read_end_pos_ = 0; - int32_t previous_batch_first_row_num_ = -1; + uint64_t previous_batch_first_row_num_ = std::numeric_limits::max(); Status next_batch_status_; bool enable_randomize_batch_size_ = true; std::vector> read_ranges_; diff --git a/src/paimon/testing/utils/read_result_collector.h b/src/paimon/testing/utils/read_result_collector.h index bbfc21ab..bd8347d8 100644 --- a/src/paimon/testing/utils/read_result_collector.h +++ b/src/paimon/testing/utils/read_result_collector.h @@ -70,10 +70,6 @@ class ReadResultCollector { return results; } - static Result> CollectResult(BatchReader* batch_reader) { - return CollectResult(batch_reader, /*max simulated data processing time*/ 0); - } - // will convert dictionary array to string array for comparing results static Result> CollectResult( BatchReader* batch_reader, int64_t max_data_processing_time_in_us) { @@ -81,35 +77,10 @@ class ReadResultCollector { int64_t seed = DateTimeUtils::GetCurrentUTCTimeUs(); std::srand(seed); while (true) { - // Prioritize calling NextBatch. If it fails (paimon inner reader e.g., - // PrefetchBatchReader, ApplyBitmapIndexBatchReader...), call NextBatchWithBitmap. - auto batch_result = batch_reader->NextBatch(); - BatchReader::ReadBatch batch; - if (!batch_result.ok()) { - if (batch_result.status().ToString().find("should use NextBatchWithBitmap") != - std::string::npos) { - PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, - batch_reader->NextBatchWithBitmap()); - if (BatchReader::IsEofBatch(batch_with_bitmap)) { - break; - } - assert(!batch_with_bitmap.second.IsEmpty()); - PAIMON_ASSIGN_OR_RAISE( - batch, ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap), - arrow::default_memory_pool())); - } else { - return batch_result.status(); - } - } else { - batch = std::move(batch_result).value(); - if (BatchReader::IsEofBatch(batch)) { - break; - } + PAIMON_ASSIGN_OR_RAISE(auto result_array, ReadOneBatch(batch_reader)); + if (result_array == nullptr) { + break; } - auto& [c_array, c_schema] = batch; - assert(c_array->length > 0); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto result_array, - arrow::ImportArray(c_array.get(), c_schema.get())); result_array_vector.push_back(result_array); if (max_data_processing_time_in_us > 0) { usleep(std::rand() % max_data_processing_time_in_us); @@ -133,6 +104,35 @@ class ReadResultCollector { return chunk_array; } + static Result> CollectResult(BatchReader* batch_reader) { + return CollectResult(batch_reader, /*max_data_processing_time_in_us=*/0); + } + + static Result> CollectResultOneBatch( + BatchReader* batch_reader) { + return CollectResultOneBatch(batch_reader, /*max_data_processing_time_in_us=*/0); + } + + static Result> CollectResultOneBatch( + BatchReader* batch_reader, int64_t max_data_processing_time_in_us) { + int64_t seed = DateTimeUtils::GetCurrentUTCTimeUs(); + std::srand(seed); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result_array, + ReadOneBatch(batch_reader)); + if (result_array == nullptr) { + return std::shared_ptr(); + } + PAIMON_ASSIGN_OR_RAISE( + auto converted_array, + DictArrayConverter::ConvertDictArray(result_array, arrow::default_memory_pool())); + if (max_data_processing_time_in_us > 0) { + usleep(std::rand() % max_data_processing_time_in_us); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr chunk_array, + arrow::ChunkedArray::Make({converted_array})); + return chunk_array; + } + static Result> GetArray(BatchReader::ReadBatch&& batch) { if (BatchReader::IsEofBatch(batch)) { return std::shared_ptr(); @@ -167,5 +167,39 @@ class ReadResultCollector { arrow::compute::Take(arrow::Datum(array), arrow::Datum(sorted_indices))); return sorted_batch.chunked_array(); } + + private: + static Result> ReadOneBatch(BatchReader* batch_reader) { + // Prioritize calling NextBatch. If it fails (paimon inner reader e.g., + // PrefetchBatchReader, ApplyBitmapIndexBatchReader...), call NextBatchWithBitmap. + auto batch_result = batch_reader->NextBatch(); + BatchReader::ReadBatch batch; + if (!batch_result.ok()) { + if (batch_result.status().ToString().find("should use NextBatchWithBitmap") != + std::string::npos) { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, + batch_reader->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + return std::shared_ptr(); + } + assert(!batch_with_bitmap.second.IsEmpty()); + PAIMON_ASSIGN_OR_RAISE( + batch, ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap), + arrow::default_memory_pool())); + } else { + return batch_result.status(); + } + } else { + batch = std::move(batch_result).value(); + if (BatchReader::IsEofBatch(batch)) { + return std::shared_ptr(); + } + } + auto& [c_array, c_schema] = batch; + assert(c_array->length > 0); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto result_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + return result_array; + } }; } // namespace paimon::test From 6ec37aed01dcf6f904310f13813eaa1396529694 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Thu, 2 Jul 2026 20:17:38 +0800 Subject: [PATCH 080/138] test: add tests for executor From 8804191fc026e59f001de278ea74f1de26e92716 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:50:50 +0800 Subject: [PATCH 081/138] fix: release global index writer before reader to avoid mem issue when write index failed From 7ed1ae74e8d91cdab081b69d34d9635e6596da3c Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:57:45 -0700 Subject: [PATCH 082/138] fix: fix IsThreadSafe() in UnionGlobalIndexReader From ff0ad88adc4359b3289df96bdd91e8e03c56be6e Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:37:54 +0800 Subject: [PATCH 083/138] test: add map shared-shredding tests for compaction and alter table and predicates --- .../core/schema/schema_validation_test.cpp | 20 + test/inte/append_compaction_inte_test.cpp | 247 ++++++ test/inte/data_evolution_table_test.cpp | 154 +++- test/inte/nested_column_pruning_inte_test.cpp | 536 ++++-------- test/inte/pk_compaction_inte_test.cpp | 281 +++++- test/inte/write_and_read_inte_test.cpp | 815 ++++++++++++++++++ 6 files changed, 1678 insertions(+), 375 deletions(-) diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index fccf6729..85a3e733 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -913,6 +913,26 @@ TEST(SchemaValidationTest, TestMapStorageLayout) { ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), "not MAP"); } + // Invalid: nested MAP paths are not shared-shredding columns; only top-level columns are + // addressable by fields..map.storage-layout. + { + auto payload = arrow::field( + "payload", + arrow::struct_({arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64()))})); + arrow::FieldVector fields = {f0, f1, payload}; + auto schema = arrow::schema(fields); + std::map options = { + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f0"}, + {"fields.payload.attrs.map.storage-layout", "shared-shredding"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0", "f1"}, options)); + ASSERT_NOK_WITH_MSG( + SchemaValidation::ValidateTableSchema(*table_schema), + "Column 'payload.attrs' is configured with map.storage-layout but does not exist in " + "table schema."); + } // Valid: default layout on a MAP column { arrow::FieldVector fields = {f0, f1, f2}; diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 418a0e96..208824e9 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -21,11 +21,15 @@ #include #include +#include "arrow/api.h" #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/commit_context.h" #include "paimon/common/data/binary_row.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/append/bucketed_append_compact_manager.h" #include "paimon/core/io/data_file_meta.h" @@ -36,10 +40,14 @@ #include "paimon/executor.h" #include "paimon/file_store_commit.h" #include "paimon/file_store_write.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/read_context.h" #include "paimon/result.h" +#include "paimon/table/source/table_read.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/data_generator.h" #include "paimon/testing/utils/io_exception_helper.h" +#include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" #include "paimon/write_context.h" @@ -244,6 +252,245 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) { } } +TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMapSharedShredding) { + auto file_format = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + auto schema = arrow::schema(fields); + + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "64"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + + ASSERT_OK_AND_ASSIGN(auto batch_0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + int64_t commit_identifier = 0; + ASSERT_OK(helper->WriteAndCommit(std::move(batch_0), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", 40], ["d", 50]]], + [4, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [5, [["e", 60], ["f", 70], ["g", 80], ["h", 90]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK(helper->write_->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN( + std::vector> commit_messages, + helper->write_->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + ASSERT_FALSE(commit_messages.empty()); + ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.value().GetCommitKind()); + + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_EQ(data_splits.size(), 1); + { + // check adaptive k + auto data_split = std::dynamic_pointer_cast(data_splits[0]); + ASSERT_TRUE(data_split); + ASSERT_EQ(data_split->DataFiles().size(), 1); + auto compact_file = data_split->DataFiles()[0]; + std::string compact_file_path = + PathUtil::JoinPath(data_split->BucketPath(), compact_file->file_name); + ASSERT_OK_AND_ASSIGN(auto unique_input_stream, + dir->GetFileSystem()->Open(compact_file_path)); + std::shared_ptr input_stream(std::move(unique_input_stream)); + ASSERT_OK_AND_ASSIGN(auto file_format_obj, FileFormatFactory::Get(file_format, options)); + ASSERT_OK_AND_ASSIGN(auto reader_builder, file_format_obj->CreateReaderBuilder(10)); + ASSERT_OK_AND_ASSIGN(auto reader, reader_builder->Build(input_stream)); + ASSERT_OK_AND_ASSIGN(auto c_file_schema, reader->GetFileSchema()); + auto file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); + auto tags_field = file_schema->GetFieldByName("tags"); + ASSERT_TRUE(tags_field); + ASSERT_TRUE(tags_field->metadata()); + ASSERT_OK_AND_ASSIGN( + auto tags_meta, + MapSharedShreddingUtils::DeserializeMetadata( + tags_field->metadata()->Copy(), MapSharedShreddingDefine::kDefaultDictCompression)); + ASSERT_EQ(4, tags_meta.num_columns); + ASSERT_EQ(4, tags_meta.max_row_width); + } + { + // recall all fields + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(data_type, data_splits, + R"([ + [0, 1, [["a", 10], ["b", 20]]], + [0, 2, [["c", 30]]], + [0, 3, [["a", 40], ["d", 50]]], + [0, 4, null], + [0, 5, [["e", 60], ["f", 70], ["g", 80], ["h", 90]]] + ])")); + ASSERT_TRUE(success); + } + { + // recall only "a,f" sub-key in map + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,f"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + + ReadContextBuilder read_context_builder(PathUtil::JoinPath(dir->Str(), "foo.db/bar")); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(expected_type, R"([ + [0, 1, [["a", 10]]], + [0, 2, []], + [0, 3, [["a", 40]]], + [0, 4, null], + [0, 5, [["f", 70]]] + ])") + .ValueOrDie(); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(expected_chunked->Equals(actual)) + << "actual=" << actual->ToString() << "\nexpected=" << expected_chunked->ToString(); + } +} + +TEST_P(AppendCompactionInteTest, + TestOrcAppendTableFullCompactionWithMapSharedShreddingStringValue) { + auto file_format = GetParam(); + if (file_format != "orc") { + return; + } + + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto map_type = arrow::map(arrow::utf8(), arrow::utf8()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + auto schema = arrow::schema(fields); + + std::map options = { + {Options::FILE_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, "local"}, + {"orc.read.enable-lazy-decoding", "true"}, + {"orc.dictionary-key-size-threshold", "1"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto batch_0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", "shared"], ["b", "hot"]]], + [2, [["c", "shared"]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_0), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["a", "shared"], ["d", "hot"]]], + [4, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [5, [["e", "shared"], ["f", "hot"], ["g", "shared"]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK(helper->write_->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN( + std::vector> commit_messages, + helper->write_->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + ASSERT_FALSE(commit_messages.empty()); + ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.value().GetCommitKind()); + + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_EQ(data_splits.size(), 1); + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(data_type, data_splits, + R"([ + [0, 1, [["a", "shared"], ["b", "hot"]]], + [0, 2, [["c", "shared"]]], + [0, 3, [["a", "shared"], ["d", "hot"]]], + [0, 4, null], + [0, 5, [["e", "shared"], ["f", "hot"], ["g", "shared"]]] + ])")); + ASSERT_TRUE(success); +} + TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithDv) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index f6a7b9b6..e75b8091 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -19,6 +19,7 @@ #include "gtest/gtest.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" @@ -48,9 +49,10 @@ class DataEvolutionTableTest : public ::testing::Test, dir_.reset(); } - void CreateTable(const std::vector& partition_keys, + void CreateTable(const arrow::FieldVector& fields, + const std::vector& partition_keys, const std::map& options) const { - auto schema = arrow::schema(fields_); + auto schema = arrow::schema(fields); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -61,6 +63,11 @@ class DataEvolutionTableTest : public ::testing::Test, /*ignore_if_exists=*/false)); } + void CreateTable(const std::vector& partition_keys, + const std::map& options) const { + CreateTable(fields_, partition_keys, options); + } + void CreateTable(const std::vector& partition_keys) const { std::map options = {{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, @@ -512,6 +519,149 @@ TEST_P(DataEvolutionTableTest, TestOnlySomeColumns) { } } +TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) { + if (GetParam() != "parquet" && GetParam() != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("map1", map_type), + arrow::field("map2", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {"fields.map1.map.storage-layout", "shared-shredding"}, + {"fields.map1.map.shared-shredding.max-columns", "1"}, + {"fields.map2.map.storage-layout", "shared-shredding"}, + {"fields.map2.map.shared-shredding.max-columns", "1"}, + }; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + std::vector write_cols0 = {"id", "map1"}; + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[1]}), R"([ + [1, [["a", 10], ["b", 20]]], + [11, [["a", 11], ["b", 21]]] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, write_cols0, src_array0)); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + std::vector write_cols1 = {"id", "map2"}; + auto src_array1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[2]}), R"([ + [2, [["c", 30], ["d", 40]]], + [12, [["c", 31], ["d", 41]]] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, write_cols1, src_array1)); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + std::vector write_cols2 = {"map1"}; + auto src_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[1]}), R"([ + [[["b", 200], ["a", 100]]], + [[["b", 201], ["a", 101]]] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, write_cols2, src_array2)); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + // Read all columns and merge values from all partial files. + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [2, [["a", 100], ["b", 200]], [["c", 30], ["d", 40]]], + [12, [["a", 101], ["b", 201]], [["c", 31], ["d", 41]]] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // Read a subset of columns and recall only the requested shared-shredding MAP column. + auto expected_column_pruned_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[0], fields[2]}), R"([ + [2, [["c", 30], ["d", 40]]], + [12, [["c", 31], ["d", 41]]] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"id", "map2"}, expected_column_pruned_array)); + + // Read selected keys from both shared-shredding MAP columns after partial overwrite merge. + { + auto map1_selected_keys = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"b"}); + auto map2_selected_keys = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"d"}); + auto read_schema = arrow::schema({ + fields[0], + fields[1]->WithMetadata(map1_selected_keys), + fields[2]->WithMetadata(map2_selected_keys), + }); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetReadSchema(std::move(c_schema)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + + auto expected_type = arrow::struct_({ + SpecialFields::ValueKind().field_, + fields[0], + fields[1], + fields[2], + }); + auto expected = arrow::ipc::internal::json::ArrayFromJSON(expected_type, R"([ + [0, 2, [["b", 200]], [["d", 40]]], + [0, 12, [["b", 201]], [["d", 41]]] + ])") + .ValueOrDie(); + auto expected_chunked = std::make_shared(expected); + ASSERT_TRUE(expected_chunked->Equals(actual)) + << "actual=" << actual->ToString() << "\nexpected=" << expected_chunked->ToString(); + } + + // Read a subset of rows after merging values from all partial files. + auto expected_partial_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [12, [["a", 101], ["b", 201]], [["c", 31], ["d", 41]]] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_partial_array, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1l, 1l)})); + + // Read row tracking fields and verify the latest partial overwrite sequence number. + auto expected_row_tracking_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_, + SpecialFields::SequenceNumber().field_}), + R"([ + [2, [["a", 100], ["b", 200]], [["c", 30], ["d", 40]], 0, 3], + [12, [["a", 101], ["b", 201]], [["c", 31], ["d", 41]], 1, 3] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"id", "map1", "map2", "_ROW_ID", "_SEQUENCE_NUMBER"}, + expected_row_tracking_array)); +} + TEST_P(DataEvolutionTableTest, TestNullValues) { CreateTable(); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp index 19a08fb9..468e87ce 100644 --- a/test/inte/nested_column_pruning_inte_test.cpp +++ b/test/inte/nested_column_pruning_inte_test.cpp @@ -29,17 +29,21 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" +#include "paimon/commit_context.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" #include "paimon/fs/file_system_factory.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" +#include "paimon/record_batch.h" #include "paimon/result.h" #include "paimon/scan_context.h" #include "paimon/status.h" @@ -50,6 +54,7 @@ #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" namespace paimon { class DataSplit; @@ -83,6 +88,40 @@ class NestedColumnPruningInteTest : public ::testing::Test, ASSERT_TRUE(is_equal); } + void ScanReadAndCheck(const std::string& table_path, + const std::shared_ptr& expected_schema, + const std::string& expected_json, + const std::shared_ptr& predicate = nullptr) const { + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + if (predicate) { + scan_context_builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_FALSE(result_plan->Splits().empty()); + + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*expected_schema, c_schema.get()).ok()); + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetReadSchema(std::move(c_schema)); + if (predicate) { + read_context_builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector expected_fields = expected_schema->fields(); + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_type = arrow::struct_(expected_fields); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_json).ValueOrDie()); + AssertChunkedArrayEquals(expected, actual); + } + protected: std::string file_format_; std::string test_dir_; @@ -130,11 +169,6 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFields) { helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - // Scan to get splits - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_FALSE(data_splits.empty()); - // Build projected schema: only read f0 (full) and f1.a (sub-field of struct) auto pruned_struct_type = arrow::struct_({ arrow::field("a", arrow::int32()), @@ -145,36 +179,12 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFields) { }; auto projected_schema = arrow::schema(projected_fields); - // Export to C ArrowSchema - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - // Read with projected schema - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - // Expected: struct with _VALUE_KIND, f0, f1{a} - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f1", arrow::struct_({arrow::field("a", arrow::int32())})), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [10]], [0, 2, [20]], [0, 3, [30]], [0, 4, [40]] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: Projecting a STRUCT column as empty struct should return this column @@ -214,10 +224,6 @@ TEST_P(NestedColumnPruningInteTest, ProjectStructColumnAsEmptyStructReturnsNullC helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_FALSE(data_splits.empty()); - // Project f1 as empty struct. arrow::FieldVector projected_fields = { arrow::field("f0", arrow::int32()), @@ -225,31 +231,11 @@ TEST_P(NestedColumnPruningInteTest, ProjectStructColumnAsEmptyStructReturnsNullC }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - auto expected_type = arrow::struct_({ - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f1", arrow::struct_({})), - }); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, null], [0, 2, null], [0, 3, null] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: Two top-level struct columns have the same nested field name; projection should @@ -295,10 +281,6 @@ TEST_P(NestedColumnPruningInteTest, PruneSameNestedFieldNameFromDifferentStructC helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_FALSE(data_splits.empty()); - // Project only s0.f1 and s1.f1; both nested field names are identical. auto projected_s0 = arrow::struct_({arrow::field("f1", arrow::int32())}); auto projected_s1 = arrow::struct_({arrow::field("f1", arrow::int32())}); @@ -309,33 +291,11 @@ TEST_P(NestedColumnPruningInteTest, PruneSameNestedFieldNameFromDifferentStructC }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("s0", arrow::struct_({arrow::field("f1", arrow::int32())})), - arrow::field("s1", arrow::struct_({arrow::field("f1", arrow::int32())})), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [11], [101]], [0, 2, [22], [202]], [0, 3, [33], [303]] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: Querying only non-existent struct sub-fields should fail fast. @@ -677,9 +637,6 @@ TEST_P(NestedColumnPruningInteTest, PruneEntireStructField) { helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - // Only read f0 and f2, skip f1 entirely. arrow::FieldVector projected_fields = { arrow::field("f0", arrow::int32()), @@ -687,32 +644,11 @@ TEST_P(NestedColumnPruningInteTest, PruneEntireStructField) { }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f2", arrow::float64()), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 100, 0.1], [0, 200, 0.2], [0, 300, 0.3] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: Nested struct — prune sub-fields of a struct inside another struct. @@ -756,9 +692,6 @@ TEST_P(NestedColumnPruningInteTest, PruneDeepNestedStruct) { helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - // Projected: f0, f1{inner{x}} — skip f1.a and f1.inner.y auto pruned_inner = arrow::struct_({ arrow::field("x", arrow::int64()), @@ -772,36 +705,11 @@ TEST_P(NestedColumnPruningInteTest, PruneDeepNestedStruct) { }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f1", arrow::struct_({ - arrow::field("inner", arrow::struct_({ - arrow::field("x", arrow::int64()), - })), - })), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [[100]]], [0, 2, [[200]]], [0, 3, [[300]]] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: Nested projected schema with special fields under row tracking. @@ -933,11 +841,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeys) { helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - // Scan to get splits - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_FALSE(data_splits.empty()); - // Build projected schema: read f0 and f1 with selected keys "a,c" auto selected_keys_metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); @@ -947,35 +850,12 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeys) { }; auto projected_schema = arrow::schema(projected_fields); - // Export to C ArrowSchema - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - // Read with projected schema - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - // Expected: only keys "a" and "c" remain in each map - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f1", arrow::map(arrow::utf8(), arrow::int32())), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [["a", 10], ["c", 30]]], [0, 2, [["a", 100], ["c", 300]]], [0, 3, [["c", 400]]] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: Selected-keys metadata on MAP nested inside STRUCT should be applied. @@ -1015,10 +895,6 @@ TEST_P(NestedColumnPruningInteTest, NestedMapSelectedKeysInStruct) { helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_FALSE(data_splits.empty()); - auto selected_keys_metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); auto projected_struct_type = arrow::struct_({ @@ -1030,34 +906,11 @@ TEST_P(NestedColumnPruningInteTest, NestedMapSelectedKeysInStruct) { }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f1", arrow::struct_({ - arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())), - })), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [[ ["a", 10], ["c", 30] ]]], [0, 2, [[ ["a", 100], ["c", 300] ]]], [0, 3, [[ ["c", 400] ]]] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: Partial STRUCT sub-field recall where one recalled child is MAP with selected keys. @@ -1098,10 +951,6 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKey helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_FALSE(data_splits.empty()); - auto selected_keys_metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); auto projected_struct_type = arrow::struct_({ @@ -1114,35 +963,11 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKey }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f1", arrow::struct_({ - arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())), - arrow::field("keep", arrow::int64()), - })), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [[ ["a", 10], ["c", 30] ], 1001]], [0, 2, [[ ["a", 100], ["c", 300] ], 1002]], [0, 3, [[ ["c", 400] ], 1003]] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: Null semantics should be preserved when pruning STRUCT sub-fields and @@ -1186,10 +1011,6 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKey helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_FALSE(data_splits.empty()); - auto selected_keys_metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"a,c"}); auto projected_struct_type = arrow::struct_({ @@ -1202,37 +1023,13 @@ TEST_P(NestedColumnPruningInteTest, PruneStructSubFieldsWithNestedMapSelectedKey }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f1", arrow::struct_({ - arrow::field("m", arrow::map(arrow::utf8(), arrow::int32())), - arrow::field("keep", arrow::int64()), - })), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [[ ["a", 10], ["c", 30] ], 1001]], [0, 2, null], [0, 3, [null, 1003]], [0, 4, [[ ["c", 400] ], null]], [0, 5, [[ ["a", 500], ["c", null] ], 1005]] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: MAP_SELECTED_KEYS metadata value is empty string, select empty-string map key. @@ -1270,11 +1067,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysEmptyStringKey) { helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - // Scan to get splits - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_FALSE(data_splits.empty()); - // Build projected schema: read f0 and f1 with selected keys metadata set to empty string. auto selected_keys_metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {""}); @@ -1284,34 +1076,12 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysEmptyStringKey) { }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - // Read with projected schema - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - // Expected: only empty-string key remains. - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f1", arrow::map(arrow::utf8(), arrow::int32())), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [["", 9]]], [0, 2, [["", 99]]], [0, 3, []] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: MAP_SELECTED_KEYS output map entry order should follow selected key order. @@ -1348,10 +1118,6 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysPreserveOrder) { helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - ASSERT_FALSE(data_splits.empty()); - // Query key order is c,a and output should follow this order. auto selected_keys_metadata = arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"c,a"}); @@ -1361,32 +1127,125 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysPreserveOrder) { }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field("f1", arrow::map(arrow::utf8(), arrow::int32())), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [["c", 30], ["a", 10]]], [0, 2, [["c", 300], ["a", 100]]], [0, 3, [["c", 400], ["a", 500]]] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); + ])"); +} + +TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) { + if (file_format_ != "parquet" && file_format_ != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int32()); + auto info_type = arrow::struct_({ + arrow::field("score", arrow::int64()), + arrow::field("label", arrow::utf8()), + arrow::field("drop", arrow::utf8()), + }); + auto payload_type = arrow::struct_({ + arrow::field("attrs", map_type), + arrow::field("info", info_type), + arrow::field("note", arrow::utf8()), + }); + arrow::FieldVector table_fields = { + arrow::field("id", arrow::int32()), + arrow::field("payload", payload_type), + arrow::field("category", arrow::utf8()), + }; + auto table_schema = arrow::schema(table_fields); - AssertChunkedArrayEquals(expected_chunked, read_result); + std::map options = { + {Options::MANIFEST_FORMAT, "AVRO"}, + {Options::FILE_FORMAT, StringUtils::ToUpperCase(file_format_)}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "-1"}, + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + {"parquet.write.max-row-group-length", "1"}, + {"parquet.read.enable-page-index-filter", "true"}, + {"orc.stripe.size", "1"}, + {"orc.row.index.stride", "1"}, + }; + + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + WriteContextBuilder write_context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + write_context_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + auto write_one_row = [&](const std::string& data) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(table_fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + return file_store_write->Write(std::move(batch)); + }; + + ASSERT_OK(write_one_row( + R"([[1, [[["a", 10], ["b", 20], ["c", 30]], [1001, "low", "x"], "n1"], "hot"]])")); + ASSERT_OK(write_one_row( + R"([[12, [[["a", 100], ["c", 300], ["d", 400]], [1002, "mid", "y"], "n2"], "warm"]])")); + ASSERT_OK(write_one_row( + R"([[21, [[["b", 200], ["c", 500], ["a", 600]], [1003, "high", "z"], "n3"], "cold"]])")); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + file_store_write->PrepareCommit(/*wait_compaction=*/false, + /*commit_identifier=*/0)); + ASSERT_OK(file_store_write->Close()); + + CommitContextBuilder commit_context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); + + // Read selected MAP keys together with nested STRUCT sub-fields. + auto selected_keys_metadata = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"c,a"}); + auto selected_payload_type = arrow::struct_({ + arrow::field("attrs", map_type)->WithMetadata(selected_keys_metadata), + arrow::field("info", arrow::struct_({arrow::field("score", arrow::int64())})), + }); + auto selected_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("payload", selected_payload_type), + }); + + ScanReadAndCheck(table_path_, selected_schema, R"([ + [0, 1, [[["c", 30], ["a", 10]], [1001]]], + [0, 12, [[["c", 300], ["a", 100]], [1002]]], + [0, 21, [[["c", 500], ["a", 600]], [1003]]] + ])"); + + // Read only part of top-level columns and part of nested STRUCT fields. + auto partial_payload_type = arrow::struct_({ + arrow::field("info", arrow::struct_({arrow::field("label", arrow::utf8())})), + }); + auto partial_schema = arrow::schema({ + arrow::field("payload", partial_payload_type), + arrow::field("category", arrow::utf8()), + }); + + ScanReadAndCheck(table_path_, partial_schema, R"([ + [0, [["low"]], "hot"], + [0, [["mid"]], "warm"], + [0, [["high"]], "cold"] + ])"); + + // Read selected nested fields with predicate pushdown. + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(10)); + + ScanReadAndCheck(table_path_, selected_schema, R"([ + [0, 12, [[["c", 300], ["a", 100]], [1002]]], + [0, 21, [[["c", 500], ["a", 600]], [1003]]] + ])", + predicate); } // Test: ORC dictionary-encoded map key/value should work with MAP_SELECTED_KEYS. @@ -1434,8 +1293,8 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto data_splits, helper->NewScan(StartupMode::LatestFull(), + /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); auto selected_keys_metadata = @@ -1481,7 +1340,8 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysWithOrcDictionaryEncodedMap) AssertChunkedArrayEquals(expected_chunked, actual_chunked); } -// Test: Deeper nested struct — prune sub-fields of a struct inside a struct inside another struct. +// Test: Deeper nested struct — prune sub-fields of a struct inside a struct inside another +// struct. TEST_P(NestedColumnPruningInteTest, PruneDeeperNestedStruct) { // Table schema: f0 (int32), f1 (struct{a: int32, inner1: struct{x: int64, inner2: struct{p: // utf8, q: float64}}}) @@ -1527,9 +1387,6 @@ TEST_P(NestedColumnPruningInteTest, PruneDeeperNestedStruct) { helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); - // Projected: f0, f1{inner1{inner2{p}}} auto pruned_inner2 = arrow::struct_({ arrow::field("p", arrow::utf8()), @@ -1546,40 +1403,11 @@ TEST_P(NestedColumnPruningInteTest, PruneDeeperNestedStruct) { }; auto projected_schema = arrow::schema(projected_fields); - auto c_schema = std::make_unique(); - ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_schema.get()).ok()); - - ReadContextBuilder read_context_builder(table_path_); - read_context_builder.SetOptions(options).SetReadSchema(std::move(c_schema)); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(data_splits)); - ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - - arrow::FieldVector expected_fields = { - arrow::field("_VALUE_KIND", arrow::int8()), - arrow::field("f0", arrow::int32()), - arrow::field( - "f1", arrow::struct_({ - arrow::field("inner1", - arrow::struct_({ - arrow::field("inner2", arrow::struct_({ - arrow::field("p", arrow::utf8()), - })), - })), - })), - }; - auto expected_type = arrow::struct_(expected_fields); - std::string expected_data = R"([ + ScanReadAndCheck(table_path_, projected_schema, R"([ [0, 1, [[[ "ppp" ]]]], [0, 2, [[[ "qqq" ]]]], [0, 3, [[[ "rrr" ]]]] - ])"; - auto expected_array = - arrow::ipc::internal::json::ArrayFromJSON(expected_type, expected_data).ValueOrDie(); - auto expected_chunked = std::make_shared(expected_array); - - AssertChunkedArrayEquals(expected_chunked, read_result); + ])"); } // Test: Nested pruning for LIST> in integration path. @@ -1620,8 +1448,8 @@ TEST_P(NestedColumnPruningInteTest, PruneListStructSubFields) { helper->WriteAndCommit(std::move(batch), commit_identifier++, /*expected_commit_messages=*/std::nullopt)); - ASSERT_OK_AND_ASSIGN(auto data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto data_splits, helper->NewScan(StartupMode::LatestFull(), + /*snapshot_id=*/std::nullopt)); ASSERT_FALSE(data_splits.empty()); auto pruned_list_elem_struct = arrow::struct_({arrow::field("x", arrow::int64())}); diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index b9962209..d3171bdd 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -44,11 +44,15 @@ #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/read_context.h" #include "paimon/record_batch.h" #include "paimon/result.h" +#include "paimon/scan_context.h" #include "paimon/status.h" #include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/data_generator.h" #include "paimon/testing/utils/io_exception_helper.h" @@ -226,13 +230,23 @@ class PkCompactionInteTest : public ::testing::Test, void ScanAndVerify(const std::string& table_path, const arrow::FieldVector& fields, const std::map, std::string>& - expected_data_per_partition_bucket) { + expected_data_per_partition_bucket, + const std::shared_ptr& predicate = nullptr) { std::map options = {{Options::FILE_SYSTEM, "local"}}; ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(table_path, options, /*is_streaming_mode=*/false)); - ASSERT_OK_AND_ASSIGN( - std::vector> data_splits, - helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.WithStreamingMode(false).SetOptions(options).AddOption( + Options::SCAN_MODE, StartupMode::LatestFull().ToString()); + if (predicate) { + scan_context_builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, + scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_plan, table_scan->CreatePlan()); + std::vector> data_splits = result_plan->Splits(); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -255,8 +269,31 @@ class PkCompactionInteTest : public ::testing::Test, auto iter = expected_data_per_partition_bucket.find(key); ASSERT_TRUE(iter != expected_data_per_partition_bucket.end()) << "Unexpected partition=" << key.first << " bucket=" << key.second; - ASSERT_OK_AND_ASSIGN(bool success, - helper->ReadAndCheckResult(data_type, splits, iter->second)); + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options); + if (predicate) { + read_context_builder.SetPredicate(predicate); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, + read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(splits)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr read_result, + ReadResultCollector::CollectResult(batch_reader.get())); + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON(data_type, iter->second).ValueOrDie(); + auto expected_chunk_array = std::make_shared(expected_array); + + bool success = expected_chunk_array->Equals(read_result); + if (!success) { + std::cout << "[expected_data_type]" << expected_chunk_array->type()->ToString() + << std::endl; + std::cout << "[actual_data_type]" << read_result->type()->ToString() << std::endl; + std::cout << "[expected]:" << expected_chunk_array->ToString() << std::endl; + std::cout << "[actual]: " << read_result->ToString() << std::endl; + } ASSERT_TRUE(success); } } @@ -320,6 +357,190 @@ class PkCompactionInteTest : public ::testing::Test, arrow::FieldVector fields_; }; +// Verify shared-shredding MAP can be read correctly after PK full compaction. +TEST_P(PkCompactionInteTest, TestKeyValueTableFullCompactionWithMapSharedShredding) { + auto file_format = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::vector primary_keys = {"id"}; + std::vector partition_keys = {}; + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, "local"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(dir_->Str(), arrow::schema(fields), partition_keys, + primary_keys, options, /*is_streaming_mode=*/true)); + + int64_t commit_identifier = 0; + ASSERT_OK_AND_ASSIGN(auto batch_0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["b", 20]]], + [2, [["c", 30]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_0), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 100], ["d", 400]]], + [3, [["e", 50]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK(helper->write_->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN( + std::vector> commit_messages, + helper->write_->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + ASSERT_FALSE(commit_messages.empty()); + ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.value().GetCommitKind()); + + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(data_type, data_splits, + R"([ + [0, 1, [["a", 100], ["d", 400]]], + [0, 2, [["c", 30]]], + [0, 3, [["e", 50]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(PkCompactionInteTest, TestKeyValueTableDvCompactionWithMapSharedShredding) { + auto file_format = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + arrow::field("padding", arrow::utf8()), + }; + std::vector primary_keys = {"id"}; + std::vector partition_keys = {}; + std::map options = { + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_COMPRESSION, "none"}, + {Options::DELETION_VECTORS_ENABLED, "true"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "2"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + {"parquet.write.max-row-group-length", "1"}, + {"parquet.read.enable-page-index-filter", "true"}, + {"orc.stripe.size", "1"}, + {"orc.row.index.stride", "1"}, + }; + CreateTable(fields, partition_keys, primary_keys, options); + std::string table_path = TablePath(); + auto data_type = arrow::struct_(fields); + int64_t commit_id = 0; + std::string padding(2048, 'X'); + + { + // clang-format off + std::string json_data = R"([ +[1, [["a", 10], ["b", 20]], ")" + padding + R"("], +[2, [["c", 30]], ")" + padding + R"("], +[3, [["d", 40]], ")" + padding + R"("], +[4, null, ")" + padding + R"("], +[6, [["j", 60], ["k", 70]], ")" + padding + R"("], +[7, [["l", 80], ["m", 90], ["n", 100]], ")" + padding + R"("], +[8, [["o", 110]], ")" + padding + R"("] +])"; + // clang-format on + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, json_data).ValueOrDie(); + ASSERT_OK(WriteAndCommit(table_path, {}, 0, array, commit_id++)); + } + + ASSERT_OK_AND_ASSIGN( + auto upgrade_msgs, + CompactAndCommit(table_path, {}, 0, /*full_compaction=*/true, commit_id++)); + ASSERT_FALSE(HasDeletionVectorIndexFiles(upgrade_msgs)) + << "Initial full compact should not produce DV index files"; + + { + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + [1, [["a", 100], ["e", 500]], "u1"], + [5, [["h", 80]], "u5"] + ])") + .ValueOrDie(); + ASSERT_OK(WriteAndCommit(table_path, {}, 0, array, commit_id++)); + } + + { + auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ + [2, [["c", 300], ["f", 600], ["g", 700]], "u2"], + [5, [["h", 800], ["i", 900]], "u5-new"] + ])") + .ValueOrDie(); + ASSERT_OK(WriteAndCommit(table_path, {}, 0, array, commit_id++)); + } + + ASSERT_OK_AND_ASSIGN( + auto dv_compact_msgs, + CompactAndCommit(table_path, {}, 0, /*full_compaction=*/false, commit_id++)); + ASSERT_TRUE(HasDeletionVectorIndexFiles(dv_compact_msgs)) + << "Non-full compact should produce DV index files"; + + std::map, std::string> expected_data; + // clang-format off + expected_data[std::make_pair("", 0)] = R"([ +[0, 3, [["d", 40]], ")" + padding + R"("], +[0, 4, null, ")" + padding + R"("], +[0, 6, [["j", 60], ["k", 70]], ")" + padding + R"("], +[0, 7, [["l", 80], ["m", 90], ["n", 100]], ")" + padding + R"("], +[0, 8, [["o", 110]], ")" + padding + R"("], +[0, 1, [["a", 100], ["e", 500]], "u1"], +[0, 2, [["c", 300], ["f", 600], ["g", 700]], "u2"], +[0, 5, [["h", 800], ["i", 900]], "u5-new"] +])"; + // clang-format on + ScanAndVerify(table_path, fields, expected_data); + + // read with predicate and dv bitmap + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(6)); + std::map, std::string> expected_predicate_data; + // clang-format off + expected_predicate_data[std::make_pair("", 0)] = R"([ +[0, 7, [["l", 80], ["m", 90], ["n", 100]], ")" + padding + R"("], +[0, 8, [["o", 110]], ")" + padding + R"("] +])"; + // clang-format on + ScanAndVerify(table_path, fields, expected_predicate_data, predicate); +} + // Test: deduplicate merge engine with deletion vectors enabled. // Verifies that a non-full compact produces DV index files when level-0 files // overlap with high-level files, and that data is correct after DV compact and full compact. @@ -1997,13 +2218,21 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK_AND_ASSIGN(auto compact_msgs, CompactAndCommit(table_path, {{"f1", "10"}}, 0, /*full_compaction=*/false, commit_id++)); - ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)) - << "Non-full compact #1 must produce DV for Alice/Bob in L5"; + ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)); + + std::map, std::string> expected_data; + expected_data[std::make_pair("f1=10/", 0)] = R"([ + [0, "Carol", 10, 0, 3.0, ")" + padding + R"("], + [0, "Dave", 10, 0, 4.0, ")" + padding + R"("], + [0, "Eve", 10, 0, 5.0, ")" + padding + R"("], + [0, "Alice", 10, 0, 10.0, "v2a"], + [0, "Bob", 10, 0, 20.0, "v2b"] + ])"; + ScanAndVerify(table_path, fields, expected_data); } // Step 3: Write batch_3 (overlap Bob/Carol) → non-full compact. - // L0 merges to a lower intermediate level; DV marks Bob in the intermediate file - // from Step 2, and Carol in L5. + // DV marks Carol in L5. { auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([ ["Bob", 10, 0, 200.0, "v3b"], @@ -2015,8 +2244,16 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK_AND_ASSIGN(auto compact_msgs, CompactAndCommit(table_path, {{"f1", "10"}}, 0, /*full_compaction=*/false, commit_id++)); - ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)) - << "Non-full compact #2 must produce DV for Bob (intermediate) and Carol (L5)"; + ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)); + std::map, std::string> expected_data; + expected_data[std::make_pair("f1=10/", 0)] = R"([ + [0, "Dave", 10, 0, 4.0, ")" + padding + R"("], + [0, "Eve", 10, 0, 5.0, ")" + padding + R"("], + [0, "Alice", 10, 0, 10.0, "v2a"], + [0, "Bob", 10, 0, 200.0, "v3b"], + [0, "Carol", 10, 0, 300.0, "v3c"] + ])"; + ScanAndVerify(table_path, fields, expected_data); } // Step 4: Write batch_4 (overlap Carol/Dave) → non-full compact. @@ -2032,8 +2269,16 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK_AND_ASSIGN(auto compact_msgs, CompactAndCommit(table_path, {{"f1", "10"}}, 0, /*full_compaction=*/false, commit_id++)); - ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)) - << "Non-full compact #3 must produce DV for Carol (intermediate) and Dave (L5)"; + ASSERT_TRUE(HasDeletionVectorIndexFiles(compact_msgs)); + std::map, std::string> expected_data; + expected_data[std::make_pair("f1=10/", 0)] = R"([ + [0, "Eve", 10, 0, 5.0, ")" + padding + R"("], + [0, "Alice", 10, 0, 10.0, "v2a"], + [0, "Bob", 10, 0, 200.0, "v3b"], + [0, "Carol", 10, 0, 3000.0, "v4c"], + [0, "Dave", 10, 0, 4000.0, "v4d"] + ])"; + ScanAndVerify(table_path, fields, expected_data); } // Step 5: Write batch_5 (overlap Dave/Eve) → leave at L0 (no compact). @@ -2047,11 +2292,11 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK(WriteAndCommit(table_path, {{"f1", "10"}}, 0, array, commit_id++)); std::map, std::string> expected_data; expected_data[std::make_pair("f1=10/", 0)] = R"([ + [0, "Eve", 10, 0, 5.0, ")" + padding + R"("], [0, "Alice", 10, 0, 10.0, "v2a"], [0, "Bob", 10, 0, 200.0, "v3b"], [0, "Carol", 10, 0, 3000.0, "v4c"], - [0, "Dave", 10, 0, 40000.0, "v5d"], - [0, "Eve", 10, 0, 50000.0, "v5e"] + [0, "Dave", 10, 0, 4000.0, "v4d"] ])"; ScanAndVerify(table_path, fields, expected_data); } @@ -2060,9 +2305,7 @@ TEST_F(PkCompactionInteTest, TestDeduplicateWithDvInAllLevels) { ASSERT_OK_AND_ASSIGN( auto final_compact_msgs, CompactAndCommit(table_path, {{"f1", "10"}}, 0, /*full_compaction=*/true, commit_id++)); - } - // Step 7: ScanAndVerify after full compact (globally sorted, all data in L5). - { + // Step 7: ScanAndVerify after full compact (globally sorted, all data in L5). std::map, std::string> expected_data; expected_data[std::make_pair("f1=10/", 0)] = R"([ [0, "Alice", 10, 0, 10.0, "v2a"], diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index c564ab66..28f55148 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -29,18 +29,27 @@ #include "arrow/c/bridge.h" #include "arrow/ipc/json_simple.h" #include "arrow/type.h" +#include "fmt/format.h" #include "gtest/gtest.h" +#include "paimon/commit_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" #include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/io/data_file_meta.h" #include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/source/data_split_impl.h" #include "paimon/defs.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/format/file_format_factory.h" #include "paimon/fs/file_system.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate_builder.h" #include "paimon/read_context.h" #include "paimon/reader/batch_reader.h" +#include "paimon/record_batch.h" #include "paimon/result.h" #include "paimon/scan_context.h" #include "paimon/status.h" @@ -51,6 +60,7 @@ #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/write_context.h" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/writer.h" @@ -180,6 +190,65 @@ class WriteAndReadInteTest return table_scan->CreatePlan(); } + Result>>> CurrentDataFiles( + const std::map& options) const { + PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + std::vector>> files; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("split cannot be cast to DataSplitImpl"); + } + for (const auto& data_file : data_split->DataFiles()) { + files.emplace_back(data_split->BucketPath(), data_file); + } + } + std::sort(files.begin(), files.end(), [](const auto& left, const auto& right) { + return left.second->min_sequence_number < right.second->min_sequence_number; + }); + return files; + } + + Result> ReadDataFileSchema( + const std::string& bucket_path, const std::shared_ptr& file, + const std::map& options) const { + std::string file_path = PathUtil::JoinPath(bucket_path, file->file_name); + PAIMON_ASSIGN_OR_RAISE(auto unique_input_stream, dir_->GetFileSystem()->Open(file_path)); + std::shared_ptr input_stream(std::move(unique_input_stream)); + PAIMON_ASSIGN_OR_RAISE(std::string format_str, file->FileFormat()); + PAIMON_ASSIGN_OR_RAISE(auto file_format, FileFormatFactory::Get(format_str, options)); + PAIMON_ASSIGN_OR_RAISE(auto reader_builder, file_format->CreateReaderBuilder(10)); + PAIMON_ASSIGN_OR_RAISE(auto reader, reader_builder->Build(input_stream)); + PAIMON_ASSIGN_OR_RAISE(auto c_file_schema, reader->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(auto file_schema, + arrow::ImportSchema(c_file_schema.get())); + return file_schema; + } + + Result ReadShreddingMeta( + const std::pair>& file, + const std::string& field_name, const std::map& options) const { + PAIMON_ASSIGN_OR_RAISE(auto file_schema, + ReadDataFileSchema(file.first, file.second, options)); + std::shared_ptr field = file_schema->GetFieldByName(field_name); + if (!field) { + return Status::Invalid( + fmt::format("field {} not found in data file schema", field_name)); + } + std::shared_ptr metadata = field->metadata(); + if (!metadata) { + return Status::Invalid( + fmt::format("field {} has no shared-shredding metadata", field_name)); + } + std::shared_ptr metadata_copy = metadata->Copy(); + if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata_copy)) { + return Status::Invalid( + fmt::format("field {} has no shared-shredding metadata", field_name)); + } + return MapSharedShreddingUtils::DeserializeMetadata( + metadata_copy, MapSharedShreddingDefine::kDefaultDictCompression); + } + private: std::string test_dir_; std::unique_ptr dir_; @@ -1408,6 +1477,429 @@ TEST_P(WriteAndReadInteTest, TestAppendSharedShreddingMap) { ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPartitionAndBucket) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("dt", arrow::utf8()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "5"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, schema, /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN( + auto p1_bucket0_first, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[1, "p1", [["a", 1]]]])", + /*partition_map=*/{{"dt", "p1"}}, + /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(p1_bucket0_first), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN( + auto p1_bucket0_second, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[2, "p1", [["b", 2]]]])", + /*partition_map=*/{{"dt", "p1"}}, + /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(p1_bucket0_second), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN( + auto p2_bucket1_first, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([[3, "p2", [["x", 10], ["y", 20], ["z", 30], ["w", 40]]]])", + /*partition_map=*/{{"dt", "p2"}}, /*bucket=*/1, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(p2_bucket1_first), /*commit_identifier=*/2, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto data_type = arrow::struct_(fields_with_row_kind); + ASSERT_OK_AND_ASSIGN(std::vector> data_splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_GE(data_splits.size(), 2); + std::string expected_data = R"([ + [0, 1, "p1", [["a", 1]]], + [0, 2, "p1", [["b", 2]]], + [0, 3, "p2", [["w", 40], ["x", 10], ["y", 20], ["z", 30]]] + ])"; + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(data_type, data_splits, expected_data)); + ASSERT_TRUE(success); + + ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options)); + ASSERT_EQ(3, files.size()); + std::vector>> p1_bucket0_files; + std::vector>> p2_bucket1_files; + for (const auto& file : files) { + if (file.first.find("dt=p1/bucket-0") != std::string::npos) { + p1_bucket0_files.push_back(file); + } else if (file.first.find("dt=p2/bucket-1") != std::string::npos) { + p2_bucket1_files.push_back(file); + } + } + ASSERT_EQ(2, p1_bucket0_files.size()); + ASSERT_EQ(1, p2_bucket1_files.size()); + + ASSERT_OK_AND_ASSIGN(auto p1_first_meta, + ReadShreddingMeta(p1_bucket0_files[0], "tags", options)); + ASSERT_EQ(5, p1_first_meta.num_columns); + ASSERT_EQ(1, p1_first_meta.max_row_width); + + ASSERT_OK_AND_ASSIGN(auto p1_second_meta, + ReadShreddingMeta(p1_bucket0_files[1], "tags", options)); + ASSERT_EQ(1, p1_second_meta.num_columns); + ASSERT_EQ(1, p1_second_meta.max_row_width); + + ASSERT_OK_AND_ASSIGN(auto p2_first_meta, + ReadShreddingMeta(p2_bucket1_files[0], "tags", options)); + ASSERT_EQ(5, p2_first_meta.num_columns); + ASSERT_EQ(4, p2_first_meta.max_row_width); +} + +TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1048576"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + {"parquet.write.max-row-group-length", "1"}, + {"parquet.read.enable-page-index-filter", "true"}, + {"orc.stripe.size", "1"}, + {"orc.row.index.stride", "1"}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + (void)helper; + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + WriteContextBuilder write_context_builder(table_path, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, + write_context_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + + auto write_one_row = [&](const std::string& data) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), data, + /*partition_map=*/{}, /*bucket=*/0, {})); + return file_store_write->Write(std::move(batch)); + }; + + ASSERT_OK(write_one_row(R"([[1, [["a", 10], ["b", 20]]]])")); + ASSERT_OK(write_one_row(R"([[12, [["c", 31], ["d", 41]]]])")); + ASSERT_OK(write_one_row(R"([[21, [["e", 50], ["f", 60]]]])")); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + file_store_write->PrepareCommit(/*wait_compaction=*/false, + /*commit_identifier=*/0)); + ASSERT_OK(file_store_write->Close()); + + CommitContextBuilder commit_context_builder(table_path, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + commit_context_builder.SetOptions(options).Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(commit->Commit(commit_msgs, /*commit_identifier=*/0)); + + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"id", + FieldType::INT, Literal(10)); + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.SetOptions(options) + .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString()) + .SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_FALSE(result_plan->Splits().empty()); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + + arrow::FieldVector fields_with_row_kind = fields; + fields_with_row_kind.insert(fields_with_row_kind.begin(), + arrow::field("_VALUE_KIND", arrow::int8())); + auto expected_type = arrow::struct_(fields_with_row_kind); + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(expected_type, R"([ + [0, 12, [["c", 31], ["d", 41]]], + [0, 21, [["e", 50], ["f", 60]]] + ])") + .ValueOrDie()); + ASSERT_TRUE(expected->Equals(actual)) << actual->ToString(); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingRestoreAdaptiveColumnCountFromFileMetadata) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, file_system}, + {Options::WRITE_ONLY, "true"}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "8"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN( + auto batch_v0, TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[1, [["a", 11]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, + TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN( + auto batch_v1, TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[2, [["b", 22]]]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options)); + ASSERT_EQ(2, files.size()); + ASSERT_OK_AND_ASSIGN(auto first_meta, ReadShreddingMeta(files[0], "metrics", options)); + ASSERT_EQ(8, first_meta.num_columns); + ASSERT_EQ(1, first_meta.max_row_width); + ASSERT_OK_AND_ASSIGN(auto second_meta, ReadShreddingMeta(files[1], "metrics", options)); + ASSERT_EQ(1, second_meta.num_columns); + ASSERT_EQ(1, second_meta.max_row_width); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits, + R"([ + [0, 1, [["a", 11]]], + [0, 2, [["b", 22]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingSwitchMapLayoutAndUseMaxColumnsWithoutMetadata) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + arrow::field("labels", map_type), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, file_system}, + {Options::WRITE_ONLY, "true"}, + {"fields.labels.map.storage-layout", "shared-shredding"}, + {"fields.labels.map.shared-shredding.max-columns", "4"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{}, + /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 11], ["b", 12]], [["x", 21]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.metrics.map.storage-layout"] = "shared-shredding"; + options_v1["fields.metrics.map.shared-shredding.max-columns"] = "3"; + options_v1["fields.labels.map.storage-layout"] = "default"; + options_v1.erase("fields.labels.map.shared-shredding.max-columns"); + ASSERT_OK( + WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1]), DataField(2, fields[2])}, + /*highest_field_id=*/2, options_v1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), + options_v1, /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch_v1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [2, [["c", 31]], [["y", 41], ["z", 42]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options_v1)); + ASSERT_EQ(2, files.size()); + ASSERT_OK_AND_ASSIGN(auto metrics_meta, ReadShreddingMeta(files[1], "metrics", options_v1)); + ASSERT_EQ(3, metrics_meta.num_columns); + ASSERT_EQ(1, metrics_meta.max_row_width); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + arrow::field("labels", map_type), + }); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits, + R"([ + [0, 1, [["a", 11], ["b", 12]], [["x", 21]]], + [0, 2, [["c", 31]], [["y", 41], ["z", 42]]] + ])")); + ASSERT_TRUE(success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingReadAfterRenameColumn) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto map_type = arrow::map(arrow::utf8(), arrow::int64()); + arrow::FieldVector fields_v0 = { + arrow::field("id", arrow::int32()), + arrow::field("metrics", map_type), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.metrics.map.storage-layout", "shared-shredding"}, + {"fields.metrics.map.shared-shredding.max-columns", "2"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields_v0), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/false)); + + ASSERT_OK_AND_ASSIGN(auto batch_v0, + TestHelper::MakeRecordBatch(arrow::struct_(fields_v0), + R"([ + [1, [["a", 11], ["b", 12]]], + [2, [["c", 21]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector fields_v1 = { + arrow::field("id", arrow::int32()), + arrow::field("renamed_metrics", map_type), + }; + std::map options_v1 = options_v0; + options_v1.erase("fields.metrics.map.storage-layout"); + options_v1.erase("fields.metrics.map.shared-shredding.max-columns"); + options_v1["fields.renamed_metrics.map.storage-layout"] = "shared-shredding"; + options_v1["fields.renamed_metrics.map.shared-shredding.max-columns"] = "2"; + ASSERT_OK(WriteNextSchema({DataField(0, fields_v1[0]), DataField(1, fields_v1[1])}, + /*highest_field_id=*/1, options_v1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, TestHelper::Create(PathUtil::JoinPath(test_dir_, "foo.db/bar"), + options_v1, /*is_streaming_mode=*/false)); + + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("renamed_metrics", map_type), + }); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits, + R"([ + [0, 1, [["a", 11], ["b", 12]]], + [0, 2, [["c", 21]]] + ])")); + ASSERT_TRUE(success); +} + TEST_P(WriteAndReadInteTest, TestSharedShreddingWithSchemaEvolution) { auto [file_format, file_system] = GetParam(); if (file_format != "parquet" && file_format != "orc") { @@ -1623,6 +2115,109 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefault) { ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultCompaction) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), + }; + std::map options_v0 = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "id"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options_v0 = AddOptionsForJindo(options_v0); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options_v0, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + int64_t commit_identifier = 0; + + ASSERT_OK_AND_ASSIGN(auto batch_v0_file1, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", 10], ["b", 11]]], + [2, [["c", 20]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0_file1), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto batch_v0_file2, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [3, [["d", 30], ["e", 31]]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0_file2), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + std::map options_v1 = options_v0; + options_v1["fields.tags.map.storage-layout"] = "default"; + options_v1.erase("fields.tags.map.shared-shredding.max-columns"); + ASSERT_OK(WriteNextSchema({DataField(0, fields[0]), DataField(1, fields[1])}, + /*highest_field_id=*/1, options_v1)); + + helper.reset(); + ASSERT_OK_AND_ASSIGN(helper, TestHelper::Create(table_path, options_v1, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN(auto batch_v1_file3, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [4, [["a", 40], ["f", 41]]], + [5, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1_file3), commit_identifier++, + /*expected_commit_messages=*/std::nullopt)); + + WriteContextBuilder write_context_builder(table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN( + auto write_context, + write_context_builder.SetOptions(options_v1).WithStreamingMode(true).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context))); + ASSERT_OK(file_store_write->Compact(/*partition=*/{}, /*bucket=*/0, + /*full_compaction=*/true)); + ASSERT_OK_AND_ASSIGN(auto compact_messages, file_store_write->PrepareCommit( + /*wait_compaction=*/true, commit_identifier)); + ASSERT_FALSE(compact_messages.empty()); + + CommitContextBuilder commit_context_builder(table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(auto commit_context, + commit_context_builder.SetOptions(options_v1).Finish()); + ASSERT_OK_AND_ASSIGN(auto file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + ASSERT_OK(file_store_commit->Commit(compact_messages, commit_identifier)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_EQ(1, splits.size()); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", 10], ["b", 11]]], + [0, 2, [["c", 20]]], + [0, 3, [["d", 30], ["e", 31]]], + [0, 4, [["a", 40], ["f", 41]]], + [0, 5, null] + ])")); + ASSERT_TRUE(success); +} + // Nested map values through both selected physical columns and overflow. TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) { auto [file_format, file_system] = GetParam(); @@ -1679,6 +2274,226 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) { ASSERT_TRUE(success); } +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithComplexValue) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto value_type = arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("scores", arrow::list(arrow::int32())), + arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64())), + }); + auto map_type = arrow::map(arrow::utf8(), value_type); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [ + ["a", ["alpha", [1, 2], [["ia", 10], ["ib", 20]]]], + ["z", ["zeta", [9], [["iz", 90]]]] + ]], + [2, [ + ["a", ["amy", null, [["ia", 30]]]], + ["b", ["beta", [], []]] + ]], + [3, null] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool full_success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [ + ["a", ["alpha", [1, 2], [["ia", 10], ["ib", 20]]]], + ["z", ["zeta", [9], [["iz", 90]]]] + ]], + [0, 2, [ + ["a", ["amy", null, [["ia", 30]]]], + ["b", ["beta", [], []]] + ]], + [0, 3, null] + ])")); + ASSERT_TRUE(full_success); + + auto selected_keys_meta = + arrow::KeyValueMetadata::Make({DataField::MAP_SELECTED_KEYS}, {"z,a"}); + auto read_schema = arrow::schema({ + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type)->WithMetadata(selected_keys_meta), + }); + auto expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("id", arrow::int32()), + arrow::field("tags", map_type), + }); + ASSERT_OK_AND_ASSIGN(bool selected_success, + ReadAndCheckWithReadSchema(options, read_schema, expected_type, + R"([ + [0, 1, [ + ["z", ["zeta", [9], [["iz", 90]]]], + ["a", ["alpha", [1, 2], [["ia", 10], ["ib", 20]]]] + ]], + [0, 2, [ + ["a", ["amy", null, [["ia", 30]]]] + ]], + [0, 3, null] + ])")); + ASSERT_TRUE(selected_success); +} + +TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionReadFails) { + auto [file_format, file_system] = GetParam(); + if (file_format != "parquet" && file_format != "orc") { + return; + } + + auto tag_value_type = arrow::struct_({ + arrow::field("v", arrow::int64()), + arrow::field("label", arrow::utf8()), + }); + auto profile_type = arrow::struct_({ + arrow::field("name", arrow::utf8()), + arrow::field("score", arrow::int64()), + }); + arrow::FieldVector fields = { + arrow::field("id", arrow::int32()), + arrow::field("tags", arrow::map(arrow::utf8(), tag_value_type)), + arrow::field("profile", profile_type), + }; + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, file_format}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::BUCKET, "-1"}, + {Options::FILE_SYSTEM, file_system}, + {"fields.tags.map.storage-layout", "shared-shredding"}, + {"fields.tags.map.shared-shredding.max-columns", "1"}, + }; + if (file_system == "jindo") { + options = AddOptionsForJindo(options); + } + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(auto batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([ + [1, [["a", [10, "one"]], ["z", [11, "overflow"]]], ["alice", 100]], + [2, [["a", [20, "two"]]], ["bob", 200]] + ])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + arrow::FieldVector expected_fields = fields; + expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8())); + ASSERT_OK_AND_ASSIGN(auto splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(bool success, + helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits, + R"([ + [0, 1, [["a", [10, "one"]], ["z", [11, "overflow"]]], ["alice", 100]], + [0, 2, [["a", [20, "two"]]], ["bob", 200]] + ])")); + ASSERT_TRUE(success); + + std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar"); + SchemaManager schema_manager(dir_->GetFileSystem(), table_path); + ASSERT_OK_AND_ASSIGN(auto schema_v0, schema_manager.ReadSchema(0)); + std::vector fields_v0 = schema_v0->Fields(); + + auto read_fields = [&](const std::vector& field_names) -> Status { + PAIMON_ASSIGN_OR_RAISE(auto plan, InnerScan(options)); + ReadContextBuilder read_context_builder(table_path); + read_context_builder.SetOptions(options).SetReadFieldNames(field_names); + PAIMON_ASSIGN_OR_RAISE(auto read_context, read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(auto actual, ReadResultCollector::CollectResult(batch_reader.get())); + (void)actual; + return Status::OK(); + }; + + auto tag_field = fields_v0[1].ArrowField(); + auto tag_map = arrow::internal::checked_pointer_cast(tag_field->type()); + auto tag_value_struct = + arrow::internal::checked_pointer_cast(tag_map->item_type()); + + // Simulate alter table changing the shared-shredding MAP value struct field type. + auto changed_tag_value_type = + arrow::map(tag_map->key_type(), tag_map->item_field()->WithType(arrow::struct_({ + tag_value_struct->field(0)->WithType(arrow::utf8()), + tag_value_struct->field(1), + }))); + std::vector fields_with_changed_tag_value = fields_v0; + fields_with_changed_tag_value[1] = + DataField(fields_v0[1].Id(), tag_field->WithType(changed_tag_value_type)); + ASSERT_OK(WriteNextSchema(fields_with_changed_tag_value, schema_v0->HighestFieldId(), options)); + ASSERT_NOK_WITH_MSG(read_fields({"tags"}), + "PruneDataType does not support partial projection inside map: src " + "map> vs target " + "map>"); + + auto profile_field = fields_v0[2].ArrowField(); + auto profile_struct = + arrow::internal::checked_pointer_cast(profile_field->type()); + + // Simulate alter table renaming a nested field inside a STRUCT column. + std::vector fields_with_renamed_profile_child = fields_v0; + auto renamed_profile_type = arrow::struct_({ + profile_struct->field(0)->WithName("renamed_name"), + profile_struct->field(1), + }); + fields_with_renamed_profile_child[2] = + DataField(fields_v0[2].Id(), profile_field->WithType(renamed_profile_type)); + ASSERT_OK( + WriteNextSchema(fields_with_renamed_profile_child, schema_v0->HighestFieldId(), options)); + ASSERT_NOK_WITH_MSG(read_fields({"profile"}), + "name mismatch: read 'renamed_name' vs data 'name'"); + + // Simulate alter table changing a nested field type inside a STRUCT column. + std::vector fields_with_changed_profile_child_type = fields_v0; + auto changed_profile_type = arrow::struct_({ + profile_struct->field(0), + profile_struct->field(1)->WithType(arrow::utf8()), + }); + fields_with_changed_profile_child_type[2] = + DataField(fields_v0[2].Id(), profile_field->WithType(changed_profile_type)); + ASSERT_OK(WriteNextSchema(fields_with_changed_profile_child_type, schema_v0->HighestFieldId(), + options)); + ASSERT_NOK_WITH_MSG(read_fields({"profile"}), + "PruneDataType nested field type mismatch for 'score': read string vs " + "data int64"); +} + // Keep ORC lazy dictionary decoding enabled across a default -> shared-shredding schema change. // The test inspects every user-visible batch directly, because ReadResultCollector would otherwise // decode dictionary arrays and hide a type mismatch between old and new files. From 67b232acb335f6e4dd89976e8ce60363a6d2bd17 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:27:12 +0800 Subject: [PATCH 084/138] feat(blob): Remove blob external storage mode --- include/paimon/defs.h | 11 - src/paimon/CMakeLists.txt | 2 - src/paimon/common/defs.cpp | 2 - src/paimon/core/append/append_only_writer.cpp | 60 +-- src/paimon/core/append/append_only_writer.h | 2 - src/paimon/core/core_options.cpp | 17 - src/paimon/core/core_options.h | 2 - src/paimon/core/core_options_test.cpp | 8 - .../core/io/blob_data_file_writer_factory.cpp | 27 +- .../core/io/blob_data_file_writer_factory.h | 8 - src/paimon/core/io/data_file_path_factory.h | 6 - .../core/io/data_file_path_factory_test.cpp | 14 - .../core/io/external_storage_blob_writer.cpp | 202 --------- .../core/io/external_storage_blob_writer.h | 114 ----- .../io/external_storage_blob_writer_test.cpp | 153 ------- .../core/operation/blob_file_context.cpp | 26 +- src/paimon/core/operation/blob_file_context.h | 21 +- .../core/operation/blob_file_context_test.cpp | 36 -- src/paimon/core/schema/schema_validation.cpp | 22 +- .../core/schema/schema_validation_test.cpp | 49 +- .../blob/blob_file_batch_reader_test.cpp | 3 +- src/paimon/format/blob/blob_format_writer.cpp | 40 +- src/paimon/format/blob/blob_format_writer.h | 13 +- .../format/blob/blob_format_writer_test.cpp | 126 ++---- src/paimon/format/blob/blob_writer_builder.h | 9 +- .../format/blob/blob_writer_builder_test.cpp | 36 -- test/inte/blob_table_inte_test.cpp | 423 ++---------------- .../blob_desc_field}/README | 7 +- ...aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc | Bin ...eee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob | Bin ...eee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob | Bin ...eee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob | Bin ...est-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 | Bin ...ist-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 | Bin ...ist-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 | Bin .../blob_desc_field}/raw_blob/b0-row-0.bin | 0 .../blob_desc_field}/raw_blob/b0-row-1.bin | 0 .../blob_desc_field}/raw_blob/b0-row-2.bin | 0 .../blob_desc_field}/schema/schema-0 | 4 +- .../blob_desc_field}/snapshot/EARLIEST | 0 .../blob_desc_field}/snapshot/LATEST | 0 .../blob_desc_field}/snapshot/snapshot-1 | 0 .../blob_desc_field}/README | 7 +- ...888c-c975-46af-9a7d-36ca13c32455-1.parquet | Bin ...749888c-c975-46af-9a7d-36ca13c32455-2.blob | Bin ...749888c-c975-46af-9a7d-36ca13c32455-3.blob | Bin ...749888c-c975-46af-9a7d-36ca13c32455-0.blob | Bin ...est-de59f444-0069-4836-8dcd-8a3a81158e02-0 | Bin ...ist-7395f790-699b-4a38-8747-10f23ceba1d6-0 | Bin ...ist-7395f790-699b-4a38-8747-10f23ceba1d6-1 | Bin .../blob_desc_field}/raw_blob/b0-row-0.bin | 0 .../blob_desc_field}/raw_blob/b0-row-1.bin | 0 .../blob_desc_field}/raw_blob/b0-row-2.bin | 0 .../blob_desc_field}/schema/schema-0 | 4 +- .../blob_desc_field}/snapshot/EARLIEST | 0 .../blob_desc_field}/snapshot/LATEST | 0 .../blob_desc_field}/snapshot/snapshot-1 | 0 57 files changed, 107 insertions(+), 1347 deletions(-) delete mode 100644 src/paimon/core/io/external_storage_blob_writer.cpp delete mode 100644 src/paimon/core/io/external_storage_blob_writer.h delete mode 100644 src/paimon/core/io/external_storage_blob_writer_test.cpp rename test/test_data/{parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => orc/blob_desc_field.db/blob_desc_field}/README (71%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/raw_blob/b0-row-0.bin (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/raw_blob/b0-row-1.bin (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/raw_blob/b0-row-2.bin (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/schema/schema-0 (88%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/snapshot/EARLIEST (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/snapshot/LATEST (100%) rename test/test_data/orc/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/snapshot/snapshot-1 (100%) rename test/test_data/{orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => parquet/blob_desc_field.db/blob_desc_field}/README (71%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-2.blob (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/raw_blob/b0-row-0.bin (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/raw_blob/b0-row-1.bin (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/raw_blob/b0-row-2.bin (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/schema/schema-0 (88%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/snapshot/EARLIEST (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/snapshot/LATEST (100%) rename test/test_data/parquet/{blob_desc_field_with_external_path.db/blob_desc_field_with_external_path => blob_desc_field.db/blob_desc_field}/snapshot/snapshot-1 (100%) diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 1d9271ae..61cba363 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -407,17 +407,6 @@ struct PAIMON_EXPORT Options { /// serialized BlobViewStruct bytes inline in data files and resolve from upstream tables at /// read time. No default value. static const char BLOB_VIEW_FIELD[]; - /// "blob-external-storage-field" - Comma-separated BLOB field names (must be a subset of - /// blob-descriptor-field ) whose raw data will be written to external storage at write time. - /// The external storage path is configured via blob-external-storage-path. Orphan file cleanup - /// is not applied to that path. No default value. - static const char BLOB_EXTERNAL_STORAGE_FIELD[]; - /// "blob-external-storage-path" - The external storage path where raw BLOB data from fields - /// configured by 'blob-external-storage-field' is written at write time. Orphan file cleanup is - /// not applied to this path. No default value. - /// @note: this option differs from the Java paimon and will be deprecated once - /// RestCatalog is supported. - static const char BLOB_EXTERNAL_STORAGE_PATH[]; /// "blob-view-upstream-warehouse" - Since the catalog capabilities are partially missing, when /// Blob View is enabled, cpp paimon cannot automatically obtain the upstream table warehouse /// path and requires manual configuration by the user. No default value. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index fa52567c..8f7b4d8a 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -241,7 +241,6 @@ set(PAIMON_CORE_SRCS core/io/map_shared_shredding_core_utils.cpp core/io/shredding_append_data_file_writer_factory.cpp core/io/shredding_key_value_data_file_writer_factory.cpp - core/io/external_storage_blob_writer.cpp core/io/multiple_blob_file_writer.cpp core/io/rolling_blob_file_writer.cpp core/manifest/file_kind.cpp @@ -636,7 +635,6 @@ if(PAIMON_BUILD_TESTS) core/io/file_index_evaluator_test.cpp core/io/single_file_writer_test.cpp core/io/rolling_blob_file_writer_test.cpp - core/io/external_storage_blob_writer_test.cpp core/global_index/indexed_split_test.cpp core/manifest/file_source_test.cpp core/manifest/file_kind_test.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index b361497b..ab6b6c97 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -104,8 +104,6 @@ const char Options::BLOB_DESCRIPTOR_FIELD[] = "blob-descriptor-field"; const char Options::FALLBACK_BLOB_DESCRIPTOR_FIELD[] = "blob.stored-descriptor-fields"; const char Options::BLOB_VIEW_FIELD[] = "blob-view-field"; const char Options::BLOB_VIEW_UPSTREAM_WAREHOUSE[] = "blob-view-upstream-warehouse"; -const char Options::BLOB_EXTERNAL_STORAGE_FIELD[] = "blob-external-storage-field"; -const char Options::BLOB_EXTERNAL_STORAGE_PATH[] = "blob-external-storage-path"; const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled"; const char Options::GLOBAL_INDEX_THREAD_NUM[] = "global-index.thread-num"; const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path"; diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 48b3d321..a4de60e8 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -38,7 +38,6 @@ #include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/data_increment.h" -#include "paimon/core/io/external_storage_blob_writer.h" #include "paimon/core/io/multiple_blob_file_writer.h" #include "paimon/core/io/rolling_blob_file_writer.h" #include "paimon/core/io/rolling_file_writer.h" @@ -87,22 +86,6 @@ Status AppendOnlyWriter::Write(std::unique_ptr&& batch) { PAIMON_ASSIGN_OR_RAISE(writer_, CreateRollingRowWriter()); } - // Transform batch for external storage descriptor fields before writing. - if (external_storage_writer_) { - auto data_type = arrow::struct_(write_schema_->fields()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, - arrow::ImportArray(batch->GetData(), data_type)); - auto struct_array = std::dynamic_pointer_cast(arrow_array); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr transformed, - external_storage_writer_->TransformBatch(struct_array)); - auto transformed_struct = std::dynamic_pointer_cast(transformed); - PAIMON_RETURN_NOT_OK(BlobUtils::ValidateBlobInlineFields( - transformed_struct, inline_descriptor_fields_, "blob-descriptor-field")); - ::ArrowArray c_transformed; - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*transformed, &c_transformed)); - return writer_->Write(&c_transformed); - } - if (!inline_descriptor_fields_.empty() || !inline_view_fields_.empty()) { auto data_type = arrow::struct_(write_schema_->fields()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, @@ -190,7 +173,6 @@ Status AppendOnlyWriter::Flush(bool wait_for_latest_compaction, bool forced_full AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWriter() { auto blob_context = BlobFileContext::Create(write_schema_, options_); - std::optional> main_write_cols = write_cols_; // Save inline descriptor and view fields for validation in Write() if (blob_context) { @@ -198,20 +180,6 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWrit inline_view_fields_ = blob_context->GetViewFields(); } - // Initialize ExternalStorageBlobWriter if needed - if (blob_context && blob_context->RequireExternalStorageWriter()) { - assert(blob_context->GetExternalStoragePath()); - external_storage_writer_ = std::make_unique( - write_schema_, blob_context->GetExternalStorageFields(), - blob_context->GetExternalStoragePath().value(), schema_id_, seq_num_counter_, - path_factory_, options_, memory_pool_); - if (!main_write_cols) { - // To align with java, when require external storage writer, main writer will set write - // cols in DataFileMeta - main_write_cols = write_schema_->field_names(); - } - } - if (blob_context && blob_context->RequireBlobFileWriter()) { // Use context-aware schema separation: inline BLOB fields stay in main auto schemas = @@ -219,18 +187,10 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWrit return CreateRollingBlobWriter(schemas, blob_context->GetInlineFields()); } - if (!blob_context) { - // No BLOB fields at all -> plain rolling writer - return std::make_unique>>( - options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterFactory(write_schema_, main_write_cols)); - } else { - // All BLOB fields are inline, no .blob files needed -> plain rolling writer - // The main data file contains all fields including inline descriptors/views. - return std::make_unique>>( - options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterFactory(write_schema_, main_write_cols)); - } + // No BLOB fields, or all BLOB fields are inline and no .blob files are needed. + return std::make_unique>>( + options_.GetTargetFileSize(/*has_primary_key=*/false), + GetDataFileWriterFactory(write_schema_, write_cols_)); } AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetDataFileWriterFactory( @@ -250,10 +210,9 @@ AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetBlobFileWriterFactory( const std::shared_ptr& single_field_schema, const std::optional>& write_cols) const { std::shared_ptr path_factory = path_factory_; - return std::make_shared( - options_, schema_id_, single_field_schema, write_cols, seq_num_counter_, path_factory, - [path_factory]() { return path_factory->NewBlobPath(); }, - blob::BlobFormatWriter::WriteConsumer(), memory_pool_); + return std::make_shared(options_, schema_id_, single_field_schema, + write_cols, seq_num_counter_, path_factory, + memory_pool_); } AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWriter( @@ -308,11 +267,6 @@ Status AppendOnlyWriter::Close() { writer_.reset(); } - if (external_storage_writer_) { - PAIMON_RETURN_NOT_OK(external_storage_writer_->Close()); - external_storage_writer_.reset(); - } - if (compact_deletion_file_ != nullptr) { compact_deletion_file_->Clean(); } diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index c95ff807..a0229b0b 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -44,7 +44,6 @@ class Schema; namespace paimon { class CommitIncrement; -class ExternalStorageBlobWriter; class MapSharedShreddingContext; class RecordBatch; template @@ -135,7 +134,6 @@ class AppendOnlyWriter : public BatchWriter { std::shared_ptr compact_deletion_file_; std::unique_ptr>> writer_; - std::unique_ptr external_storage_writer_; std::set inline_descriptor_fields_; std::set inline_view_fields_; diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 7ea5edd4..b1838ed2 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -384,7 +384,6 @@ struct CoreOptions::Impl { std::vector blob_fields; std::vector blob_descriptor_fields; std::vector blob_view_fields; - std::vector blob_external_storage_fields; std::string partition_default_name = "__DEFAULT_PARTITION__"; StartupMode startup_mode = StartupMode::Default(); @@ -397,7 +396,6 @@ struct CoreOptions::Impl { std::optional field_default_func; std::optional scan_fallback_branch; std::optional data_file_external_paths; - std::optional blob_external_storage_path; std::optional blob_view_upstream_warehouse; std::map raw_options; @@ -566,13 +564,6 @@ struct CoreOptions::Impl { // Parse blob-view-upstream-warehouse - warehouse path for configured blob view fields PAIMON_RETURN_NOT_OK( parser.Parse(Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, &blob_view_upstream_warehouse)); - // Parse blob-external-storage-field - descriptor BLOB fields written to external storage - PAIMON_RETURN_NOT_OK(parser.ParseList( - Options::BLOB_EXTERNAL_STORAGE_FIELD, Options::FIELDS_SEPARATOR, - &blob_external_storage_fields, /*need_trim=*/true)); - // Parse blob-external-storage-path - external storage path for configured BLOB fields - PAIMON_RETURN_NOT_OK( - parser.Parse(Options::BLOB_EXTERNAL_STORAGE_PATH, &blob_external_storage_path)); return Status::OK(); } @@ -1494,14 +1485,6 @@ std::vector CoreOptions::GetBlobInlineFields() const { return blob_inline_fields; } -const std::vector& CoreOptions::GetBlobExternalStorageFields() const { - return impl_->blob_external_storage_fields; -} - -std::optional CoreOptions::GetBlobExternalStoragePath() const { - return impl_->blob_external_storage_path; -} - int64_t CoreOptions::GetLookupCacheFileRetentionMs() const { return impl_->lookup_cache_file_retention_ms; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index fb41957a..a7a7c473 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -199,8 +199,6 @@ class PAIMON_EXPORT CoreOptions { const std::vector& GetBlobViewFields() const; std::optional GetBlobViewUpstreamWarehouse() const; std::vector GetBlobInlineFields() const; - const std::vector& GetBlobExternalStorageFields() const; - std::optional GetBlobExternalStoragePath() const; const std::map& ToMap() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index acf8df3d..a4f99430 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -124,9 +124,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_TRUE(core_options.GetBlobDescriptorFields().empty()); ASSERT_TRUE(core_options.GetBlobViewFields().empty()); ASSERT_TRUE(core_options.GetBlobInlineFields().empty()); - ASSERT_TRUE(core_options.GetBlobExternalStorageFields().empty()); ASSERT_EQ(std::nullopt, core_options.GetBlobViewUpstreamWarehouse()); - ASSERT_EQ(std::nullopt, core_options.GetBlobExternalStoragePath()); ASSERT_TRUE(core_options.LegacyPartitionNameEnabled()); ASSERT_TRUE(core_options.GlobalIndexEnabled()); ASSERT_EQ(std::nullopt, core_options.GetGlobalIndexExternalPath()); @@ -229,8 +227,6 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::BLOB_FIELD, "blob1,blob2"}, {Options::BLOB_DESCRIPTOR_FIELD, "blob3,blob4"}, {Options::BLOB_VIEW_FIELD, "blob5"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "blob3,blob4"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, "FILE:///tmp/blob_external_storage/"}, {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, "FILE:///tmp/blob_view_upstream_warehouse/"}, {Options::PARTITION_GENERATE_LEGACY_NAME, "false"}, {Options::GLOBAL_INDEX_ENABLED, "false"}, @@ -369,10 +365,6 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(core_options.GetBlobViewFields(), std::vector({"blob5"})); ASSERT_EQ(core_options.GetBlobInlineFields(), std::vector({"blob3", "blob4", "blob5"})); - ASSERT_EQ(core_options.GetBlobExternalStorageFields(), - std::vector({"blob3", "blob4"})); - ASSERT_EQ(core_options.GetBlobExternalStoragePath(), - std::optional("FILE:///tmp/blob_external_storage/")); ASSERT_EQ(core_options.GetBlobViewUpstreamWarehouse(), std::optional("FILE:///tmp/blob_view_upstream_warehouse/")); ASSERT_FALSE(core_options.LegacyPartitionNameEnabled()); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.cpp b/src/paimon/core/io/blob_data_file_writer_factory.cpp index d5f4fe5f..31dce727 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.cpp +++ b/src/paimon/core/io/blob_data_file_writer_factory.cpp @@ -25,7 +25,6 @@ #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/manifest/file_source.h" -#include "paimon/format/blob/blob_writer_builder.h" #include "paimon/format/file_format.h" #include "paimon/format/file_format_factory.h" #include "paimon/fs/file_system.h" @@ -37,15 +36,13 @@ BlobDataFileWriterFactory::BlobDataFileWriterFactory( const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, - const std::shared_ptr& path_factory, PathCreator path_creator, - blob::BlobFormatWriter::WriteConsumer write_consumer, const std::shared_ptr& pool) + const std::shared_ptr& path_factory, + const std::shared_ptr& pool) : DataFileWriterFactory(options, schema_id, pool), file_schema_(file_schema), write_cols_(write_cols), seq_num_counter_(seq_num_counter), - path_factory_(path_factory), - path_creator_(std::move(path_creator)), - write_consumer_(std::move(write_consumer)) {} + path_factory_(path_factory) {} Result>>> BlobDataFileWriterFactory::CreateWriter() const { @@ -54,26 +51,12 @@ BlobDataFileWriterFactory::CreateWriter() const { PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, file_schema_, /*create_stats_extractor=*/true)); - if (write_consumer_) { - auto blob_writer_builder = - std::dynamic_pointer_cast(resources.writer_builder); - if (!blob_writer_builder) { - return Status::Invalid( - "writer_builder cannot be casted to BlobWriterBuilder " - "in BlobDataFileWriterFactory"); - } - blob_writer_builder->WithWriteConsumer(write_consumer_); - } - auto writer = std::make_unique( /*compression=*/"none", std::function(), schema_id_, seq_num_counter_, FileSource::Append(), resources.stats_extractor, path_factory_->IsExternalPath(), write_cols_, pool_); - if (!path_creator_) { - return Status::Invalid("BlobDataFileWriterFactory path creator is empty."); - } - PAIMON_RETURN_NOT_OK( - writer->Init(options_.GetFileSystem(), path_creator_(), resources.writer_builder)); + PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), path_factory_->NewBlobPath(), + resources.writer_builder)); return std::unique_ptr>>( std::move(writer)); } diff --git a/src/paimon/core/io/blob_data_file_writer_factory.h b/src/paimon/core/io/blob_data_file_writer_factory.h index fd66b263..15286020 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.h +++ b/src/paimon/core/io/blob_data_file_writer_factory.h @@ -20,7 +20,6 @@ #pragma once #include -#include #include #include #include @@ -30,7 +29,6 @@ #include "paimon/core/io/data_file_writer.h" #include "paimon/core/io/data_file_writer_factory.h" #include "paimon/core/io/single_file_writer_factory.h" -#include "paimon/format/blob/blob_format_writer.h" #include "paimon/result.h" namespace arrow { @@ -48,15 +46,11 @@ class BlobDataFileWriterFactory : public DataFileWriterFactory, public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> { public: - using PathCreator = std::function; - BlobDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, const std::shared_ptr& path_factory, - PathCreator path_creator, - blob::BlobFormatWriter::WriteConsumer write_consumer, const std::shared_ptr& pool); Result>>> @@ -67,8 +61,6 @@ class BlobDataFileWriterFactory std::optional> write_cols_; std::shared_ptr seq_num_counter_; std::shared_ptr path_factory_; - PathCreator path_creator_; - blob::BlobFormatWriter::WriteConsumer write_consumer_; }; } // namespace paimon diff --git a/src/paimon/core/io/data_file_path_factory.h b/src/paimon/core/io/data_file_path_factory.h index 90ab01f3..b49154f1 100644 --- a/src/paimon/core/io/data_file_path_factory.h +++ b/src/paimon/core/io/data_file_path_factory.h @@ -64,12 +64,6 @@ class DataFilePathFactory : public PathFactory { return NewPathFromName(NewFileName(data_file_prefix_, ".blob")); } - /// Creates a new blob file path under the given external storage path for descriptor fields. - std::string NewExternalStorageBlobPath(const std::string& external_storage_path) const { - std::string file_name = NewFileName(data_file_prefix_, ".blob"); - return PathUtil::JoinPath(external_storage_path, file_name); - } - std::string NewPathFromName(const std::string& file_name) const { if (external_path_provider_ != nullptr) { return external_path_provider_->GetNextExternalDataPath(file_name); diff --git a/src/paimon/core/io/data_file_path_factory_test.cpp b/src/paimon/core/io/data_file_path_factory_test.cpp index 50010ee5..12618a2e 100644 --- a/src/paimon/core/io/data_file_path_factory_test.cpp +++ b/src/paimon/core/io/data_file_path_factory_test.cpp @@ -59,20 +59,6 @@ TEST_F(DataFilePathFactoryTest, TestNewPath) { ASSERT_EQ(factory_.NewPathFromName("index-file"), "/tmp/index-file"); } -TEST_F(DataFilePathFactoryTest, TestNewExternalStorageBlobPath) { - std::string blob_path1 = factory_.NewExternalStorageBlobPath("/tmp/external_blob"); - std::string blob_path2 = factory_.NewExternalStorageBlobPath("/tmp/external_blob"); - - // Paths are unique (counter increments) - ASSERT_NE(blob_path1, blob_path2); - // Both start with the external storage path joined with the data file prefix - ASSERT_TRUE(StringUtils::StartsWith(blob_path1, "/tmp/external_blob/data-")); - ASSERT_TRUE(StringUtils::StartsWith(blob_path2, "/tmp/external_blob/data-")); - // Both end with .blob extension - ASSERT_TRUE(StringUtils::EndsWith(blob_path1, ".blob")); - ASSERT_TRUE(StringUtils::EndsWith(blob_path2, ".blob")); -} - TEST_F(DataFilePathFactoryTest, TestNewPathWithDataFilePrefixAndExternalPath) { DataFilePathFactory factory; ASSERT_OK_AND_ASSIGN( diff --git a/src/paimon/core/io/external_storage_blob_writer.cpp b/src/paimon/core/io/external_storage_blob_writer.cpp deleted file mode 100644 index 1ea730ce..00000000 --- a/src/paimon/core/io/external_storage_blob_writer.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "paimon/core/io/external_storage_blob_writer.h" - -#include -#include - -#include "arrow/array/array_nested.h" -#include "arrow/array/builder_binary.h" -#include "arrow/c/bridge.h" -#include "arrow/type.h" -#include "paimon/common/data/blob_descriptor.h" -#include "paimon/common/data/blob_utils.h" -#include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/core/io/blob_data_file_writer_factory.h" -#include "paimon/core/io/data_file_path_factory.h" -#include "paimon/fs/file_system.h" -#include "paimon/memory/memory_pool.h" - -namespace paimon { - -ExternalStorageBlobWriter::ExternalStorageBlobWriter( - const std::shared_ptr& write_schema, - const std::set& external_storage_fields, const std::string& external_storage_path, - int64_t schema_id, const std::shared_ptr& seq_num_counter, - const std::shared_ptr& path_factory, const CoreOptions& options, - const std::shared_ptr& memory_pool) - : write_schema_(write_schema), - external_storage_fields_(external_storage_fields), - external_storage_path_(external_storage_path), - schema_id_(schema_id), - seq_num_counter_(seq_num_counter), - path_factory_(path_factory), - memory_pool_(memory_pool), - options_(options) {} - -Result> -ExternalStorageBlobWriter::CreateFieldRollingWriter(FieldWriter* field_writer) { - auto field = write_schema_->GetFieldByName(field_writer->field_name); - if (!field) { - return Status::Invalid("External storage field '{}' not found in write schema", - field_writer->field_name); - } - - auto single_field_schema = arrow::schema({field}); - auto write_consumer = [field_writer](std::unique_ptr descriptor) -> bool { - field_writer->captured_descriptors.push_back(std::move(descriptor)); - return true; // Always flush for single row. - }; - - std::vector write_cols = {field_writer->field_name}; - std::shared_ptr path_factory = path_factory_; - std::string external_storage_path = external_storage_path_; - auto writer_factory = std::make_shared( - options_, schema_id_, single_field_schema, write_cols, seq_num_counter_, path_factory, - [path_factory, external_storage_path]() { - return path_factory->NewExternalStorageBlobPath(external_storage_path); - }, - write_consumer, memory_pool_); - - return std::make_unique(options_.GetBlobTargetFileSize(), writer_factory); -} - -Status ExternalStorageBlobWriter::InitializeFieldWritersIfNeeded() { - if (initialized_) { - return Status::OK(); - } - for (int32_t i = 0; i < write_schema_->num_fields(); ++i) { - const auto& field = write_schema_->field(i); - if (external_storage_fields_.count(field->name()) > 0) { - FieldWriter fw; - fw.field_name = field->name(); - fw.field_index = i; - field_writers_.push_back(std::move(fw)); - } - } - // Create rolling writers after push_back so FieldWriter addresses are stable - // for the consumer lambda capture. - for (auto& fw : field_writers_) { - PAIMON_ASSIGN_OR_RAISE(fw.rolling_writer, CreateFieldRollingWriter(&fw)); - } - initialized_ = true; - return Status::OK(); -} - -Result> ExternalStorageBlobWriter::TransformField( - const std::shared_ptr& column, FieldWriter* field_writer) { - int64_t num_rows = column->length(); - - // Clear captured descriptors before processing this batch - field_writer->captured_descriptors.clear(); - - // Write each row via RollingFileWriter; the consumer captures the descriptor - for (int64_t row = 0; row < num_rows; ++row) { - std::shared_ptr slice = column->Slice(row, 1); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( - std::shared_ptr single_row_struct, - arrow::StructArray::Make({slice}, {field_writer->field_name})); - - ::ArrowArray c_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*single_row_struct, &c_array)); - PAIMON_RETURN_NOT_OK(field_writer->rolling_writer->Write(&c_array)); - } - - // Validate captured descriptor count - if (static_cast(field_writer->captured_descriptors.size()) != num_rows) { - return Status::Invalid( - "Captured descriptor count {} does not match row count {} for field '{}'", - field_writer->captured_descriptors.size(), num_rows, field_writer->field_name); - } - - // Build descriptor column from captured descriptors - arrow::LargeBinaryBuilder descriptor_builder; - PAIMON_RETURN_NOT_OK_FROM_ARROW(descriptor_builder.Reserve(num_rows)); - for (int64_t row = 0; row < num_rows; ++row) { - const auto& descriptor = field_writer->captured_descriptors[row]; - if (!descriptor) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(descriptor_builder.AppendNull()); - } else { - auto serialized = descriptor->Serialize(memory_pool_); - PAIMON_RETURN_NOT_OK_FROM_ARROW( - descriptor_builder.Append(serialized->data(), serialized->size())); - } - } - - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr descriptor_array, - descriptor_builder.Finish()); - return descriptor_array; -} - -Result> ExternalStorageBlobWriter::TransformBatch( - const std::shared_ptr& batch) { - if (external_storage_fields_.empty()) { - return batch; - } - - PAIMON_RETURN_NOT_OK(InitializeFieldWritersIfNeeded()); - - if (field_writers_.empty()) { - return batch; - } - - // Collect all arrays and field names from the original batch - std::vector> result_arrays; - std::vector result_names; - result_arrays.reserve(batch->num_fields()); - result_names.reserve(batch->num_fields()); - - for (int32_t col = 0; col < batch->num_fields(); ++col) { - result_names.push_back(batch->type()->field(col)->name()); - result_arrays.push_back(batch->field(col)); - } - - // Transform each external storage field and replace in result - for (FieldWriter& fw : field_writers_) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr descriptor_array, - TransformField(batch->field(fw.field_index), &fw)); - result_arrays[fw.field_index] = descriptor_array; - } - - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr result, - arrow::StructArray::Make(result_arrays, result_names)); - return result; -} - -Status ExternalStorageBlobWriter::Close() { - for (FieldWriter& fw : field_writers_) { - if (fw.rolling_writer) { - PAIMON_RETURN_NOT_OK(fw.rolling_writer->Close()); - } - } - return Status::OK(); -} - -void ExternalStorageBlobWriter::Abort() { - for (FieldWriter& fw : field_writers_) { - if (fw.rolling_writer) { - fw.rolling_writer->Abort(); - fw.rolling_writer.reset(); - } - } - field_writers_.clear(); -} - -} // namespace paimon diff --git a/src/paimon/core/io/external_storage_blob_writer.h b/src/paimon/core/io/external_storage_blob_writer.h deleted file mode 100644 index f4319cac..00000000 --- a/src/paimon/core/io/external_storage_blob_writer.h +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include - -#include "paimon/common/data/blob_descriptor.h" -#include "paimon/core/core_options.h" -#include "paimon/core/io/data_file_meta.h" -#include "paimon/core/io/rolling_file_writer.h" -#include "paimon/core/io/single_file_writer.h" -#include "paimon/logging.h" -#include "paimon/result.h" -#include "paimon/status.h" -namespace arrow { -class Schema; -class StructArray; -} // namespace arrow - -namespace paimon { - -class FileSystem; -class LongCounter; -class MemoryPool; -class DataFilePathFactory; - -/// Batch-oriented writer for descriptor BLOB fields that writes raw data to external storage. -/// -/// For each configured external_storage field, this writer: -/// 1. Uses RollingFileWriter (same infra as MultipleBlobFileWriter) with BlobFormatWriter -/// 2. Injects a WriteConsumer into BlobFormatWriter to capture each row's BlobDescriptor -/// 3. After writing a batch, constructs a descriptor column from captured descriptors -/// -/// After TransformBatch(), the returned StructArray has descriptor columns replaced with -/// serialized BlobDescriptor bytes (large_binary), ready to be written into the main data file. -class ExternalStorageBlobWriter { - public: - using BlobRollingWriter = RollingFileWriter<::ArrowArray*, std::shared_ptr>; - - ExternalStorageBlobWriter(const std::shared_ptr& write_schema, - const std::set& external_storage_fields, - const std::string& external_storage_path, int64_t schema_id, - const std::shared_ptr& seq_num_counter, - const std::shared_ptr& path_factory, - const CoreOptions& options, - const std::shared_ptr& memory_pool); - - /// Transforms a batch by writing external storage fields to .blob files and replacing - /// the BLOB values with serialized BlobDescriptor bytes. - Result> TransformBatch( - const std::shared_ptr& batch); - - /// Closes all internal blob writers and flushes pending data. - Status Close(); - - /// Aborts all internal blob writers. - void Abort(); - - private: - /// Per-field writer state for one external storage blob field. - struct FieldWriter { - std::string field_name; - int32_t field_index; - std::unique_ptr rolling_writer; - /// Descriptors captured by the WriteConsumer callback during writes. - std::vector> captured_descriptors; - }; - - /// Lazily initializes per-field writers on first call to TransformBatch. - Status InitializeFieldWritersIfNeeded(); - - /// Writes all rows of a single external blob field via RollingFileWriter and returns - /// a descriptor column (LargeBinary) built from captured BlobDescriptors. - Result> TransformField( - const std::shared_ptr& column, FieldWriter* field_writer); - - /// Creates a RollingFileWriter for one external storage blob field with consumer injected. - Result> CreateFieldRollingWriter(FieldWriter* field_writer); - - std::shared_ptr write_schema_; - std::set external_storage_fields_; - std::string external_storage_path_; - int64_t schema_id_; - std::shared_ptr seq_num_counter_; - std::shared_ptr path_factory_; - std::shared_ptr memory_pool_; - CoreOptions options_; - - std::vector field_writers_; - bool initialized_ = false; -}; - -} // namespace paimon diff --git a/src/paimon/core/io/external_storage_blob_writer_test.cpp b/src/paimon/core/io/external_storage_blob_writer_test.cpp deleted file mode 100644 index 32950d26..00000000 --- a/src/paimon/core/io/external_storage_blob_writer_test.cpp +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "paimon/core/io/external_storage_blob_writer.h" - -#include -#include - -#include "arrow/api.h" -#include "arrow/ipc/json_simple.h" -#include "gtest/gtest.h" -#include "paimon/common/data/blob_descriptor.h" -#include "paimon/common/data/blob_utils.h" -#include "paimon/common/utils/long_counter.h" -#include "paimon/core/core_options.h" -#include "paimon/core/io/data_file_path_factory.h" -#include "paimon/memory/memory_pool.h" -#include "paimon/testing/utils/testharness.h" - -namespace paimon::test { - -class ExternalStorageBlobWriterTest : public ::testing::Test { - protected: - void SetUp() override { - dir_ = UniqueTestDirectory::Create(); - ASSERT_TRUE(dir_); - - pool_ = GetDefaultPool(); - seq_num_counter_ = std::make_shared(0); - - // Create CoreOptions with blob format - ASSERT_OK_AND_ASSIGN(options_, CoreOptions::FromMap({})); - file_system_ = options_.GetFileSystem(); - - // Create external storage directory - external_storage_path_ = dir_->Str() + "/external_blob"; - ASSERT_OK(file_system_->Mkdirs(external_storage_path_)); - - // Initialize DataFilePathFactory - path_factory_ = std::make_shared(); - ASSERT_OK(path_factory_->Init(dir_->Str(), "blob", "data-", nullptr)); - - // Schema: int_col (int32) + blob_col (blob) - auto int_field = arrow::field("int_col", arrow::int32()); - auto blob_field = BlobUtils::ToArrowField("blob_col", false); - write_schema_ = arrow::schema({int_field, blob_field}); - } - - std::unique_ptr dir_; - std::shared_ptr pool_; - std::shared_ptr seq_num_counter_; - CoreOptions options_; - std::shared_ptr file_system_; - std::shared_ptr path_factory_; - std::shared_ptr write_schema_; - std::string external_storage_path_; -}; - -TEST_F(ExternalStorageBlobWriterTest, TestEmptyExternalFields) { - // No external storage fields -> TransformBatch returns original batch - ExternalStorageBlobWriter writer(write_schema_, /*external_storage_fields=*/{}, - external_storage_path_, /*schema_id=*/0, seq_num_counter_, - path_factory_, options_, pool_); - - auto input = std::static_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_schema_->fields()), - R"([[42, "hello"]])") - .ValueOrDie()); - - ASSERT_OK_AND_ASSIGN(auto result, writer.TransformBatch(input)); - ASSERT_TRUE(result->Equals(*input)); - - ASSERT_OK(writer.Close()); -} - -TEST_F(ExternalStorageBlobWriterTest, TestTransformBatchReplacesBlob) { - std::set external_fields = {"blob_col"}; - ExternalStorageBlobWriter writer(write_schema_, external_fields, external_storage_path_, - /*schema_id=*/0, seq_num_counter_, path_factory_, options_, - pool_); - - auto struct_type = arrow::struct_(write_schema_->fields()); - auto input = std::static_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(struct_type, R"([[10, "data1"], [20, "data2"]])") - .ValueOrDie()); - - auto original_int_col = input->field(0); - - ASSERT_OK_AND_ASSIGN(auto result, writer.TransformBatch(input)); - - // int_col should be unchanged - ASSERT_EQ(result->num_fields(), 2); - ASSERT_TRUE(result->field(0)->Equals(*original_int_col)); - - // blob_col should be replaced with serialized BlobDescriptors - auto descriptor_col = std::static_pointer_cast(result->field(1)); - ASSERT_EQ(descriptor_col->length(), 2); - - for (int64_t i = 0; i < 2; ++i) { - ASSERT_FALSE(descriptor_col->IsNull(i)); - auto view = descriptor_col->GetView(i); - ASSERT_OK_AND_ASSIGN(auto descriptor, - BlobDescriptor::Deserialize(view.data(), view.size())); - ASSERT_EQ(descriptor->Length(), 5); - ASSERT_TRUE(descriptor->Uri().find(external_storage_path_) != std::string::npos); - } - - ASSERT_OK(writer.Close()); -} - -TEST_F(ExternalStorageBlobWriterTest, TestAbort) { - std::set external_fields = {"blob_col"}; - ExternalStorageBlobWriter writer(write_schema_, external_fields, external_storage_path_, - /*schema_id=*/0, seq_num_counter_, path_factory_, options_, - pool_); - - auto input = std::static_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(write_schema_->fields()), - R"([[1, "abort_test"]])") - .ValueOrDie()); - - ASSERT_OK(writer.TransformBatch(input)); - - // Verify blob files exist before abort - std::vector> files_before; - ASSERT_OK(file_system_->ListDir(external_storage_path_, &files_before)); - ASSERT_FALSE(files_before.empty()); - - // Abort should clean up written blob files - writer.Abort(); - - std::vector> files_after; - ASSERT_OK(file_system_->ListDir(external_storage_path_, &files_after)); - ASSERT_TRUE(files_after.empty()); -} - -} // namespace paimon::test diff --git a/src/paimon/core/operation/blob_file_context.cpp b/src/paimon/core/operation/blob_file_context.cpp index 270a6c4a..047cc6fb 100644 --- a/src/paimon/core/operation/blob_file_context.cpp +++ b/src/paimon/core/operation/blob_file_context.cpp @@ -30,15 +30,11 @@ namespace paimon { BlobFileContext::BlobFileContext(std::set descriptor_fields, std::set view_fields, std::set inline_fields, - std::set external_storage_fields, - std::set blob_file_fields, - std::optional external_storage_path) + std::set blob_file_fields) : descriptor_fields_(std::move(descriptor_fields)), view_fields_(std::move(view_fields)), inline_fields_(std::move(inline_fields)), - external_storage_fields_(std::move(external_storage_fields)), - blob_file_fields_(std::move(blob_file_fields)), - external_storage_path_(std::move(external_storage_path)) {} + blob_file_fields_(std::move(blob_file_fields)) {} std::unique_ptr BlobFileContext::Create( const std::shared_ptr& schema, const CoreOptions& options) { @@ -81,17 +77,6 @@ std::unique_ptr BlobFileContext::Create( } } - // Populate external storage fields - std::set external_storage_fields; - for (const auto& name : options.GetBlobExternalStorageFields()) { - if (schema_blob_fields.count(name) > 0) { - external_storage_fields.insert(name); - } - } - - // Populate external storage path - std::optional external_storage_path = options.GetBlobExternalStoragePath(); - // Determine blob_file_fields: schema BLOB fields that are NOT inline std::set blob_file_fields; for (const auto& name : schema_blob_fields) { @@ -102,16 +87,11 @@ std::unique_ptr BlobFileContext::Create( return std::unique_ptr( new BlobFileContext(std::move(descriptor_fields), std::move(view_fields), - std::move(inline_fields), std::move(external_storage_fields), - std::move(blob_file_fields), std::move(external_storage_path))); + std::move(inline_fields), std::move(blob_file_fields))); } bool BlobFileContext::RequireBlobFileWriter() const { return !blob_file_fields_.empty(); } -bool BlobFileContext::RequireExternalStorageWriter() const { - return !external_storage_fields_.empty(); -} - } // namespace paimon diff --git a/src/paimon/core/operation/blob_file_context.h b/src/paimon/core/operation/blob_file_context.h index e3891d1f..ee3b7019 100644 --- a/src/paimon/core/operation/blob_file_context.h +++ b/src/paimon/core/operation/blob_file_context.h @@ -20,7 +20,6 @@ #pragma once #include -#include #include #include @@ -38,8 +37,6 @@ class CoreOptions; /// - descriptor_fields: stored as BlobDescriptor bytes inline in the main data file. /// - view_fields: stored as BlobViewStruct bytes inline in the main data file. /// - inline_fields: descriptor_fields ∪ view_fields. These stay in the main data file. -/// - external_storage_fields: subset of descriptor_fields whose raw data is written to an -/// external storage path (the descriptor still goes into the main data file). /// - blob_file_fields: BLOB fields that are NOT inline. These go into separate .blob files. class BlobFileContext { public: @@ -52,9 +49,6 @@ class BlobFileContext { /// Returns true if there are any BLOB fields that need a .blob file writer. bool RequireBlobFileWriter() const; - /// Returns true if there are any external storage fields that need an external writer. - bool RequireExternalStorageWriter() const; - const std::set& GetDescriptorFields() const { return descriptor_fields_; } @@ -67,31 +61,18 @@ class BlobFileContext { return inline_fields_; } - const std::set& GetExternalStorageFields() const { - return external_storage_fields_; - } - const std::set& GetBlobFileFields() const { return blob_file_fields_; } - const std::optional& GetExternalStoragePath() const { - return external_storage_path_; - } - private: BlobFileContext(std::set descriptor_fields, std::set view_fields, - std::set inline_fields, - std::set external_storage_fields, - std::set blob_file_fields, - std::optional external_storage_path); + std::set inline_fields, std::set blob_file_fields); std::set descriptor_fields_; std::set view_fields_; std::set inline_fields_; - std::set external_storage_fields_; std::set blob_file_fields_; - std::optional external_storage_path_; }; } // namespace paimon diff --git a/src/paimon/core/operation/blob_file_context_test.cpp b/src/paimon/core/operation/blob_file_context_test.cpp index 9565aaeb..a18dc333 100644 --- a/src/paimon/core/operation/blob_file_context_test.cpp +++ b/src/paimon/core/operation/blob_file_context_test.cpp @@ -69,7 +69,6 @@ TEST_F(BlobFileContextTest, AllInlineNoExternalStorage) { ASSERT_EQ(context->GetInlineFields(), std::set({"image", "video"})); ASSERT_TRUE(context->GetBlobFileFields().empty()); ASSERT_FALSE(context->RequireBlobFileWriter()); - ASSERT_FALSE(context->RequireExternalStorageWriter()); } TEST_F(BlobFileContextTest, MixedInlineAndBlobFile) { @@ -96,29 +95,6 @@ TEST_F(BlobFileContextTest, MixedInlineAndBlobFile) { // Requires blob file writer for video and audio ASSERT_TRUE(context->RequireBlobFileWriter()); - ASSERT_FALSE(context->RequireExternalStorageWriter()); -} - -TEST_F(BlobFileContextTest, ExternalStorageFields) { - auto schema = MakeSchema({"id"}, {"image", "video"}); - std::map opts_map = { - {Options::BLOB_DESCRIPTOR_FIELD, "image,video"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "image"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, "oss://bucket/blob/"}, - }; - ASSERT_OK_AND_ASSIGN(auto options, CoreOptions::FromMap(opts_map)); - auto context = BlobFileContext::Create(schema, options); - ASSERT_TRUE(context); - - ASSERT_EQ(context->GetDescriptorFields(), std::set({"image", "video"})); - ASSERT_EQ(context->GetInlineFields(), std::set({"image", "video"})); - ASSERT_EQ(context->GetExternalStorageFields(), std::set({"image"})); - ASSERT_TRUE(context->GetExternalStoragePath()); - ASSERT_EQ(context->GetExternalStoragePath(), "oss://bucket/blob/"); - ASSERT_TRUE(context->GetBlobFileFields().empty()); - - ASSERT_FALSE(context->RequireBlobFileWriter()); - ASSERT_TRUE(context->RequireExternalStorageWriter()); } TEST_F(BlobFileContextTest, ViewFields) { @@ -137,7 +113,6 @@ TEST_F(BlobFileContextTest, ViewFields) { ASSERT_EQ(context->GetBlobFileFields(), std::set({"raw_blob"})); ASSERT_TRUE(context->RequireBlobFileWriter()); - ASSERT_FALSE(context->RequireExternalStorageWriter()); } TEST_F(BlobFileContextTest, DescriptorAndViewTogether) { @@ -145,8 +120,6 @@ TEST_F(BlobFileContextTest, DescriptorAndViewTogether) { std::map opts_map = { {Options::BLOB_DESCRIPTOR_FIELD, "desc_blob"}, {Options::BLOB_VIEW_FIELD, "view_blob"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "desc_blob"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, "/tmp/ext/"}, }; ASSERT_OK_AND_ASSIGN(auto options, CoreOptions::FromMap(opts_map)); auto context = BlobFileContext::Create(schema, options); @@ -155,13 +128,9 @@ TEST_F(BlobFileContextTest, DescriptorAndViewTogether) { ASSERT_EQ(context->GetDescriptorFields(), std::set({"desc_blob"})); ASSERT_EQ(context->GetViewFields(), std::set({"view_blob"})); ASSERT_EQ(context->GetInlineFields(), std::set({"desc_blob", "view_blob"})); - ASSERT_EQ(context->GetExternalStorageFields(), std::set({"desc_blob"})); - ASSERT_TRUE(context->GetExternalStoragePath()); - ASSERT_EQ(context->GetExternalStoragePath(), "/tmp/ext/"); ASSERT_EQ(context->GetBlobFileFields(), std::set({"normal_blob"})); ASSERT_TRUE(context->RequireBlobFileWriter()); - ASSERT_TRUE(context->RequireExternalStorageWriter()); } TEST_F(BlobFileContextTest, PartialSchemaIgnoresAbsentFields) { @@ -170,8 +139,6 @@ TEST_F(BlobFileContextTest, PartialSchemaIgnoresAbsentFields) { std::map opts_map = { {Options::BLOB_DESCRIPTOR_FIELD, "image,audio"}, {Options::BLOB_VIEW_FIELD, "video"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "image,video"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, "oss://bucket/blob/"}, }; ASSERT_OK_AND_ASSIGN(auto options, CoreOptions::FromMap(opts_map)); auto context = BlobFileContext::Create(schema, options); @@ -181,13 +148,11 @@ TEST_F(BlobFileContextTest, PartialSchemaIgnoresAbsentFields) { ASSERT_EQ(context->GetDescriptorFields(), std::set({"image"})); ASSERT_TRUE(context->GetViewFields().empty()); ASSERT_EQ(context->GetInlineFields(), std::set({"image"})); - ASSERT_EQ(context->GetExternalStorageFields(), std::set({"image"})); // No non-inline blob field remains in the schema. ASSERT_TRUE(context->GetBlobFileFields().empty()); ASSERT_FALSE(context->RequireBlobFileWriter()); - ASSERT_TRUE(context->RequireExternalStorageWriter()); } TEST_F(BlobFileContextTest, PartialSchemaWithOnlyBlobFileField) { @@ -206,7 +171,6 @@ TEST_F(BlobFileContextTest, PartialSchemaWithOnlyBlobFileField) { ASSERT_EQ(context->GetBlobFileFields(), std::set({"audio"})); ASSERT_TRUE(context->RequireBlobFileWriter()); - ASSERT_FALSE(context->RequireExternalStorageWriter()); } } // namespace paimon diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 07a8c42b..8d1fcef1 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -453,9 +453,7 @@ Status SchemaValidation::ValidateBlobFields(const TableSchema& schema, const Cor const auto& configured_blob_names = options.GetBlobFields(); const auto& blob_descriptor_names = options.GetBlobDescriptorFields(); const auto& blob_view_names = options.GetBlobViewFields(); - const auto& blob_external_storage_names = options.GetBlobExternalStorageFields(); - if (configured_blob_names.empty() && blob_descriptor_names.empty() && blob_view_names.empty() && - blob_external_storage_names.empty()) { + if (configured_blob_names.empty() && blob_descriptor_names.empty() && blob_view_names.empty()) { return Status::OK(); } @@ -480,8 +478,6 @@ Status SchemaValidation::ValidateBlobFields(const TableSchema& schema, const Cor PAIMON_RETURN_NOT_OK( validate_blob_fields(blob_descriptor_names, Options::BLOB_DESCRIPTOR_FIELD)); PAIMON_RETURN_NOT_OK(validate_blob_fields(blob_view_names, Options::BLOB_VIEW_FIELD)); - PAIMON_RETURN_NOT_OK( - validate_blob_fields(blob_external_storage_names, Options::BLOB_EXTERNAL_STORAGE_FIELD)); std::set blob_descriptor_name_set(blob_descriptor_names.begin(), blob_descriptor_names.end()); @@ -492,22 +488,6 @@ Status SchemaValidation::ValidateBlobFields(const TableSchema& schema, const Cor Options::BLOB_DESCRIPTOR_FIELD)); } } - - for (const auto& blob_external_storage_name : blob_external_storage_names) { - if (blob_descriptor_name_set.count(blob_external_storage_name) == 0) { - return Status::Invalid( - fmt::format("Field '{}' in '{}' must also be in '{}'.", blob_external_storage_name, - Options::BLOB_EXTERNAL_STORAGE_FIELD, Options::BLOB_DESCRIPTOR_FIELD)); - } - } - if (!blob_external_storage_names.empty()) { - auto external_storage_path = options.GetBlobExternalStoragePath(); - if (!external_storage_path || external_storage_path->empty()) { - return Status::Invalid(fmt::format("'{}' must be set when '{}' is configured.", - Options::BLOB_EXTERNAL_STORAGE_PATH, - Options::BLOB_EXTERNAL_STORAGE_FIELD)); - } - } return Status::OK(); } diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 85a3e733..1e411f98 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -104,14 +104,11 @@ TEST(SchemaValidationTest, TestWithBlobField) { auto schema = arrow::schema(fields); std::vector primary_keys = {}; std::vector partition_keys = {"f1"}; - std::map options = { - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "f3"}, - {Options::BLOB_VIEW_FIELD, "f4"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "f3"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, "FILE:///tmp/blob_external_storage/"}}; + std::map options = {{Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "f3"}, + {Options::BLOB_VIEW_FIELD, "f4"}}; ASSERT_OK_AND_ASSIGN( std::shared_ptr table_schema, TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options)); @@ -150,42 +147,6 @@ TEST(SchemaValidationTest, TestWithBlobField) { SchemaValidation::ValidateTableSchema(*table_schema), "Field 'f3' in 'blob-view-field' can not also be in 'blob-descriptor-field'."); } - { - arrow::FieldVector fields = {f0, f1, f2, f3, f4}; - auto schema = arrow::schema(fields); - std::vector primary_keys = {}; - std::vector partition_keys = {"f1"}; - std::map options = { - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "f3"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "f4"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, "FILE:///tmp/blob_external_storage/"}}; - ASSERT_OK_AND_ASSIGN( - std::shared_ptr table_schema, - TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options)); - ASSERT_NOK_WITH_MSG( - SchemaValidation::ValidateTableSchema(*table_schema), - "Field 'f4' in 'blob-external-storage-field' must also be in 'blob-descriptor-field'."); - } - { - arrow::FieldVector fields = {f0, f1, f2, f3}; - auto schema = arrow::schema(fields); - std::vector primary_keys = {}; - std::vector partition_keys = {"f1"}; - std::map options = {{Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "f3"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "f3"}}; - ASSERT_OK_AND_ASSIGN( - std::shared_ptr table_schema, - TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options)); - ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), - "'blob-external-storage-path' must be set when " - "'blob-external-storage-field' is configured."); - } { arrow::FieldVector fields = {f0, f1, f2, f3}; auto schema = arrow::schema(fields); diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 61960e5c..9b2b3206 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -237,8 +237,7 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { std::shared_ptr blob_field = BlobUtils::ToArrowField("blob_col"); auto struct_type = arrow::struct_({blob_field}); ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream, struct_type, - /*write_consumer=*/nullptr, file_system, pool_)); + BlobFormatWriter::Create(output_stream, struct_type, file_system, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index fc4042ee..7da5810c 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -38,15 +38,9 @@ namespace paimon::blob { BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, - WriteConsumer write_consumer, const std::shared_ptr& fs, const std::shared_ptr& pool) - : out_(out), - uri_(uri), - data_type_(data_type), - fs_(fs), - pool_(pool), - write_consumer_(std::move(write_consumer)) { + : out_(out), uri_(uri), data_type_(data_type), fs_(fs), pool_(pool) { metrics_ = std::make_shared(); tmp_buffer_ = Bytes::AllocateBytes(kTmpBufferSize, pool_.get()); magic_number_bytes_ = IntegerToLittleEndian(BlobDefs::kMagicNumber, pool_); @@ -54,8 +48,7 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con Result> BlobFormatWriter::Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - WriteConsumer write_consumer, const std::shared_ptr& fs, - const std::shared_ptr& pool) { + const std::shared_ptr& fs, const std::shared_ptr& pool) { if (out == nullptr) { return Status::Invalid("blob format writer create failed. out is nullptr"); } @@ -74,8 +67,7 @@ Result> BlobFormatWriter::Create( fmt::format("field {} is not BLOB", data_type->field(0)->ToString())); } PAIMON_ASSIGN_OR_RAISE(std::string uri, out->GetUri()); - return std::unique_ptr( - new BlobFormatWriter(out, uri, data_type, std::move(write_consumer), fs, pool)); + return std::unique_ptr(new BlobFormatWriter(out, uri, data_type, fs, pool)); } Status BlobFormatWriter::AddBatch(ArrowArray* batch) { @@ -99,9 +91,6 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { // Child-level null: record kNullBinLength, skip data writing (aligned with Java) if (child_array->IsNull(0)) { bin_lengths_.push_back(BlobDefs::kNullBinLength); - if (write_consumer_) { - write_consumer_(/*descriptor=*/nullptr); - } return Status::OK(); } @@ -113,28 +102,7 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { arrow::internal::checked_cast(*child_array); assert(blob_array.length() == 1); PAIMON_RETURN_NOT_OK(WriteBlob(blob_array.GetView(0))); - - if (write_consumer_) { - // Construct BlobDescriptor from the blob just written. - // blob format: magic(4) + content + bin_length(8) + crc32(4) - // bin_length covers all of the above, so content_length = bin_length - 16. - // The stream is now positioned at the end of crc32, i.e., previous_pos + bin_length. - int64_t bin_length = bin_lengths_.back(); - PAIMON_ASSIGN_OR_RAISE(int64_t end_pos, out_->GetPos()); - int64_t blob_start_pos = end_pos - bin_length; - int64_t content_offset = blob_start_pos + BlobDefs::kContentStartOffset; - int64_t content_length = bin_length - BlobDefs::kTotalMetaLength; - - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr descriptor, - BlobDescriptor::Create(uri_, content_offset, content_length)); - bool should_flush = write_consumer_(std::move(descriptor)); - if (should_flush) { - PAIMON_RETURN_NOT_OK(Flush()); - } - } else { - // Java does not flush when writeConsumer is null. - PAIMON_RETURN_NOT_OK(Flush()); - } + PAIMON_RETURN_NOT_OK(Flush()); return Status::OK(); } diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 9a78c2a6..50641d9c 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -19,7 +19,6 @@ #pragma once #include -#include #include #include #include @@ -41,7 +40,6 @@ struct ArrowArray; namespace paimon { class Blob; -class BlobDescriptor; class FileSystem; class Metrics; class OutputStream; @@ -53,15 +51,9 @@ namespace paimon::blob { // https://cwiki.apache.org/confluence/display/PAIMON/PIP-35%3A+Introduce+Blob+to+store+multimodal+data class BlobFormatWriter : public FormatWriter { public: - /// Callback invoked after each blob row is written. - /// Receives the BlobDescriptor of the written blob (nullptr for null blobs). - /// Similar to Java's BlobConsumer. Returns true if the output stream should be flushed. - using WriteConsumer = std::function descriptor)>; - static Result> Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - WriteConsumer write_consumer, const std::shared_ptr& fs, - const std::shared_ptr& pool); + const std::shared_ptr& fs, const std::shared_ptr& pool); Status AddBatch(ArrowArray* batch) override; @@ -80,7 +72,7 @@ class BlobFormatWriter : public FormatWriter { private: BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, - WriteConsumer write_consumer, const std::shared_ptr& fs, + const std::shared_ptr& fs, const std::shared_ptr& pool); Status WriteBlob(std::string_view blob_data); @@ -105,7 +97,6 @@ class BlobFormatWriter : public FormatWriter { std::shared_ptr fs_; std::shared_ptr pool_; std::shared_ptr metrics_; - WriteConsumer write_consumer_; }; } // namespace paimon::blob diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index 81ee781f..4962b4af 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -24,7 +24,6 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" -#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/stream_utils.h" #include "paimon/data/blob.h" @@ -93,9 +92,9 @@ INSTANTIATE_TEST_SUITE_P(BlobAsDescriptor, BlobFormatWriterTest, ::testing::Valu TEST_P(BlobFormatWriterTest, TestSimple) { // write - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); std::vector> expected_blobs; std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; @@ -153,82 +152,39 @@ TEST_P(BlobFormatWriterTest, TestSimple) { } } -TEST_P(BlobFormatWriterTest, TestWriteConsumerReceivesDescriptors) { - std::vector> captured_descriptors; - BlobFormatWriter::WriteConsumer consumer = - [&captured_descriptors](std::unique_ptr descriptor) -> bool { - captured_descriptors.push_back(std::move(descriptor)); - return true; // request flush - }; - - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, consumer, file_system_, pool_)); - - // Write a normal blob row - std::string file = paimon::test::GetDataDir() + "/xxhash.data"; - ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, - Blob::FromPath(file, /*offset=*/0, /*length=*/91)); - ASSERT_OK_AND_ASSIGN(auto array, PrepareBlobArray(blob)); - ASSERT_OK(AddBatchOnce(writer, array)); - - ASSERT_EQ(captured_descriptors.size(), 1); - ASSERT_TRUE(captured_descriptors[0]); - ASSERT_EQ(captured_descriptors[0]->Uri(), dir_->Str() + "/file.blob"); - ASSERT_EQ(captured_descriptors[0]->Offset(), 4); // after magic(4) - ASSERT_EQ(captured_descriptors[0]->Length(), 91); - - // Write a null blob row — consumer should receive nullptr descriptor - arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), - {std::make_shared()}); - auto blob_builder = static_cast(struct_builder.field_builder(0)); - ASSERT_TRUE(struct_builder.Append().ok()); - ASSERT_TRUE(blob_builder->AppendNull().ok()); - std::shared_ptr null_array; - ASSERT_TRUE(struct_builder.Finish(&null_array).ok()); - ASSERT_OK(AddBatchOnce(writer, null_array)); - - ASSERT_EQ(captured_descriptors.size(), 2); - ASSERT_FALSE(captured_descriptors[1]); - - ASSERT_OK(writer->Finish()); -} - TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) { // Test with nullptr output stream - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(nullptr, struct_type_, /*write_consumer=*/nullptr, - file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(nullptr, struct_type_, file_system_, pool_), "blob format writer create failed. out is nullptr"); // Test with nullptr data type - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, nullptr, - /*write_consumer=*/nullptr, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, nullptr, file_system_, pool_), "blob format writer create failed. data_type is nullptr"); // Test with nullptr memory pool - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, nullptr), - "blob format writer create failed. pool is nullptr"); + ASSERT_NOK_WITH_MSG( + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, nullptr), + "blob format writer create failed. pool is nullptr"); // Test with invalid field count (more than 1 field) auto multi_field_type = arrow::struct_( {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, multi_field_type, - /*write_consumer=*/nullptr, file_system_, pool_), - "blob data type field number 2 is not 1"); + ASSERT_NOK_WITH_MSG( + BlobFormatWriter::Create(output_stream_, multi_field_type, file_system_, pool_), + "blob data type field number 2 is not 1"); // Test with non-blob field (missing blob metadata) auto non_blob_field = arrow::field("regular_col", arrow::binary()); auto non_blob_type = arrow::struct_({non_blob_field}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, non_blob_type, - /*write_consumer=*/nullptr, file_system_, pool_), - "field regular_col: binary is not BLOB"); + ASSERT_NOK_WITH_MSG( + BlobFormatWriter::Create(output_stream_, non_blob_type, file_system_, pool_), + "field regular_col: binary is not BLOB"); } TEST_P(BlobFormatWriterTest, TestInvalidCase) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); // Test nullptr batch ASSERT_NOK_WITH_MSG(writer->AddBatch(nullptr), @@ -245,9 +201,9 @@ TEST_P(BlobFormatWriterTest, TestInvalidCase) { } TEST_P(BlobFormatWriterTest, TestAddBatchWithInvalidBatchLength) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); // Test batch with wrong length (not 1) arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), @@ -273,9 +229,9 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithInvalidBatchLength) { } TEST_P(BlobFormatWriterTest, TestReachTargetSize) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); // Initially should not reach target size ASSERT_OK_AND_ASSIGN(bool reached, writer->ReachTargetSize(true, 1000)); @@ -298,9 +254,9 @@ TEST_P(BlobFormatWriterTest, TestReachTargetSize) { } TEST_P(BlobFormatWriterTest, TestGetWriterMetrics) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); auto metrics = writer->GetWriterMetrics(); ASSERT_TRUE(metrics); @@ -308,9 +264,9 @@ TEST_P(BlobFormatWriterTest, TestGetWriterMetrics) { TEST_P(BlobFormatWriterTest, TestEmptyWriter) { // Test creating a writer and finishing without adding any data - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -329,9 +285,9 @@ TEST_P(BlobFormatWriterTest, TestEmptyWriter) { } TEST_P(BlobFormatWriterTest, TestLargeBlob) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); // Create a temporary large file for testing std::string large_file_path = dir_->Str() + "/large_test_file.bin"; @@ -384,9 +340,9 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { } TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); // Write one row with child-level null blob arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), @@ -432,18 +388,18 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { ASSERT_TRUE(struct_builder2.Finish(&null_struct_array).ok()); auto null_c_array = std::make_unique(); ASSERT_TRUE(arrow::ExportArray(*null_struct_array, null_c_array.get()).ok()); - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer2, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer2, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); ASSERT_NOK_WITH_MSG(writer2->AddBatch(null_c_array.get()), "BlobFormatWriter does not support struct-level null."); ArrowArrayRelease(null_c_array.get()); } TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, - /*write_consumer=*/nullptr, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); // Create a zero-length file std::string zero_file_path = dir_->Str() + "/zero_length_file.bin"; diff --git a/src/paimon/format/blob/blob_writer_builder.h b/src/paimon/format/blob/blob_writer_builder.h index d7aae78c..2b594e72 100644 --- a/src/paimon/format/blob/blob_writer_builder.h +++ b/src/paimon/format/blob/blob_writer_builder.h @@ -61,19 +61,13 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { return this; } - /// Sets a write consumer that will be called after each blob row is written. - BlobWriterBuilder* WithWriteConsumer(BlobFormatWriter::WriteConsumer consumer) { - write_consumer_ = std::move(consumer); - return this; - } - Result> Build(const std::shared_ptr& out, const std::string& compression) override { assert(out); if (fs_ == nullptr) { return Status::Invalid("File system is nullptr. Please call WithFileSystem() first."); } - return BlobFormatWriter::Create(out, data_type_, write_consumer_, fs_, pool_); + return BlobFormatWriter::Create(out, data_type_, fs_, pool_); } private: @@ -81,7 +75,6 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { std::shared_ptr data_type_; std::map options_; std::shared_ptr fs_; - BlobFormatWriter::WriteConsumer write_consumer_; }; } // namespace paimon::blob diff --git a/src/paimon/format/blob/blob_writer_builder_test.cpp b/src/paimon/format/blob/blob_writer_builder_test.cpp index 8052a97e..6adbca72 100644 --- a/src/paimon/format/blob/blob_writer_builder_test.cpp +++ b/src/paimon/format/blob/blob_writer_builder_test.cpp @@ -18,16 +18,10 @@ #include "paimon/format/blob/blob_writer_builder.h" -#include - #include "arrow/api.h" -#include "arrow/c/bridge.h" #include "gtest/gtest.h" -#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/arrow/status_utils.h" -#include "paimon/defs.h" -#include "paimon/format/format_writer.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/testing/utils/testharness.h" @@ -61,34 +55,4 @@ TEST_F(BlobWriterBuilderTest, TestSimple) { ASSERT_OK(builder.Build(output_stream_, "none")); } -TEST_F(BlobWriterBuilderTest, TestWithWriteConsumer) { - std::vector> captured; - BlobWriterBuilder builder(struct_type_, {{Options::BLOB_AS_DESCRIPTOR, "false"}}); - builder.WithFileSystem(file_system_); - builder.WithWriteConsumer([&captured](std::unique_ptr descriptor) -> bool { - captured.push_back(std::move(descriptor)); - return true; - }); - - ASSERT_OK_AND_ASSIGN(auto writer, builder.Build(output_stream_, "none")); - - // Build a single-row struct array with raw blob data - arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), - {std::make_shared()}); - auto blob_builder = static_cast(struct_builder.field_builder(0)); - ASSERT_TRUE(struct_builder.Append().ok()); - ASSERT_TRUE(blob_builder->Append("hello", 5).ok()); - std::shared_ptr array; - ASSERT_TRUE(struct_builder.Finish(&array).ok()); - - auto c_array = std::make_unique(); - ASSERT_TRUE(arrow::ExportArray(*array, c_array.get()).ok()); - ASSERT_OK(writer->AddBatch(c_array.get())); - - ASSERT_EQ(captured.size(), 1); - ASSERT_TRUE(captured[0]); - ASSERT_EQ(captured[0]->Length(), 5); - ASSERT_OK(writer->Finish()); -} - } // namespace paimon::blob::test diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index f0e6f2da..9dcef3ad 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -1680,11 +1680,11 @@ TEST_P(BlobTableInteTest, TestReadTableWithMultiBlobFields) { } } -TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWithoutExternalStorage) { +TEST_P(BlobTableInteTest, TestBlobDescriptorField) { if (GetParam() == "lance") { return; } - // Two blob fields configured via BLOB_DESCRIPTOR_FIELD, no external storage. + // Two blob fields configured via BLOB_DESCRIPTOR_FIELD and stored inline as descriptors. arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), BlobUtils::ToArrowField("b1", true)}; @@ -1713,7 +1713,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWithoutExternalStorage) { WriteArray(table_path, {}, schema->field_names(), {desc_array})); ASSERT_OK(Commit(table_path, commit_msgs)); - // Scan and verify DataFileMeta: no external storage -> write_cols should be nullopt + // Scan and verify DataFileMeta: all blob fields are inline descriptors, so write_cols is unset. ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); VerifyDataFileMetas(plan, /*expected_file_count=*/1, /*expected_row_counts=*/{3}, /*expected_min_seqs=*/{1}, /*expected_max_seqs=*/{1}, @@ -1736,144 +1736,12 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWithoutExternalStorage) { ASSERT_TRUE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); } -TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWithExternalStorage) { - if (GetParam() == "lance") { - return; - } - // Two blob fields configured via BLOB_DESCRIPTOR_FIELD + BLOB_EXTERNAL_STORAGE_FIELD - // with BLOB_EXTERNAL_STORAGE_PATH pointing to blob_dir_. - arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), - BlobUtils::ToArrowField("b0", true), - BlobUtils::ToArrowField("b1", true)}; - - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, - {Options::FILE_SYSTEM, "local"}}; - CreateTable(fields, /*partition_keys=*/{}, options); - std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - - // Input uses plain raw bytes for readability - std::string raw_json = R"([ - [1, "image_data_0", "video_data_0"], - [2, "image_data_1", "video_data_1"], - [3, "image_data_2", "video_data_2"] - ])"; - auto raw_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"b0", "b1"})); - - // write descriptor array - auto schema = arrow::schema(fields); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, - WriteArray(table_path, {}, schema->field_names(), {desc_array})); - ASSERT_OK(Commit(table_path, commit_msgs)); - - // Scan and verify DataFileMeta: with external storage -> write_cols should be explicit - ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); - VerifyDataFileMetas(plan, /*expected_file_count=*/1, /*expected_row_counts=*/{3}, - /*expected_min_seqs=*/{1}, /*expected_max_seqs=*/{1}, - /*expected_first_row_ids=*/{0}, - /*expected_write_cols=*/{std::vector{"f0", "b0", "b1"}}); - - // Read and resolve descriptors back to raw bytes - std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; - ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, - /*predicate=*/nullptr, read_options)); - ASSERT_TRUE(result.chunked_array); - auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); - auto read_struct = std::dynamic_pointer_cast(read_concat); - ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); - ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); - ASSERT_TRUE(resolved->Equals(expected_with_rk)); - - // Descriptor bytes should differ (repacked by external storage) - ASSERT_FALSE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); - ASSERT_FALSE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); -} - -TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialExternalStorage) { - if (GetParam() == "lance") { - return; - } - // 4 blob fields: b0,b1 have external storage, b2,b3 are descriptor-only (no external storage). - arrow::FieldVector fields = { - arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), - BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), - BlobUtils::ToArrowField("b3", true)}; - - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1,b2,b3"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, - {Options::FILE_SYSTEM, "local"}}; - CreateTable(fields, /*partition_keys=*/{}, options); - std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - - // Input uses plain raw bytes for readability; some blob fields are null - std::string raw_json = R"([ - [1, "img_0", null, "doc_0", "log_0"], - [2, null, "vid_1", null, "log_1"], - [3, "img_2", "vid_2", "doc_2", null ] - ])"; - auto raw_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto desc_array, - ConvertRawBlobToDescriptor(raw_array, {"b0", "b1", "b2", "b3"})); - - // write descriptor array - auto schema = arrow::schema(fields); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, - WriteArray(table_path, {}, schema->field_names(), {desc_array})); - ASSERT_OK(Commit(table_path, commit_msgs)); - - // Scan and verify DataFileMeta: external storage on b0,b1 -> write_cols should be explicit - ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); - VerifyDataFileMetas( - plan, /*expected_file_count=*/1, /*expected_row_counts=*/{3}, - /*expected_min_seqs=*/{1}, /*expected_max_seqs=*/{1}, - /*expected_first_row_ids=*/{0}, - /*expected_write_cols=*/{std::vector{"f0", "b0", "b1", "b2", "b3"}}); - - // Read and resolve all descriptors back to raw bytes - std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; - ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, - /*predicate=*/nullptr, read_options)); - ASSERT_TRUE(result.chunked_array); - auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); - auto read_struct = std::dynamic_pointer_cast(read_concat); - ASSERT_OK_AND_ASSIGN(auto resolved, - ConvertDescriptorToRawBlob(read_struct, {"b0", "b1", "b2", "b3"})); - ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); - ASSERT_TRUE(resolved->Equals(expected_with_rk)); - - // b0,b1 repacked by external storage, should differ - ASSERT_FALSE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); - ASSERT_FALSE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); - // b2,b3 inline descriptor, should match - ASSERT_TRUE(read_struct->GetFieldByName("b2")->Equals(desc_array->GetFieldByName("b2"))); - ASSERT_TRUE(read_struct->GetFieldByName("b3")->Equals(desc_array->GetFieldByName("b3"))); -} - TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) { if (GetParam() == "lance") { return; } - // 4 blob fields: b0,b1 are descriptor (inline), b2,b3 are regular blob (written to .blob - // files). No external storage. + // 4 blob fields: b0,b1 are inline descriptors; b2,b3 are regular blob fields written to + // .blob files. arrow::FieldVector fields = { arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), @@ -1932,243 +1800,21 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) { ASSERT_TRUE(resolved->Equals(expected_with_rk)); } -TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialExternalStorageRepack) { - if (GetParam() == "lance") { - return; - } - // 4 blob fields: b0,b1 are descriptor + external-storage-field WITH external-storage-path. - // b2,b3 are regular blob (written to .blob files). - // All blob descriptors get repacked by external storage or .blob writer. - arrow::FieldVector fields = { - arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), - BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), - BlobUtils::ToArrowField("b3", true)}; - - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, - {Options::FILE_SYSTEM, "local"}}; - CreateTable(fields, /*partition_keys=*/{}, options); - std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - - // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls - std::string raw_json = R"([ - [1, "img_0", null, "raw_2_0", "raw_3_0"], - [2, "img_1", "vid_1", "raw_2_1", null ], - [3, "img_2", null, "raw_2_2", "raw_3_2" ] - ])"; - auto raw_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto desc_array, - ConvertRawBlobToDescriptor(raw_array, {"b0", "b1", "b2", "b3"})); - - auto schema = arrow::schema(fields); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, - WriteArray(table_path, {}, schema->field_names(), {desc_array})); - ASSERT_OK(Commit(table_path, commit_msgs)); - - // b0,b1 repacked to external storage; b2,b3 go to .blob files. - // Main file contains f0,b0,b1; .blob files for b2 and b3. - ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); - VerifyDataFileMetas(plan, /*expected_file_count=*/3, /*expected_row_counts=*/{3, 3, 3}, - /*expected_min_seqs=*/{1, 1, 1}, /*expected_max_seqs=*/{1, 1, 1}, - /*expected_first_row_ids=*/{0, 0, 0}, - /*expected_write_cols=*/ - {std::vector{"f0", "b0", "b1"}, std::vector{"b2"}, - std::vector{"b3"}}); - - std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; - ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, - /*predicate=*/nullptr, read_options)); - ASSERT_TRUE(result.chunked_array); - auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); - auto read_struct = std::dynamic_pointer_cast(read_concat); - - // Resolve descriptors back to raw bytes and compare - ASSERT_OK_AND_ASSIGN(auto resolved, - ConvertDescriptorToRawBlob(read_struct, {"b0", "b1", "b2", "b3"})); - ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); - ASSERT_TRUE(resolved->Equals(expected_with_rk)); - - // All blob columns should differ from input desc_array (all repacked) - ASSERT_FALSE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); - ASSERT_FALSE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); - ASSERT_FALSE(read_struct->GetFieldByName("b2")->Equals(desc_array->GetFieldByName("b2"))); - ASSERT_FALSE(read_struct->GetFieldByName("b3")->Equals(desc_array->GetFieldByName("b3"))); -} - -TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialExternalStorageSingleField) { - if (GetParam() == "lance") { - return; - } - // 4 blob fields: b0,b1 are descriptor; only b1 has external storage. - // b2,b3 are regular blob (written to .blob files). - arrow::FieldVector fields = { - arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), - BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), - BlobUtils::ToArrowField("b3", true)}; - - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, - {Options::FILE_SYSTEM, "local"}}; - CreateTable(fields, /*partition_keys=*/{}, options); - std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - - // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls - std::string raw_json = R"([ - [1, "img_0", null, "raw_2_0", "raw_3_0"], - [2, "img_1", "vid_1", "raw_2_1", null ], - [3, "img_2", null, "raw_2_2", "raw_3_2" ] - ])"; - auto raw_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto desc_array, - ConvertRawBlobToDescriptor(raw_array, {"b0", "b1", "b2", "b3"})); - - auto schema = arrow::schema(fields); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, - WriteArray(table_path, {}, schema->field_names(), {desc_array})); - ASSERT_OK(Commit(table_path, commit_msgs)); - - // b1 repacked to external storage; b2,b3 go to .blob files; b0 stays inline in main file. - // Main file contains f0,b0,b1; .blob files for b2 and b3. - ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); - VerifyDataFileMetas(plan, /*expected_file_count=*/3, /*expected_row_counts=*/{3, 3, 3}, - /*expected_min_seqs=*/{1, 1, 1}, /*expected_max_seqs=*/{1, 1, 1}, - /*expected_first_row_ids=*/{0, 0, 0}, - /*expected_write_cols=*/ - {std::vector{"f0", "b0", "b1"}, std::vector{"b2"}, - std::vector{"b3"}}); - - std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; - ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, - /*predicate=*/nullptr, read_options)); - ASSERT_TRUE(result.chunked_array); - auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); - auto read_struct = std::dynamic_pointer_cast(read_concat); - - // Resolve all descriptors back to raw bytes and compare - ASSERT_OK_AND_ASSIGN(auto resolved, - ConvertDescriptorToRawBlob(read_struct, {"b0", "b1", "b2", "b3"})); - ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); - ASSERT_TRUE(resolved->Equals(expected_with_rk)); - - // b0 is inline descriptor (not repacked), should match input - ASSERT_TRUE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); - // b1 is repacked by external storage, should differ - ASSERT_FALSE(read_struct->GetFieldByName("b1")->Equals(desc_array->GetFieldByName("b1"))); - // b2,b3 are repacked by .blob writer, should differ - ASSERT_FALSE(read_struct->GetFieldByName("b2")->Equals(desc_array->GetFieldByName("b2"))); - ASSERT_FALSE(read_struct->GetFieldByName("b3")->Equals(desc_array->GetFieldByName("b3"))); -} - -TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialExternalStorageNoAsDescriptor) { - if (GetParam() == "lance") { - return; - } - // Same as TestBlobDescriptorFieldPartialExternalStorageSingleField but without - // BLOB_AS_DESCRIPTOR in table options. Only b0 is explicitly converted to descriptor before - // write. b1 is written as raw bytes but still configured as descriptor field, so paimon should - // auto-convert it to descriptor internally (write auto-detects descriptor via magic header). - // After read with BLOB_AS_DESCRIPTOR=true, b0 and b1 are both stored as descriptor. - arrow::FieldVector fields = { - arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), - BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), - BlobUtils::ToArrowField("b3", true)}; - - std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, - {Options::FILE_SYSTEM, "local"}}; - CreateTable(fields, /*partition_keys=*/{}, options); - std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); - - // b0: all non-null, b1: has nulls, b2: all non-null, b3: has nulls - std::string raw_json = R"([ - [1, "img_0", null, "raw_2_0", "raw_3_0"], - [2, "img_1", "vid_1", "raw_2_1", null ], - [3, "img_2", null, "raw_2_2", "raw_3_2" ] - ])"; - auto raw_array = std::dynamic_pointer_cast( - arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); - // Only convert b0 to descriptor; b1,b2,b3 remain as raw bytes - ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"b0"})); - - auto schema = arrow::schema(fields); - ASSERT_OK_AND_ASSIGN(auto commit_msgs, - WriteArray(table_path, {}, schema->field_names(), {desc_array})); - ASSERT_OK(Commit(table_path, commit_msgs)); - - // b1 repacked to external storage; b2,b3 go to .blob files; b0 stays inline in main file. - ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); - VerifyDataFileMetas(plan, /*expected_file_count=*/3, /*expected_row_counts=*/{3, 3, 3}, - /*expected_min_seqs=*/{1, 1, 1}, /*expected_max_seqs=*/{1, 1, 1}, - /*expected_first_row_ids=*/{0, 0, 0}, - /*expected_write_cols=*/ - {std::vector{"f0", "b0", "b1"}, std::vector{"b2"}, - std::vector{"b3"}}); - - std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "false"}}; - ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan, - /*predicate=*/nullptr, read_options)); - ASSERT_TRUE(result.chunked_array); - auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); - auto read_struct = std::dynamic_pointer_cast(read_concat); - - // After read, b0 and b1 are both descriptor-stored; resolve all back to raw bytes - ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"b0", "b1"})); - ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(raw_array)); - ASSERT_TRUE(resolved->Equals(expected_with_rk)); - - // b0 is inline descriptor (not repacked), should match input desc_array - ASSERT_TRUE(read_struct->GetFieldByName("b0")->Equals(desc_array->GetFieldByName("b0"))); -} - TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { if (GetParam() == "lance") { return; } - // Similar to TestBlobDescriptorFieldPartialExternalStorageNoAsDescriptor but: - // 1. Multiple write+commit rounds - // 2. Read schema is shuffled: b3, b2, b1, b0, f0 + // Multiple write+commit rounds with a shuffled read schema: b3, b2, b1, b0, f0. arrow::FieldVector fields = { arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), BlobUtils::ToArrowField("b3", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, - {Options::FILE_SYSTEM, "local"}}; + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); auto schema = arrow::schema(fields); @@ -2180,7 +1826,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { ])"; auto raw_array_1 = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json_1).ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto desc_array_1, ConvertRawBlobToDescriptor(raw_array_1, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto desc_array_1, ConvertRawBlobToDescriptor(raw_array_1, {"b0", "b1"})); ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, WriteArray(table_path, {}, schema->field_names(), {desc_array_1})); ASSERT_OK(Commit(table_path, commit_msgs_1)); @@ -2192,7 +1838,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { ])"; auto raw_array_2 = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json_2).ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto desc_array_2, ConvertRawBlobToDescriptor(raw_array_2, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto desc_array_2, ConvertRawBlobToDescriptor(raw_array_2, {"b0", "b1"})); ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, WriteArray(table_path, {}, schema->field_names(), {desc_array_2})); ASSERT_OK(Commit(table_path, commit_msgs_2)); @@ -2204,7 +1850,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { ])"; auto raw_array_3 = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json_3).ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto desc_array_3, ConvertRawBlobToDescriptor(raw_array_3, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto desc_array_3, ConvertRawBlobToDescriptor(raw_array_3, {"b0", "b1"})); ASSERT_OK_AND_ASSIGN(auto commit_msgs_3, WriteArray(table_path, {}, schema->field_names(), {desc_array_3})); ASSERT_OK(Commit(table_path, commit_msgs_3)); @@ -2532,8 +2178,7 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { return; } // Test DataEvolution (split-column write) combined with blob descriptor fields. - // Schema: f0(int32), b0(blob descriptor inline), b1(blob descriptor+external), b2(blob), - // b3(blob) + // Schema: f0(int32), b0/b1(blob descriptor inline), b2/b3(blob). // Commit 1: file A writes (f0, b2, b3) // Commit 2: file B writes (f0, b0, b1) with SetFirstRowId(0) // -> merges with commit 1 @@ -2546,16 +2191,10 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { BlobUtils::ToArrowField("b3", true)}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, - {Options::TARGET_FILE_SIZE, "700"}, - {Options::BUCKET, "-1"}, - {Options::ROW_TRACKING_ENABLED, "true"}, - {Options::DATA_EVOLUTION_ENABLED, "true"}, - {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, - {Options::FILE_SYSTEM, "local"}}; + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, {Options::FILE_SYSTEM, "local"}}; CreateTable(fields, /*partition_keys=*/{}, options); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); @@ -2579,7 +2218,8 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { auto file_b1_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_b1_fields), file_b1_json) .ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto file_b1_desc, ConvertRawBlobToDescriptor(file_b1_array, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto file_b1_desc, + ConvertRawBlobToDescriptor(file_b1_array, {"b0", "b1"})); ASSERT_OK_AND_ASSIGN(auto commit_msgs_a1, WriteArray(table_path, {}, {"f0", "b2", "b3"}, {file_a1_array})); @@ -2601,7 +2241,8 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { auto file_a2_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_a2_fields), file_a2_json) .ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto file_a2_desc, ConvertRawBlobToDescriptor(file_a2_array, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto file_a2_desc, + ConvertRawBlobToDescriptor(file_a2_array, {"b0", "b1"})); std::string file_b2_json = R"([ ["img_3", "vid_3", "raw_3_3"], @@ -2612,7 +2253,8 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { auto file_b2_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(file_b2_fields), file_b2_json) .ValueOrDie()); - ASSERT_OK_AND_ASSIGN(auto file_b2_desc, ConvertRawBlobToDescriptor(file_b2_array, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto file_b2_desc, + ConvertRawBlobToDescriptor(file_b2_array, {"b0", "b1"})); ASSERT_OK_AND_ASSIGN(auto commit_msgs_a2, WriteArray(table_path, {}, {"f0", "b0", "b1", "b3"}, {file_a2_desc})); @@ -2659,9 +2301,8 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) { if (GetParam() == "lance") { return; } - // Similar to TestBlobDescriptorFieldWithoutExternalStorage but writes raw bytes directly - // without converting to descriptor first. The writer should auto-detect that the data - // is NOT a descriptor (no magic header) and handle it accordingly. + // Similar to TestBlobDescriptorField but writes raw bytes directly without converting to + // descriptor first. Descriptor fields reject values without the descriptor magic header. arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), BlobUtils::ToArrowField("b1", true)}; @@ -2852,14 +2493,13 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { } } -TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamExternalStorageBlob) { +TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) { auto file_format = GetParam(); if (GetParam() == "lance") { return; } - // Upstream table has two blob descriptor fields: b0 (field_id=1, inline descriptor) and - // b1 (field_id=2, descriptor + external storage). The downstream view references cells from - // both b0 and b1. + // Upstream table has two blob descriptor fields. The downstream view references cells from + // both b0 (field_id=1) and b1 (field_id=2). const std::string upstream_db_name = "upstream_two_blob"; const std::string upstream_table_name = "upstream_two_blob"; arrow::FieldVector upstream_fields = {arrow::field("f0", arrow::int32()), @@ -2873,8 +2513,6 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamExternalStorageBlob) { {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_DESCRIPTOR_FIELD, "b0,b1"}, - {Options::BLOB_EXTERNAL_STORAGE_FIELD, "b1"}, - {Options::BLOB_EXTERNAL_STORAGE_PATH, blob_dir_->Str()}, {Options::FILE_SYSTEM, "local"}}; ::ArrowSchema upstream_c_schema; @@ -3317,8 +2955,7 @@ TEST_P(BlobTableInteTest, TestReadBlobDescriptorFieldFromJava) { return; } std::string table_path = - GetDataDir() + "/" + file_format + - "/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path"; + GetDataDir() + "/" + file_format + "/blob_desc_field.db/blob_desc_field"; arrow::FieldVector fields = { arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), BlobUtils::ToArrowField("b1", true), BlobUtils::ToArrowField("b2", true), diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README b/test/test_data/orc/blob_desc_field.db/blob_desc_field/README similarity index 71% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/README index 79f90708..7d923daf 100644 --- a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/README @@ -4,16 +4,13 @@ target-file-size: 700 row-tracking.enabled: true data-evolution.enabled: true blob-descriptor-field: b0,b1 -blob-external-storage-field: b1 -blob-external-storage-path:
/external_blob (absolute path at generation time) b0: descriptor field, inline in main file, source .bin files in raw_blob/ -b1: descriptor field, repacked to external storage in external_blob/ +b1: descriptor field whose descriptor points to external_blob/ b2: regular blob, written to .blob files b3: regular blob, written to .blob files -Note: b0 is passed as descriptor via Blob.fromLocal(); b1/b2/b3 are raw bytes. -Paimon auto-converts b1 to descriptor internally. +Note: this fixture keeps historical descriptor URIs pointing at raw_blob/ and external_blob/. Msgs: snapshot-1 diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc b/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-1.orc diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob b/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-2.blob diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob b/test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/bucket-0/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-3.blob diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob b/test/test_data/orc/blob_desc_field.db/blob_desc_field/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/external_blob/data-aeee2df5-4d8d-46e1-823b-437e1a2bc30e-0.blob diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-3978fbf9-7623-4e3e-b6ee-0d55d07377fa-0 diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-0 diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/manifest/manifest-list-85cd267b-28d7-4aab-93b8-cf7e8ac57c07-1 diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-0.bin b/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-0.bin rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-1.bin b/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-1.bin rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-2.bin b/test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-2.bin rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/schema/schema-0 similarity index 88% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/schema/schema-0 index f3243718..d013d274 100644 --- a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 +++ b/test/test_data/orc/blob_desc_field.db/blob_desc_field/schema/schema-0 @@ -28,9 +28,7 @@ "options" : { "bucket" : "-1", "row-tracking.enabled" : "true", - "blob-external-storage-path" : "external_blob", "target-file-size" : "700", - "blob-external-storage-field" : "b1", "data-evolution.enabled" : "true", "file-system" : "local", "manifest.format" : "orc", @@ -38,4 +36,4 @@ "blob-descriptor-field" : "b0,b1" }, "timeMillis" : 1781088844975 -} \ No newline at end of file +} diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/EARLIEST b/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/EARLIEST rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/LATEST b/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/LATEST similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/LATEST rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/LATEST diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/snapshot-1 b/test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 similarity index 100% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/snapshot-1 rename to test/test_data/orc/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 diff --git a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/README similarity index 71% rename from test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/README index 79f90708..7d923daf 100644 --- a/test/test_data/orc/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/README +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/README @@ -4,16 +4,13 @@ target-file-size: 700 row-tracking.enabled: true data-evolution.enabled: true blob-descriptor-field: b0,b1 -blob-external-storage-field: b1 -blob-external-storage-path:
/external_blob (absolute path at generation time) b0: descriptor field, inline in main file, source .bin files in raw_blob/ -b1: descriptor field, repacked to external storage in external_blob/ +b1: descriptor field whose descriptor points to external_blob/ b2: regular blob, written to .blob files b3: regular blob, written to .blob files -Note: b0 is passed as descriptor via Blob.fromLocal(); b1/b2/b3 are raw bytes. -Paimon auto-converts b1 to descriptor internally. +Note: this fixture keeps historical descriptor URIs pointing at raw_blob/ and external_blob/. Msgs: snapshot-1 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-1.parquet diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-2.blob b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-2.blob similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-2.blob rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-2.blob diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/bucket-0/data-e749888c-c975-46af-9a7d-36ca13c32455-3.blob diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/external_blob/data-e749888c-c975-46af-9a7d-36ca13c32455-0.blob diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-de59f444-0069-4836-8dcd-8a3a81158e02-0 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-0 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/manifest/manifest-list-7395f790-699b-4a38-8747-10f23ceba1d6-1 diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-0.bin b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-0.bin rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-0.bin diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-1.bin b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-1.bin rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-1.bin diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-2.bin b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/raw_blob/b0-row-2.bin rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/raw_blob/b0-row-2.bin diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/schema/schema-0 similarity index 88% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/schema/schema-0 index 174973a9..0c314d4f 100644 --- a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/schema/schema-0 +++ b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/schema/schema-0 @@ -28,9 +28,7 @@ "options" : { "bucket" : "-1", "row-tracking.enabled" : "true", - "blob-external-storage-path" : "external_blob", "target-file-size" : "700", - "blob-external-storage-field" : "b1", "data-evolution.enabled" : "true", "file-system" : "local", "manifest.format" : "avro", @@ -38,4 +36,4 @@ "blob-descriptor-field" : "b0,b1" }, "timeMillis" : 1781088620905 -} \ No newline at end of file +} diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/EARLIEST b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/EARLIEST rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/EARLIEST diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/LATEST b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/LATEST similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/LATEST rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/LATEST diff --git a/test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/snapshot-1 b/test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 similarity index 100% rename from test/test_data/parquet/blob_desc_field_with_external_path.db/blob_desc_field_with_external_path/snapshot/snapshot-1 rename to test/test_data/parquet/blob_desc_field.db/blob_desc_field/snapshot/snapshot-1 From 8eb43d5df6555f5f1a837afa6bf390f7b456e3bc Mon Sep 17 00:00:00 2001 From: gripleaf <425797155@qq.com> Date: Mon, 6 Jul 2026 14:40:44 +0800 Subject: [PATCH 085/138] feat(schema): scan context support set table schema --- include/paimon/read_context.h | 2 +- include/paimon/scan_context.h | 18 +++++ .../prefetch_file_batch_reader_impl.cpp | 8 ++- .../reader/prefetch_file_batch_reader_impl.h | 1 + src/paimon/core/operation/scan_context.cpp | 11 ++- .../core/operation/scan_context_test.cpp | 3 + src/paimon/core/table/source/table_scan.cpp | 19 +++-- test/inte/scan_inte_test.cpp | 71 +++++++++++++++++++ 8 files changed, 124 insertions(+), 9 deletions(-) diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h index 4597268c..8fdac7b3 100644 --- a/include/paimon/read_context.h +++ b/include/paimon/read_context.h @@ -108,7 +108,7 @@ class PAIMON_EXPORT ReadContext { uint32_t GetRowToBatchThreadNumber() const { return row_to_batch_thread_number_; } - const std::optional& GetSpecificTableSchema() { + const std::optional& GetSpecificTableSchema() const { return table_schema_; } std::shared_ptr GetMemoryPool() const { diff --git a/include/paimon/scan_context.h b/include/paimon/scan_context.h index 93cdf413..2dea6ac5 100644 --- a/include/paimon/scan_context.h +++ b/include/paimon/scan_context.h @@ -51,6 +51,7 @@ class PAIMON_EXPORT ScanContext { const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, + const std::optional& table_schema, const std::map& options, const std::shared_ptr& cache); @@ -90,6 +91,10 @@ class PAIMON_EXPORT ScanContext { return specific_file_system_; } + const std::optional& GetSpecificTableSchema() const { + return table_schema_; + } + std::shared_ptr GetCache() const { return cache_; } @@ -103,6 +108,7 @@ class PAIMON_EXPORT ScanContext { std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; + std::optional table_schema_; std::map options_; std::shared_ptr cache_; }; @@ -187,6 +193,18 @@ class PAIMON_EXPORT ScanContextBuilder { /// @note If not set, use default file system (configured in `Options::FILE_SYSTEM`) ScanContextBuilder& WithFileSystem(const std::shared_ptr& file_system); + /// Set the table schema as a string to avoid schema loading I/O operations. + /// + /// This optimization allows the scanner to use a pre-loaded schema instead of + /// reading it from the table metadata, which can improve performance especially + /// in scenarios with many small scan operations. + /// + /// @param table_schema String representation of the table schema. + /// @return Reference to this builder for method chaining. + /// @note The user must ensure that the schema string is valid and matches the table. + /// @note If not set, the schema will be loaded from the table path. + ScanContextBuilder& SetTableSchema(const std::string& table_schema); + /// Inject a cache for scan operations. Passing nullptr disables cache. /// @return Reference to this builder for method chaining. ScanContextBuilder& WithCache(const std::shared_ptr& cache); diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp index 5f3e4c72..973ac888 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp @@ -155,6 +155,7 @@ PrefetchFileBatchReaderImpl::~PrefetchFileBatchReaderImpl() { Status PrefetchFileBatchReaderImpl::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { + PAIMON_RETURN_NOT_OK(CleanUp()); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, arrow::ImportSchema(read_schema)); for (const auto& reader : readers_) { @@ -164,11 +165,15 @@ Status PrefetchFileBatchReaderImpl::SetReadSchema( } selection_bitmap_ = selection_bitmap; predicate_ = predicate; - return RefreshReadRanges(); + return RefreshReadRangesAfterCleanUp(); } Status PrefetchFileBatchReaderImpl::RefreshReadRanges() { PAIMON_RETURN_NOT_OK(CleanUp()); + return RefreshReadRangesAfterCleanUp(); +} + +Status PrefetchFileBatchReaderImpl::RefreshReadRangesAfterCleanUp() { bool need_prefetch; PAIMON_ASSIGN_OR_RAISE(auto read_ranges, readers_[0]->GenReadRanges(&need_prefetch)); @@ -281,6 +286,7 @@ Status PrefetchFileBatchReaderImpl::CleanUp() { read_ranges_.clear(); read_ranges_in_group_.clear(); current_batch_global_row_ids_.clear(); + read_ranges_freshed_ = false; clean_prefetch_queue(); for (size_t i = 0; i < readers_pos_.size(); i++) { readers_pos_[i]->store(0); diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h index c54cda49..36673e8b 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.h +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.h @@ -131,6 +131,7 @@ class PrefetchFileBatchReaderImpl : public PrefetchFileBatchReader { static std::vector>> DispatchReadRanges( const std::vector>& read_ranges, size_t reader_count); + Status RefreshReadRangesAfterCleanUp(); Result> EofRange() const; std::optional> GetCurrentReadRange(size_t reader_idx) const; Status EnsureReaderPosition(size_t reader_idx, diff --git a/src/paimon/core/operation/scan_context.cpp b/src/paimon/core/operation/scan_context.cpp index 684439b5..b59fa098 100644 --- a/src/paimon/core/operation/scan_context.cpp +++ b/src/paimon/core/operation/scan_context.cpp @@ -35,6 +35,7 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, + const std::optional& table_schema, const std::map& options, const std::shared_ptr& cache) : path_(path), @@ -45,6 +46,7 @@ ScanContext::ScanContext(const std::string& path, bool is_streaming_mode, memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), + table_schema_(table_schema), options_(options), cache_(cache) {} @@ -64,6 +66,7 @@ class ScanContextBuilder::Impl { memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); specific_file_system_.reset(); + table_schema_ = std::nullopt; options_.clear(); cache_.reset(); } @@ -79,6 +82,7 @@ class ScanContextBuilder::Impl { std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); std::shared_ptr specific_file_system_; + std::optional table_schema_; std::map options_; std::shared_ptr cache_; }; @@ -149,6 +153,11 @@ ScanContextBuilder& ScanContextBuilder::WithFileSystem( return *this; } +ScanContextBuilder& ScanContextBuilder::SetTableSchema(const std::string& table_schema) { + impl_->table_schema_ = table_schema; + return *this; +} + ScanContextBuilder& ScanContextBuilder::WithCache(const std::shared_ptr& cache) { impl_->cache_ = cache; return *this; @@ -164,7 +173,7 @@ Result> ScanContextBuilder::Finish() { std::make_shared(impl_->predicates_, impl_->partition_filters_, impl_->bucket_filter_), impl_->global_index_result_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_, impl_->cache_); + impl_->specific_file_system_, impl_->table_schema_, impl_->options_, impl_->cache_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/scan_context_test.cpp b/src/paimon/core/operation/scan_context_test.cpp index 1f74463d..6c86c049 100644 --- a/src/paimon/core/operation/scan_context_test.cpp +++ b/src/paimon/core/operation/scan_context_test.cpp @@ -68,6 +68,7 @@ TEST(ScanContextTest, TestSetContent) { builder.WithExecutor(executor); auto fs = std::make_shared(); builder.WithFileSystem(fs); + builder.SetTableSchema("table-schema-json"); auto manifest_cache = std::make_shared(1024); builder.WithCache(manifest_cache); ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); @@ -81,6 +82,8 @@ TEST(ScanContextTest, TestSetContent) { ASSERT_EQ("{1,2,4,5}", ctx->GetGlobalIndexResult()->ToString()); ASSERT_EQ(memory_pool, ctx->GetMemoryPool()); ASSERT_EQ(executor, ctx->GetExecutor()); + ASSERT_TRUE(ctx->GetSpecificTableSchema().has_value()); + ASSERT_EQ("table-schema-json", ctx->GetSpecificTableSchema().value()); std::map expected_options = {{"key", "value"}}; ASSERT_EQ(expected_options, ctx->GetOptions()); ASSERT_EQ(fs, ctx->GetSpecificFileSystem()); diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index ba45c116..197c8d4e 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -197,13 +197,20 @@ Result> NewDataTableScan(const std::shared_ptrGetOptions(), context->GetSpecificFileSystem(), {})); std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); - SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); - PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, - schema_manager.Latest()); - if (latest_table_schema == std::nullopt) { - return Status::Invalid("not found latest schema"); + std::shared_ptr table_schema; + const auto& specific_table_schema = context->GetSpecificTableSchema(); + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && specific_table_schema) { + PAIMON_ASSIGN_OR_RAISE(table_schema, + TableSchema::CreateFromJson(specific_table_schema.value())); + } else { + SchemaManager schema_manager(tmp_options.GetFileSystem(), context->GetPath(), branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_table_schema, + schema_manager.Latest()); + if (latest_table_schema == std::nullopt) { + return Status::Invalid("not found latest schema"); + } + table_schema = latest_table_schema.value(); } - const auto& table_schema = latest_table_schema.value(); // merge options auto options = table_schema->Options(); for (const auto& [key, value] : context->GetOptions()) { diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index a3b267df..3a0531a9 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -399,6 +400,76 @@ TEST_P(ScanInteTest, TestScanAppendWithSnapshot3) { CheckResult(expected_data_splits, result_data_splits); } +TEST_P(ScanInteTest, TestScanAppendWithSpecificTableSchema) { + std::string data_dir = paimon::test::GetDataDir(); + if (!std::filesystem::exists(data_dir + "orc/append_09.db/append_09/schema/schema-0")) { + data_dir = "../" + data_dir; + } + std::string table_path = data_dir + "orc/append_09.db/append_09"; + + auto check_result = [&](const std::optional& specific_table_schema) { + ScanContextBuilder context_builder(table_path); + context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); + if (specific_table_schema) { + context_builder.SetTableSchema(specific_table_schema.value()); + } + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(context_builder)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + + ASSERT_EQ(result_plan->SnapshotId().value(), 3); + + auto result_data_splits = CollectDataSplits(result_plan); + DataSplitImpl::Builder builder1(BinaryRowGenerator::GenerateRow({10}, pool_.get()), + /*bucket=*/0, /*bucket_path=*/ + data_dir + "orc/append_09.db/append_09/f1=10/bucket-0", + {meta_snapshot1_partition10_bucket0_}); + auto expected_data_split1 = + std::dynamic_pointer_cast(builder1.WithTotalBuckets(2) + .WithSnapshot(3) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + DataSplitImpl::Builder builder2( + BinaryRowGenerator::GenerateRow({10}, pool_.get()), /*bucket=*/1, /*bucket_path=*/ + data_dir + "orc/append_09.db/append_09/f1=10/bucket-1", + {meta_snapshot1_partition10_bucket1_, meta_snapshot2_partition10_bucket1_, + meta_snapshot3_partition10_bucket1_}); + auto expected_data_split2 = + std::dynamic_pointer_cast(builder2.WithTotalBuckets(2) + .WithSnapshot(3) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + DataSplitImpl::Builder builder3( + BinaryRowGenerator::GenerateRow({20}, pool_.get()), /*bucket=*/0, /*bucket_path=*/ + data_dir + "orc/append_09.db/append_09/f1=20/bucket-0", + {meta_snapshot1_partition20_bucket0_, meta_snapshot2_partition20_bucket0_}); + auto expected_data_split3 = + std::dynamic_pointer_cast(builder3.WithTotalBuckets(2) + .WithSnapshot(3) + .IsStreaming(false) + .RawConvertible(true) + .Build() + .value()); + + std::vector> expected_data_splits = { + expected_data_split1, expected_data_split2, expected_data_split3}; + CheckResult(expected_data_splits, result_data_splits); + }; + + check_result(std::nullopt); + + auto fs = std::make_shared(); + std::string schema_str; + ASSERT_OK(fs->ReadFile(table_path + "/schema/schema-0", &schema_str)); + check_result(std::optional(schema_str)); +} + TEST_P(ScanInteTest, TestScanInvalidSnapshot) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); From dd6e22770e935f8f1c45e4f47bc808708c8d3029 Mon Sep 17 00:00:00 2001 From: gripleaf <425797155@qq.com> Date: Mon, 6 Jul 2026 16:59:33 +0800 Subject: [PATCH 086/138] feat(manifest): support snapshot live manifest cache --- docs/source/user_guide.rst | 1 + .../user_guide/manifest_entry_cache.rst | 91 +++++ include/paimon/cache/cache.h | 4 + include/paimon/defs.h | 5 + src/paimon/CMakeLists.txt | 1 + src/paimon/common/defs.cpp | 2 + src/paimon/common/io/cache/cache_key.cpp | 49 +++ src/paimon/common/io/cache/lru_cache_test.cpp | 17 + src/paimon/core/core_options.cpp | 13 + src/paimon/core/core_options.h | 1 + src/paimon/core/core_options_test.cpp | 6 + .../snapshot_live_manifest_entries.cpp | 139 ++++++++ .../manifest/snapshot_live_manifest_entries.h | 66 ++++ .../append_only_file_store_scan_test.cpp | 70 ++++ src/paimon/core/operation/file_store_scan.cpp | 207 +++++++++-- src/paimon/core/operation/file_store_scan.h | 37 +- .../core/operation/file_store_scan_test.cpp | 88 +++++ src/paimon/core/table/source/table_scan.cpp | 35 +- .../testing/utils/counting_cache_test_utils.h | 20 ++ test/inte/data_evolution_table_test.cpp | 87 +++-- test/inte/scan_and_read_inte_test.cpp | 335 ++++++++++-------- test/inte/scan_inte_test.cpp | 112 +++++- 22 files changed, 1151 insertions(+), 235 deletions(-) create mode 100644 docs/source/user_guide/manifest_entry_cache.rst create mode 100644 src/paimon/core/manifest/snapshot_live_manifest_entries.cpp create mode 100644 src/paimon/core/manifest/snapshot_live_manifest_entries.h diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index 5d503beb..dc444aa1 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -28,6 +28,7 @@ User Guide user_guide/snapshot user_guide/manifest user_guide/manifest_cache + user_guide/manifest_entry_cache user_guide/parquet_metadata_cache user_guide/data_types user_guide/primary_key_table diff --git a/docs/source/user_guide/manifest_entry_cache.rst b/docs/source/user_guide/manifest_entry_cache.rst new file mode 100644 index 00000000..97b33cde --- /dev/null +++ b/docs/source/user_guide/manifest_entry_cache.rst @@ -0,0 +1,91 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Manifest Entry Cache +==================== + +Overview +-------- + +Large tables may contain many manifest entries, while a scan may only need a +small subset after bucket, partition, and statistics pruning. The snapshot live +manifest entry cache reduces repeated manifest decoding cost for successive full +scans that target the same bucket. + +The cache stores decoded and merged live manifest entries by table path, branch, +and bucket for ``ScanMode::ALL``. Each cache value can retain several snapshot +results for that bucket. Exact snapshot hits are served from the cache; cache +misses rebuild the target snapshot bucket from the target snapshot's data +manifests and store the rebuilt live entries. + +Request-specific filters are not stored in the cache. Partition, level, and +predicate filters are still evaluated for each scan, so cached entries can be +reused safely across different scan predicates for the same bucket. + +Configuration +------------- + +Manifest entry caching reuses the cache instance provided by +``ScanContextBuilder::WithCache()`` and stores bucket-scoped snapshot entries +under +``CacheKind::SNAPSHOT_LIVE_MANIFEST``: + +.. code-block:: cpp + + auto cache = std::make_shared(128 * 1024 * 1024); + ScanContextBuilder context_builder(table_path); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan_context, + context_builder + .WithCache(cache) + .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") + .Finish()); + +Cache entries are scoped by table path, branch, and bucket, so they can be +reused across newly created ``TableScan`` and ``FileStoreScan`` instances as +long as they share the same cache object and scan the same bucket. + +``Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS`` controls how many snapshot +results are retained in each table/branch/bucket cache value. Older snapshots in +the same bucket are evicted first. The default value is ``0``, which disables +the cache path. Set it to a positive value to enable the cache when +``ScanContextBuilder::WithCache()`` is also configured. Physical cache eviction +is still controlled by the configured ``Cache`` implementation, for example the +capacity of ``LruCache``. + +If no cache is provided through ``ScanContextBuilder::WithCache()``, this +optimization is skipped. The snapshot manifest entry cache shares the same +``Cache`` interface with raw manifest and data-file footer caches, but it uses a +dedicated ``CacheKind`` and a table/branch/bucket key instead of file byte +ranges. + +Limitations +----------- + +The cache is currently used only for ``ScanMode::ALL`` scans that can determine +a single target bucket. It is skipped for scans without a bucket filter because +reading or deserializing all buckets would be too expensive for selective +queries. It is also skipped for row-range scans because row-range pruning is +applied at manifest-meta level. + +Metrics +------- + +The scan metrics expose existing counters for the last scan: + +- ``lastScannedManifests``: how many manifest files were loaded during this + scan before manifest entry decoding. diff --git a/include/paimon/cache/cache.h b/include/paimon/cache/cache.h index 1edcb472..5bff2a12 100644 --- a/include/paimon/cache/cache.h +++ b/include/paimon/cache/cache.h @@ -35,6 +35,7 @@ enum class CacheKind { DEFAULT, MANIFEST, DATA_FILE_FOOTER, + SNAPSHOT_LIVE_MANIFEST, }; class PAIMON_EXPORT CacheKey { @@ -43,6 +44,9 @@ class PAIMON_EXPORT CacheKey { int32_t length, bool is_index); static std::shared_ptr ForKind(const std::string& file_path, int64_t position, int32_t length, CacheKind kind); + static std::shared_ptr ForSnapshotLiveManifestEntries(const std::string& table_path, + const std::string& branch, + int32_t bucket); public: virtual ~CacheKey() = default; diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 61cba363..5d91b959 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -170,6 +170,11 @@ struct PAIMON_EXPORT Options { /// "latest-full", "latest", "from-snapshot", "from-snapshot-full". Default value is "default". static const char SCAN_MODE[]; + /// "scan.manifest-entry-cache.max-snapshots" - Maximum number of snapshot live manifest entry + /// results retained per table, branch, and bucket. Setting it to 0 disables manifest entry + /// cache. Default value is 0. + static const char SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS[]; + /// "read.batch-size" - Read batch size for any file format if it supports. /// The default value is 1024. static const char READ_BATCH_SIZE[]; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 8f7b4d8a..4a1e8bc4 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -306,6 +306,7 @@ set(PAIMON_CORE_SRCS core/operation/raw_file_split_read.cpp core/operation/read_context.cpp core/operation/scan_context.cpp + core/manifest/snapshot_live_manifest_entries.cpp core/operation/write_context.cpp core/operation/write_restore.cpp core/postpone/postpone_bucket_writer.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index ab6b6c97..4e68916f 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -50,6 +50,8 @@ const char Options::SOURCE_SPLIT_TARGET_SIZE[] = "source.split.target-size"; const char Options::SOURCE_SPLIT_OPEN_FILE_COST[] = "source.split.open-file-cost"; const char Options::SCAN_SNAPSHOT_ID[] = "scan.snapshot-id"; const char Options::SCAN_MODE[] = "scan.mode"; +const char Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS[] = + "scan.manifest-entry-cache.max-snapshots"; const char Options::READ_BATCH_SIZE[] = "read.batch-size"; const char Options::WRITE_BATCH_SIZE[] = "write.batch-size"; const char Options::WRITE_BUFFER_SIZE[] = "write-buffer-size"; diff --git a/src/paimon/common/io/cache/cache_key.cpp b/src/paimon/common/io/cache/cache_key.cpp index 4529e87d..c84c8f90 100644 --- a/src/paimon/common/io/cache/cache_key.cpp +++ b/src/paimon/common/io/cache/cache_key.cpp @@ -19,6 +19,49 @@ #include "paimon/common/io/cache/cache_key.h" namespace paimon { +namespace { + +class SnapshotLiveManifestEntriesCacheKey : public CacheKey { + public: + SnapshotLiveManifestEntriesCacheKey(const std::string& table_path, const std::string& branch, + int32_t bucket) + : CacheKey(CacheKind::SNAPSHOT_LIVE_MANIFEST), + table_path_(table_path), + branch_(branch), + bucket_(bucket) {} + + bool IsIndex() const override { + return false; + } + + bool Equals(const CacheKey& other) const override { + const auto* rhs = dynamic_cast(&other); + if (!rhs) { + return false; + } + return table_path_ == rhs->table_path_ && branch_ == rhs->branch_ && + bucket_ == rhs->bucket_ && GetKind() == rhs->GetKind(); + } + + size_t HashCode() const override { + size_t seed = 0; + seed ^= std::hash{}(table_path_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); + seed ^= std::hash{}(branch_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); + seed ^= std::hash{}(bucket_) + HASH_CONSTANT + (seed << 6) + (seed >> 2); + seed ^= std::hash{}(static_cast(GetKind())) + HASH_CONSTANT + + (seed << 6) + (seed >> 2); + return seed; + } + + private: + static constexpr uint64_t HASH_CONSTANT = 0x9e3779b97f4a7c15ULL; + + const std::string table_path_; + const std::string branch_; + const int32_t bucket_; +}; + +} // namespace std::shared_ptr CacheKey::ForPosition(const std::string& file_path, int64_t position, int32_t length, bool is_index) { @@ -33,6 +76,12 @@ std::shared_ptr CacheKey::ForKind(const std::string& file_path, int64_ return key; } +std::shared_ptr CacheKey::ForSnapshotLiveManifestEntries(const std::string& table_path, + const std::string& branch, + int32_t bucket) { + return std::make_shared(table_path, branch, bucket); +} + bool PositionCacheKey::IsIndex() const { return is_index_; } diff --git a/src/paimon/common/io/cache/lru_cache_test.cpp b/src/paimon/common/io/cache/lru_cache_test.cpp index ff3b72c0..1d644c70 100644 --- a/src/paimon/common/io/cache/lru_cache_test.cpp +++ b/src/paimon/common/io/cache/lru_cache_test.cpp @@ -383,6 +383,23 @@ TEST_F(LruCacheTest, TestForKindSetsKeyKind) { ASSERT_EQ(CacheKind::MANIFEST, put_key->GetKind()); } +TEST_F(LruCacheTest, TestForSnapshotLiveManifestEntries) { + auto main_key = CacheKey::ForSnapshotLiveManifestEntries("table_path", "main", 0); + auto same_key = CacheKey::ForSnapshotLiveManifestEntries("table_path", "main", 0); + auto branch_key = CacheKey::ForSnapshotLiveManifestEntries("table_path", "dev", 0); + auto table_key = CacheKey::ForSnapshotLiveManifestEntries("other_table_path", "main", 0); + auto bucket_key = CacheKey::ForSnapshotLiveManifestEntries("table_path", "main", 1); + auto hash_in_path_key = CacheKey::ForSnapshotLiveManifestEntries("table#path", "main", 0); + auto hash_in_branch_key = CacheKey::ForSnapshotLiveManifestEntries("table", "path#main", 0); + + ASSERT_EQ(CacheKind::SNAPSHOT_LIVE_MANIFEST, main_key->GetKind()); + ASSERT_TRUE(CacheKeyEqual()(main_key, same_key)); + ASSERT_FALSE(CacheKeyEqual()(main_key, branch_key)); + ASSERT_FALSE(CacheKeyEqual()(main_key, table_key)); + ASSERT_FALSE(CacheKeyEqual()(main_key, bucket_key)); + ASSERT_FALSE(CacheKeyEqual()(hash_in_path_key, hash_in_branch_key)); +} + /// Verifies that multiple evictions happen when a single large entry is inserted. TEST_F(LruCacheTest, TestMultipleEvictions) { LruCache cache(300); diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index b1838ed2..dd21c147 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -403,6 +403,7 @@ struct CoreOptions::Impl { int32_t bucket = -1; int32_t manifest_merge_min_count = 30; + int32_t scan_manifest_entry_cache_max_snapshots = 0; int32_t read_batch_size = 1024; int32_t write_batch_size = 1024; int32_t local_sort_max_num_file_handles = 128; @@ -710,6 +711,13 @@ struct CoreOptions::Impl { } // Parse scan.mode - scanning behavior of the source, default "default" PAIMON_RETURN_NOT_OK(parser.ParseStartupMode(&startup_mode)); + // Parse scan.manifest-entry-cache.max-snapshots - cached snapshots per bucket. + PAIMON_RETURN_NOT_OK(parser.Parse(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, + &scan_manifest_entry_cache_max_snapshots)); + if (scan_manifest_entry_cache_max_snapshots < 0) { + return Status::Invalid(fmt::format("{} must be non-negative", + Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS)); + } // Parse scan.fallback-branch - fallback branch when partition not found PAIMON_RETURN_NOT_OK(parser.Parse(Options::SCAN_FALLBACK_BRANCH, &scan_fallback_branch)); // Parse branch - branch name, default "main" @@ -961,6 +969,11 @@ std::optional CoreOptions::GetScanSnapshotId() const { std::optional CoreOptions::GetScanTimestampMillis() const { return impl_->scan_timestamp_millis; } + +int32_t CoreOptions::GetScanManifestEntryCacheMaxSnapshots() const { + return impl_->scan_manifest_entry_cache_max_snapshots; +} + int64_t CoreOptions::GetManifestTargetFileSize() const { return impl_->manifest_target_file_size; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index a7a7c473..e6b08312 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -81,6 +81,7 @@ class PAIMON_EXPORT CoreOptions { int64_t GetSourceSplitOpenFileCost() const; std::optional GetScanSnapshotId() const; std::optional GetScanTimestampMillis() const; + int32_t GetScanManifestEntryCacheMaxSnapshots() const; int64_t GetManifestTargetFileSize() const; std::shared_ptr GetCache() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index a4f99430..f5a977ed 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -57,6 +57,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(8 * 1024 * 1024L, core_options.GetManifestTargetFileSize()); ASSERT_EQ(16 * 1024 * 1024L, core_options.GetManifestFullCompactionThresholdSize()); ASSERT_EQ(30, core_options.GetManifestMergeMinCount()); + ASSERT_EQ(0, core_options.GetScanManifestEntryCacheMaxSnapshots()); ASSERT_EQ(nullptr, core_options.GetCache()); ASSERT_EQ(128 * 1024 * 1024L, core_options.GetSourceSplitTargetSize()); ASSERT_EQ(4 * 1024 * 1024L, core_options.GetSourceSplitOpenFileCost()); @@ -186,6 +187,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::COMMIT_MAX_RETRIES, "20"}, {Options::SCAN_SNAPSHOT_ID, "5"}, {Options::SCAN_MODE, "from-snapshot-full"}, + {Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "7"}, {Options::SNAPSHOT_NUM_RETAINED_MIN, "15"}, {Options::SNAPSHOT_NUM_RETAINED_MAX, "30"}, {Options::SNAPSHOT_EXPIRE_LIMIT, "20"}, @@ -306,6 +308,7 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(120 * 1000, core_options.GetCommitTimeout()); ASSERT_EQ(20, core_options.GetCommitMaxRetries()); ASSERT_EQ(5, core_options.GetScanSnapshotId().value_or(-1)); + ASSERT_EQ(7, core_options.GetScanManifestEntryCacheMaxSnapshots()); ExpireConfig expire_config = core_options.GetExpireConfig(); ASSERT_EQ(15, expire_config.GetSnapshotRetainMin()); ASSERT_EQ(30, expire_config.GetSnapshotRetainMax()); @@ -431,6 +434,9 @@ TEST(CoreOptionsTest, TestInvalidCase) { "invalid lookup mode: invalid"); ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::LOOKUP_COMPACT_MAX_INTERVAL, "invalid"}}), "Invalid Config [lookup-compact.max-interval: invalid]"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "-1"}}), + "scan.manifest-entry-cache.max-snapshots must be non-negative"); ASSERT_NOK_WITH_MSG( CoreOptions::FromMap({{Options::LOOKUP_CACHE_HIGH_PRIO_POOL_RATIO, "1.1"}}), "The high priority pool ratio should in the range [0, 1), while input is 1.1"); diff --git a/src/paimon/core/manifest/snapshot_live_manifest_entries.cpp b/src/paimon/core/manifest/snapshot_live_manifest_entries.cpp new file mode 100644 index 00000000..8982209a --- /dev/null +++ b/src/paimon/core/manifest/snapshot_live_manifest_entries.cpp @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/manifest/snapshot_live_manifest_entries.h" + +#include +#include +#include + +#include "paimon/common/io/memory_segment_output_stream.h" +#include "paimon/core/manifest/manifest_entry_serializer.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/io/data_input_stream.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/memory/memory_segment.h" + +namespace paimon { +namespace { + +constexpr int32_t kMagic = 0x534d4543; // SMEC + +size_t NormalizeMaxSnapshots(int32_t max_snapshots) { + return static_cast(std::max(0, max_snapshots)); +} + +std::shared_ptr ToBytes(const MemorySegmentOutputStream& out, + const std::shared_ptr& pool) { + auto bytes = Bytes::AllocateBytes(static_cast(out.CurrentSize()), pool.get()); + int64_t offset = 0; + for (const auto& segment : out.Segments()) { + int64_t copy_size = + std::min(segment.Size(), static_cast(bytes->size()) - offset); + if (copy_size <= 0) { + break; + } + std::memcpy(bytes->data() + offset, segment.Data(), static_cast(copy_size)); + offset += copy_size; + } + return bytes; +} + +} // namespace + +SnapshotLiveManifestEntries::SnapshotLiveManifestEntries(int32_t max_snapshots) + : max_snapshots_(max_snapshots) {} + +std::optional SnapshotLiveManifestEntries::LatestBeforeOrEqual( + int64_t snapshot_id) const { + auto iter = entries_by_snapshot_.upper_bound(snapshot_id); + if (iter == entries_by_snapshot_.begin()) { + return std::optional(); + } + --iter; + return Entry{iter->first, iter->second}; +} + +void SnapshotLiveManifestEntries::Put(int64_t snapshot_id, std::vector&& entries) { + if (NormalizeMaxSnapshots(max_snapshots_) == 0) { + return; + } + entries_by_snapshot_[snapshot_id] = + std::make_shared>(std::move(entries)); + EvictIfNeeded(); +} + +size_t SnapshotLiveManifestEntries::Size() const { + return entries_by_snapshot_.size(); +} + +Result> SnapshotLiveManifestEntries::Serialize( + const std::shared_ptr& pool) const { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); + out.WriteValue(kMagic); + out.WriteValue(static_cast(entries_by_snapshot_.size())); + + ManifestEntrySerializer serializer(pool); + for (const auto& [snapshot_id, entries] : entries_by_snapshot_) { + out.WriteValue(snapshot_id); + PAIMON_RETURN_NOT_OK(serializer.SerializeList(*entries, &out)); + } + return ToBytes(out, pool); +} + +Result SnapshotLiveManifestEntries::Deserialize( + const MemorySegment& segment, int32_t max_snapshots, const std::shared_ptr& pool) { + SnapshotLiveManifestEntries snapshot_live_manifest_entries(max_snapshots); + if (segment.Data() == nullptr || segment.Size() == 0) { + return snapshot_live_manifest_entries; + } + + auto bytes = segment.GetOrCreateHeapMemory(pool.get()); + auto input_stream = std::make_shared(bytes->data(), bytes->size()); + DataInputStream in(input_stream); + + PAIMON_ASSIGN_OR_RAISE(int32_t magic, in.ReadValue()); + if (magic != kMagic) { + return Status::Invalid("invalid snapshot live manifest entries magic"); + } + PAIMON_ASSIGN_OR_RAISE(int32_t snapshot_count, in.ReadValue()); + if (snapshot_count < 0) { + return Status::Invalid("snapshot live manifest entries snapshot count is negative"); + } + + ManifestEntrySerializer serializer(pool); + for (int32_t i = 0; i < snapshot_count; i++) { + PAIMON_ASSIGN_OR_RAISE(int64_t snapshot_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(std::vector entries, serializer.DeserializeList(&in)); + snapshot_live_manifest_entries.entries_by_snapshot_[snapshot_id] = + std::make_shared>(std::move(entries)); + } + snapshot_live_manifest_entries.EvictIfNeeded(); + return snapshot_live_manifest_entries; +} + +void SnapshotLiveManifestEntries::EvictIfNeeded() { + size_t max_snapshots = NormalizeMaxSnapshots(max_snapshots_); + while (entries_by_snapshot_.size() > max_snapshots) { + entries_by_snapshot_.erase(entries_by_snapshot_.begin()); + } +} + +} // namespace paimon diff --git a/src/paimon/core/manifest/snapshot_live_manifest_entries.h b/src/paimon/core/manifest/snapshot_live_manifest_entries.h new file mode 100644 index 00000000..acf1d8e7 --- /dev/null +++ b/src/paimon/core/manifest/snapshot_live_manifest_entries.h @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/result.h" + +namespace paimon { +class Bytes; +class MemoryPool; +class MemorySegment; + +/// Live manifest entries retained for one bucket across multiple snapshots. +/// +/// This value object owns merged live manifest entries by snapshot id. It does not own or access a +/// cache; callers are responsible for storing the serialized bytes in the cache layer. +class SnapshotLiveManifestEntries { + public: + struct Entry { + int64_t snapshot_id; + std::shared_ptr> entries; + }; + + explicit SnapshotLiveManifestEntries(int32_t max_snapshots); + + std::optional LatestBeforeOrEqual(int64_t snapshot_id) const; + void Put(int64_t snapshot_id, std::vector&& entries); + size_t Size() const; + + Result> Serialize(const std::shared_ptr& pool) const; + static Result Deserialize(const MemorySegment& segment, + int32_t max_snapshots, + const std::shared_ptr& pool); + + private: + void EvictIfNeeded(); + + std::map>> entries_by_snapshot_; + int32_t max_snapshots_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/append_only_file_store_scan_test.cpp b/src/paimon/core/operation/append_only_file_store_scan_test.cpp index d644bec0..1fe75338 100644 --- a/src/paimon/core/operation/append_only_file_store_scan_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_scan_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/io/cache/lru_cache.h" #include "paimon/core/manifest/partition_entry.h" #include "paimon/core/operation/metrics/scan_metrics.h" #include "paimon/core/schema/schema_manager.h" @@ -177,4 +179,72 @@ TEST(AppendOnlyFileStoreScanTest, TestScanDurationMetric) { ASSERT_LE(stats.p50, stats.p99); ASSERT_LE(stats.p99, stats.max); } + +namespace { + +std::shared_ptr BuildScan(const std::string& table_path, + const std::shared_ptr& cache, + const std::optional& bucket = std::nullopt) { + ScanContextBuilder context_builder(table_path); + context_builder.AddOption(Options::FILE_FORMAT, "orc") + .AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "8") + .WithCache(cache); + if (bucket) { + context_builder.SetBucketFilter(bucket.value()); + } + EXPECT_OK_AND_ASSIGN(auto scan_context, context_builder.Finish()); + EXPECT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + auto typed_table_scan = dynamic_cast(table_scan.get()); + EXPECT_TRUE(typed_table_scan); + return typed_table_scan->snapshot_reader_->scan_; +} + +} // namespace + +TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCachePath) { + TimezoneGuard guard("Asia/Shanghai"); + std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + auto cache = std::make_shared(/*max_weight=*/16 * 1024 * 1024); + + // First scan on snapshot 5: cache miss, entries rebuilt from all manifests. + auto scan_first = BuildScan(table_path, cache, /*bucket=*/0); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot_5, + scan_first->GetSnapshotManager()->LoadSnapshot(/*snapshot_id=*/5)); + scan_first->WithSnapshot(snapshot_5); + ASSERT_OK_AND_ASSIGN(auto plan_first, scan_first->CreatePlan()); + size_t first_size = plan_first->Files().size(); + + // Second scan on the same snapshot should read the same bucket live entries from cache. + auto scan_second = BuildScan(table_path, cache, /*bucket=*/0); + scan_second->WithSnapshot(snapshot_5); + ASSERT_OK_AND_ASSIGN(auto plan_second, scan_second->CreatePlan()); + ASSERT_EQ(first_size, plan_second->Files().size()); +} + +TEST(AppendOnlyFileStoreScanTest, TestSnapshotLiveManifestCacheRebuildOnMiss) { + TimezoneGuard guard("Asia/Shanghai"); + std::string table_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; + auto cache = std::make_shared(/*max_weight=*/16 * 1024 * 1024); + + // Seed the cache with an earlier snapshot. + auto scan_base = BuildScan(table_path, cache, /*bucket=*/0); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot_3, + scan_base->GetSnapshotManager()->LoadSnapshot(/*snapshot_id=*/3)); + scan_base->WithSnapshot(snapshot_3); + ASSERT_OK_AND_ASSIGN(auto plan_base, scan_base->CreatePlan()); + (void)plan_base; + + // Advance to a newer snapshot: cache miss rebuilds the target snapshot bucket. + auto scan_next = BuildScan(table_path, cache, /*bucket=*/0); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot_5, + scan_next->GetSnapshotManager()->LoadSnapshot(/*snapshot_id=*/5)); + scan_next->WithSnapshot(snapshot_5); + ASSERT_OK_AND_ASSIGN(auto plan_next, scan_next->CreatePlan()); + + auto scan_expected = BuildScan(table_path, /*cache=*/nullptr, /*bucket=*/0); + scan_expected->WithSnapshot(snapshot_5); + ASSERT_OK_AND_ASSIGN(auto plan_expected, scan_expected->CreatePlan()); + ASSERT_EQ(plan_expected->Files().size(), plan_next->Files().size()); +} } // namespace paimon::test diff --git a/src/paimon/core/operation/file_store_scan.cpp b/src/paimon/core/operation/file_store_scan.cpp index e49ef28f..681e772e 100644 --- a/src/paimon/core/operation/file_store_scan.cpp +++ b/src/paimon/core/operation/file_store_scan.cpp @@ -18,6 +18,7 @@ #include "paimon/core/operation/file_store_scan.h" +#include #include #include #include @@ -28,6 +29,7 @@ #include "arrow/type.h" #include "fmt/format.h" +#include "paimon/cache/cache.h" #include "paimon/common/data/binary_array.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/executor/future.h" @@ -40,13 +42,17 @@ #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/manifest/snapshot_live_manifest_entries.h" #include "paimon/core/operation/metrics/scan_metrics.h" #include "paimon/core/partition/partition_info.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/stats/simple_stats_evolution.h" +#include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/duration.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/snapshot_manager.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" #include "paimon/predicate/literal.h" #include "paimon/predicate/predicate.h" #include "paimon/predicate/predicate_builder.h" @@ -113,7 +119,8 @@ Result> FileStoreScan::ReadPartitionEntries() const PAIMON_RETURN_NOT_OK( ReadManifests(&snapshot, &all_manifest_file_metas, &filtered_manifest_file_metas)); std::vector manifest_entries; - PAIMON_RETURN_NOT_OK(ReadFileEntries(filtered_manifest_file_metas, &manifest_entries)); + PAIMON_RETURN_NOT_OK(ReadFileEntries(filtered_manifest_file_metas, &manifest_entries, + /*apply_scan_filter=*/true)); std::unordered_map partitions; PAIMON_RETURN_NOT_OK(PartitionEntry::Merge(manifest_entries, &partitions)); @@ -136,7 +143,26 @@ Result> FileStoreScan::CreatePlan() cons ReadManifests(&snapshot, &all_manifest_file_metas, &filtered_manifest_file_metas)); std::vector manifest_entries; - PAIMON_RETURN_NOT_OK(ReadManifestEntries(filtered_manifest_file_metas, &manifest_entries)); + const bool use_snapshot_live_manifest_cache = + snapshot.has_value() && scan_mode_ == ScanMode::ALL && + core_options_.GetScanManifestEntryCacheMaxSnapshots() > 0 && + core_options_.GetCache() != nullptr && !table_path_.empty() && + !row_range_index_.has_value() && bucket_filter_.has_value(); + if (use_snapshot_live_manifest_cache) { + PAIMON_RETURN_NOT_OK(ReadManifestEntriesWithCache( + snapshot.value(), all_manifest_file_metas, bucket_filter_.value(), &manifest_entries)); + std::vector filtered_entries; + filtered_entries.reserve(manifest_entries.size()); + for (auto& entry : manifest_entries) { + PAIMON_ASSIGN_OR_RAISE(bool keep, FilterManifestEntry(entry)); + if (keep) { + filtered_entries.emplace_back(std::move(entry)); + } + } + manifest_entries = std::move(filtered_entries); + } else { + PAIMON_RETURN_NOT_OK(ReadManifestEntries(filtered_manifest_file_metas, &manifest_entries)); + } PAIMON_ASSIGN_OR_RAISE(manifest_entries, PostFilterManifestEntries(std::move(manifest_entries))); @@ -171,7 +197,8 @@ Result> FileStoreScan::CreatePlan() cons metrics_->ObserveHistogram(ScanMetrics::SCAN_DURATION, static_cast(scan_duration_ms)); metrics_->SetCounter(ScanMetrics::LAST_SCANNED_SNAPSHOT_ID, snapshot.has_value() ? snapshot.value().Id() : int64_t{0}); - metrics_->SetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS, filtered_manifest_file_metas.size()); + metrics_->SetCounter(ScanMetrics::LAST_SCANNED_MANIFESTS, + static_cast(filtered_manifest_file_metas.size())); metrics_->SetCounter( ScanMetrics::LAST_SCAN_SKIPPED_TABLE_FILES, std::max(int64_t{0}, all_data_files - static_cast(manifest_entries.size()))); @@ -220,12 +247,19 @@ Status FileStoreScan::ReadManifestsWithSnapshot(const Snapshot& snapshot, } Status FileStoreScan::ReadFileEntries(const std::vector& manifest_metas, - std::vector* manifest_entries) const { + std::vector* manifest_entries, + bool apply_scan_filter) const { std::vector>>> futures; for (const auto& meta : manifest_metas) { - auto read_meta_task = [this, meta]() -> Result> { + auto read_meta_task = [this, meta, + apply_scan_filter]() -> Result> { std::vector tmp_entries; - PAIMON_RETURN_NOT_OK(ReadManifestFileMeta(meta, &tmp_entries)); + if (apply_scan_filter) { + PAIMON_RETURN_NOT_OK(ReadManifestFileMeta(meta, &tmp_entries)); + } else { + PAIMON_RETURN_NOT_OK( + manifest_file_->Read(meta.FileName(), /*filter=*/nullptr, &tmp_entries)); + } return tmp_entries; }; futures.push_back(Via(executor_.get(), read_meta_task)); @@ -253,29 +287,129 @@ Status FileStoreScan::ReadManifestEntries(const std::vector& m return ReadAndNoMergeFileEntries(manifest_metas, manifest_entries); } -Status FileStoreScan::ReadAndMergeFileEntries(const std::vector& manifest_metas, - std::vector* merged_entries) const { +// Cache merged live manifest entries for one bucket before applying scan filters. Each cache value +// keeps a bounded number of snapshot results for the same table/branch/bucket. Exact snapshot hits +// can be returned directly; cache misses rebuild the target snapshot bucket from the target +// snapshot's data manifests. +Status FileStoreScan::ReadManifestEntriesWithCache( + const Snapshot& snapshot, const std::vector& all_manifest_metas, + int32_t bucket, std::vector* manifest_entries) const { + PAIMON_ASSIGN_OR_RAISE(SnapshotLiveManifestEntries cached_entries, + LoadSnapshotLiveManifestEntries(bucket)); + std::optional cached = + cached_entries.LatestBeforeOrEqual(snapshot.Id()); + if (cached && cached->snapshot_id == snapshot.Id()) { + *manifest_entries = *cached->entries; + return Status::OK(); + } + + // Rebuild the target snapshot bucket from all manifests and write the live entries back to the + // cache. + std::vector bucket_manifest_metas; + for (const auto& meta : all_manifest_metas) { + if (MayContainBucket(meta, bucket)) { + bucket_manifest_metas.push_back(meta); + } + } + PAIMON_RETURN_NOT_OK( + ReadAndMergeBucketFileEntries(bucket_manifest_metas, bucket, manifest_entries)); + std::vector cache_entries = *manifest_entries; + cached_entries.Put(snapshot.Id(), std::move(cache_entries)); + PAIMON_RETURN_NOT_OK(StoreSnapshotLiveManifestEntries(bucket, cached_entries)); + return Status::OK(); +} + +std::shared_ptr FileStoreScan::SnapshotLiveManifestEntriesCacheKey(int32_t bucket) const { + return CacheKey::ForSnapshotLiveManifestEntries( + table_path_, BranchManager::NormalizeBranch(core_options_.GetBranch()), bucket); +} + +Result FileStoreScan::LoadSnapshotLiveManifestEntries( + int32_t bucket) const { + auto supplier = [](const std::shared_ptr&) -> Result> { + return std::shared_ptr(); + }; + std::shared_ptr cache_key = SnapshotLiveManifestEntriesCacheKey(bucket); + const auto max_snapshots = core_options_.GetScanManifestEntryCacheMaxSnapshots(); + Result> cache_result = + core_options_.GetCache()->Get(cache_key, supplier); + if (!cache_result.ok() || !cache_result.value()) { + return SnapshotLiveManifestEntries(max_snapshots); + } + Result deserialized = SnapshotLiveManifestEntries::Deserialize( + cache_result.value()->GetSegment(), max_snapshots, pool_); + if (!deserialized.ok()) { + return SnapshotLiveManifestEntries(max_snapshots); + } + return std::move(deserialized.value()); +} + +Status FileStoreScan::StoreSnapshotLiveManifestEntries( + int32_t bucket, const SnapshotLiveManifestEntries& entries) const { + Result> bytes_result = entries.Serialize(pool_); + if (!bytes_result.ok()) { + return Status::OK(); + } + auto cache_value = + std::make_shared(MemorySegment::Wrap(bytes_result.value()), CacheCallback()); + Status status = + core_options_.GetCache()->Put(SnapshotLiveManifestEntriesCacheKey(bucket), cache_value); + return status.ok() ? status : Status::OK(); +} + +Status FileStoreScan::ReadAndMergeBucketFileEntries( + const std::vector& manifest_metas, int32_t bucket, + std::vector* merged_entries) const { std::vector unmerged_entries; - PAIMON_RETURN_NOT_OK(ReadFileEntries(manifest_metas, &unmerged_entries)); + std::vector entries; + PAIMON_RETURN_NOT_OK(ReadFileEntries(manifest_metas, &entries, /*apply_scan_filter=*/false)); + unmerged_entries.reserve(entries.size()); + for (auto& entry : entries) { + if (entry.Bucket() == bucket) { + unmerged_entries.emplace_back(std::move(entry)); + } + } + return MergeLiveEntries(unmerged_entries, merged_entries); +} + +Status FileStoreScan::MergeLiveEntries(const std::vector& unmerged_entries, + std::vector* live_entries) { std::unordered_set deleted_entries; for (const auto& entry : unmerged_entries) { if (entry.Kind() == FileKind::Delete()) { deleted_entries.insert(entry.CreateIdentifier()); } } - for (auto& entry : unmerged_entries) { + for (const auto& entry : unmerged_entries) { if (entry.Kind() == FileKind::Add() && deleted_entries.find(entry.CreateIdentifier()) == deleted_entries.end()) { - merged_entries->push_back(std::move(entry)); + live_entries->push_back(entry); } } return Status::OK(); } +Status FileStoreScan::ReadAndMergeFileEntries(const std::vector& manifest_metas, + std::vector* merged_entries) const { + std::vector unmerged_entries; + PAIMON_RETURN_NOT_OK( + ReadFileEntries(manifest_metas, &unmerged_entries, /*apply_scan_filter=*/true)); + return MergeLiveEntries(unmerged_entries, merged_entries); +} + Status FileStoreScan::ReadAndNoMergeFileEntries( const std::vector& manifest_metas, std::vector* manifest_entries) const { - return ReadFileEntries(manifest_metas, manifest_entries); + return ReadFileEntries(manifest_metas, manifest_entries, /*apply_scan_filter=*/true); +} + +bool FileStoreScan::MayContainBucket(const ManifestFileMeta& manifest, int32_t bucket) const { + const std::optional& min_bucket = manifest.MinBucket(); + const std::optional& max_bucket = manifest.MaxBucket(); + if (min_bucket && max_bucket) { + return bucket >= min_bucket.value() && bucket <= max_bucket.value(); + } + return true; } Result FileStoreScan::FilterManifestFileMeta(const ManifestFileMeta& manifest) const { @@ -321,37 +455,38 @@ bool FileStoreScan::FilterManifestByRowRanges(const ManifestFileMeta& manifest) Status FileStoreScan::ReadManifestFileMeta(const ManifestFileMeta& manifest, std::vector* entries) const { - auto filter = [&](const ManifestEntry& entry) -> Result { - if (partition_filter_) { - PAIMON_ASSIGN_OR_RAISE(bool res, - partition_filter_->Test(partition_schema_, entry.Partition())); - if (!res) { - return false; - } - } - if (only_read_real_buckets_ && entry.Bucket() < 0) { - return false; - } - if (bucket_filter_ != std::nullopt && entry.Bucket() != bucket_filter_.value()) { - return false; - } - if (level_filter_ != nullptr && !level_filter_(entry.Level())) { - return false; - } - return true; - }; std::vector unfiltered_entries; - PAIMON_RETURN_NOT_OK(manifest_file_->Read(manifest.FileName(), filter, &unfiltered_entries)); + PAIMON_RETURN_NOT_OK(manifest_file_->Read( + manifest.FileName(), + [this](const ManifestEntry& entry) -> Result { return FilterManifestEntry(entry); }, + &unfiltered_entries)); entries->reserve(entries->size() + unfiltered_entries.size()); for (auto& entry : unfiltered_entries) { - PAIMON_ASSIGN_OR_RAISE(bool res, FilterByStats(entry)); - if (res) { - entries->emplace_back(std::move(entry)); - } + entries->emplace_back(std::move(entry)); } return Status::OK(); } +Result FileStoreScan::FilterManifestEntry(const ManifestEntry& entry) const { + if (partition_filter_) { + PAIMON_ASSIGN_OR_RAISE(bool res, + partition_filter_->Test(partition_schema_, entry.Partition())); + if (!res) { + return false; + } + } + if (only_read_real_buckets_ && entry.Bucket() < 0) { + return false; + } + if (bucket_filter_ != std::nullopt && entry.Bucket() != bucket_filter_.value()) { + return false; + } + if (level_filter_ != nullptr && !level_filter_(entry.Level())) { + return false; + } + return FilterByStats(entry); +} + Status FileStoreScan::SplitAndSetFilter(const std::vector& partition_keys, const std::shared_ptr& arrow_schema, const std::shared_ptr& scan_filters) { diff --git a/src/paimon/core/operation/file_store_scan.h b/src/paimon/core/operation/file_store_scan.h index 0361a9a6..53d7825a 100644 --- a/src/paimon/core/operation/file_store_scan.h +++ b/src/paimon/core/operation/file_store_scan.h @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -43,6 +42,7 @@ #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" #include "paimon/core/manifest/partition_entry.h" +#include "paimon/core/manifest/snapshot_live_manifest_entries.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/scan_mode.h" @@ -59,6 +59,7 @@ class Schema; } // namespace arrow namespace paimon { +class CacheKey; class Executor; class FileKind; class ManifestFile; @@ -122,6 +123,11 @@ class FileStoreScan { return this; } + FileStoreScan* WithTablePath(const std::string& table_path) { + table_path_ = table_path; + return this; + } + virtual FileStoreScan* EnableValueFilter() { return this; } @@ -239,6 +245,26 @@ class FileStoreScan { Status ReadManifestEntries(const std::vector& manifest_metas, std::vector* manifest_entries) const; + Status ReadManifestEntriesWithCache(const Snapshot& snapshot, + const std::vector& bucket_manifest_metas, + int32_t bucket, + std::vector* manifest_entries) const; + std::shared_ptr SnapshotLiveManifestEntriesCacheKey(int32_t bucket) const; + Result LoadSnapshotLiveManifestEntries(int32_t bucket) const; + Status StoreSnapshotLiveManifestEntries(int32_t bucket, + const SnapshotLiveManifestEntries& entries) const; + + Status ReadAndMergeBucketFileEntries(const std::vector& manifest_metas, + int32_t bucket, + std::vector* merged_entries) const; + + /// Merge raw manifest entries into the set of currently-live files. Entries are deduplicated + /// by identifier (matching Add cancels a prior or following Delete), and lingering Delete + /// entries are dropped so the caller receives Add-only output, matching the semantics of + /// `ReadAndMergeFileEntries`. + static Status MergeLiveEntries(const std::vector& unmerged_entries, + std::vector* live_entries); + Status ReadAndMergeFileEntries(const std::vector& manifest_metas, std::vector* merged_entries) const; @@ -246,7 +272,10 @@ class FileStoreScan { std::vector* manifest_entries) const; Status ReadFileEntries(const std::vector& manifest_metas, - std::vector* manifest_entries) const; + std::vector* manifest_entries, + bool apply_scan_filter) const; + + bool MayContainBucket(const ManifestFileMeta& manifest, int32_t bucket) const; Result FilterManifestFileMeta(const ManifestFileMeta& manifest) const; @@ -255,6 +284,8 @@ class FileStoreScan { Status ReadManifestFileMeta(const ManifestFileMeta& manifest, std::vector* entries) const; + Result FilterManifestEntry(const ManifestEntry& entry) const; + protected: std::shared_ptr pool_; std::shared_ptr schema_manager_; @@ -267,7 +298,6 @@ class FileStoreScan { CoreOptions core_options_; private: - mutable std::mutex lock_; bool only_read_real_buckets_ = false; std::shared_ptr snapshot_manager_; std::shared_ptr manifest_list_; @@ -279,5 +309,6 @@ class FileStoreScan { std::function level_filter_; std::optional specified_snapshot_; std::shared_ptr metrics_; + std::string table_path_; }; } // namespace paimon diff --git a/src/paimon/core/operation/file_store_scan_test.cpp b/src/paimon/core/operation/file_store_scan_test.cpp index e35987ff..c4f0289e 100644 --- a/src/paimon/core/operation/file_store_scan_test.cpp +++ b/src/paimon/core/operation/file_store_scan_test.cpp @@ -18,13 +18,17 @@ #include "paimon/core/operation/file_store_scan.h" +#include + #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_kind.h" #include "paimon/core/manifest/file_source.h" +#include "paimon/core/manifest/snapshot_live_manifest_entries.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/data/timestamp.h" +#include "paimon/memory/memory_segment.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -185,4 +189,88 @@ TEST_F(FileStoreScanTest, TestFilterManifestByRowRanges) { file_store_scan->WithRowRangeIndex(row_range_index); ASSERT_TRUE(file_store_scan->FilterManifestByRowRanges(manifest2)); } + +TEST_F(FileStoreScanTest, TestSnapshotLiveManifestEntries) { + std::vector snapshot1; + ASSERT_OK_AND_ASSIGN( + auto file1, + DataFileMeta::ForAppend("file-1", /*file_size=*/10, /*row_count=*/1, + SimpleStats::EmptyStats(), /*min_sequence_number=*/0, + /*max_sequence_number=*/0, /*schema_id=*/0, + /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt)); + snapshot1.emplace_back(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0, + /*total_buckets=*/1, file1); + SnapshotLiveManifestEntries entries(/*max_snapshots=*/2); + entries.Put(/*snapshot_id=*/1, std::move(snapshot1)); + ASSERT_EQ(entries.Size(), 1); + auto hit = entries.LatestBeforeOrEqual(/*snapshot_id=*/1); + ASSERT_TRUE(hit); + ASSERT_EQ(hit->snapshot_id, 1); + ASSERT_EQ(hit->entries->size(), 1); + ASSERT_EQ((*hit->entries)[0].FileName(), "file-1"); + auto latest_before_2 = entries.LatestBeforeOrEqual(/*snapshot_id=*/2); + ASSERT_TRUE(latest_before_2); + ASSERT_EQ(latest_before_2->snapshot_id, 1); + + std::vector snapshot3; + ASSERT_OK_AND_ASSIGN( + auto file3, + DataFileMeta::ForAppend("file-3", /*file_size=*/10, /*row_count=*/1, + SimpleStats::EmptyStats(), /*min_sequence_number=*/0, + /*max_sequence_number=*/0, /*schema_id=*/0, + /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt)); + snapshot3.emplace_back(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0, + /*total_buckets=*/1, file3); + entries.Put(/*snapshot_id=*/3, std::move(snapshot3)); + + auto latest_before_4 = entries.LatestBeforeOrEqual(/*snapshot_id=*/4); + ASSERT_TRUE(latest_before_4); + ASSERT_EQ(latest_before_4->snapshot_id, 3); + + entries.Put(/*snapshot_id=*/5, {}); + ASSERT_EQ(entries.Size(), 2); + ASSERT_FALSE(entries.LatestBeforeOrEqual(/*snapshot_id=*/1)); + ASSERT_TRUE(entries.LatestBeforeOrEqual(/*snapshot_id=*/3)); + ASSERT_TRUE(entries.LatestBeforeOrEqual(/*snapshot_id=*/5)); +} + +TEST_F(FileStoreScanTest, TestSnapshotLiveManifestEntriesSerialization) { + std::vector manifest_entries; + ASSERT_OK_AND_ASSIGN( + auto file1, + DataFileMeta::ForAppend("file-1", /*file_size=*/10, /*row_count=*/1, + SimpleStats::EmptyStats(), /*min_sequence_number=*/0, + /*max_sequence_number=*/0, /*schema_id=*/0, + /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt)); + manifest_entries.emplace_back(FileKind::Add(), BinaryRow::EmptyRow(), /*bucket=*/0, + /*total_buckets=*/1, file1); + SnapshotLiveManifestEntries entries(/*max_snapshots=*/2); + entries.Put(/*snapshot_id=*/1, std::move(manifest_entries)); + entries.Put(/*snapshot_id=*/3, {}); + + ASSERT_OK_AND_ASSIGN(auto bytes, entries.Serialize(GetDefaultPool())); + ASSERT_OK_AND_ASSIGN(auto deserialized, + SnapshotLiveManifestEntries::Deserialize( + MemorySegment::Wrap(bytes), /*max_snapshots=*/2, GetDefaultPool())); + ASSERT_EQ(deserialized.Size(), 2); + auto hit = deserialized.LatestBeforeOrEqual(/*snapshot_id=*/2); + ASSERT_TRUE(hit); + ASSERT_EQ(hit->snapshot_id, 1); + ASSERT_EQ(hit->entries->size(), 1); + ASSERT_EQ((*hit->entries)[0].FileName(), "file-1"); + ASSERT_EQ(deserialized.LatestBeforeOrEqual(/*snapshot_id=*/4)->snapshot_id, 3); + + ASSERT_OK_AND_ASSIGN(auto evicted_deserialized, SnapshotLiveManifestEntries::Deserialize( + MemorySegment::Wrap(bytes), + /*max_snapshots=*/1, GetDefaultPool())); + ASSERT_EQ(evicted_deserialized.Size(), 1); + ASSERT_FALSE(evicted_deserialized.LatestBeforeOrEqual(/*snapshot_id=*/1)); + ASSERT_EQ(evicted_deserialized.LatestBeforeOrEqual(/*snapshot_id=*/3)->snapshot_id, 3); +} } // namespace paimon::test diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 197c8d4e..f85db826 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -20,6 +20,7 @@ #include "paimon/table/source/table_scan.h" #include +#include #include #include #include @@ -98,19 +99,35 @@ class TableScanImpl { ManifestFile::Create(fs, manifest_file_format, core_options.GetManifestCompression(), path_factory, core_options.GetManifestTargetFileSize(), memory_pool, core_options, partition_schema)); + std::unique_ptr scan; if (table_schema->PrimaryKeys().empty()) { if (core_options.DataEvolutionEnabled()) { - return DataEvolutionFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, context->GetScanFilters(), core_options, executor, memory_pool); + PAIMON_ASSIGN_OR_RAISE( + scan, DataEvolutionFileStoreScan::Create( + snapshot_manager, schema_manager, manifest_list, manifest_file, + table_schema, arrow_schema, context->GetScanFilters(), core_options, + executor, memory_pool)); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, AppendOnlyFileStoreScan::Create( + snapshot_manager, schema_manager, manifest_list, manifest_file, + table_schema, arrow_schema, context->GetScanFilters(), core_options, + executor, memory_pool)); } - return AppendOnlyFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, context->GetScanFilters(), core_options, executor, memory_pool); + } else { + PAIMON_ASSIGN_OR_RAISE( + scan, KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, + context->GetScanFilters(), core_options, + executor, memory_pool)); } - return KeyValueFileStoreScan::Create( - snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, - arrow_schema, context->GetScanFilters(), core_options, executor, memory_pool); + return WithTablePath(std::move(scan), context); + } + + static std::unique_ptr WithTablePath(std::unique_ptr&& scan, + const ScanContext* context) { + scan->WithTablePath(context->GetPath()); + return std::move(scan); } static Result> CreateSplitGenerator( diff --git a/src/paimon/testing/utils/counting_cache_test_utils.h b/src/paimon/testing/utils/counting_cache_test_utils.h index c366fdf4..7bd806e9 100644 --- a/src/paimon/testing/utils/counting_cache_test_utils.h +++ b/src/paimon/testing/utils/counting_cache_test_utils.h @@ -50,12 +50,14 @@ class CountingRoutingCache : public Cache { supplier) override { ++get_count_; last_kind_ = key->GetKind(); + ++get_count_by_kind_[key->GetKind()]; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache, GetCache(key)); return cache->Get( key, [this, supplier = std::move(supplier)](const std::shared_ptr& supplier_key) -> Result> { ++supplier_call_count_; + ++supplier_call_count_by_kind_[supplier_key->GetKind()]; return supplier(supplier_key); }); } @@ -91,15 +93,31 @@ class CountingRoutingCache : public Cache { return get_count_; } + int64_t GetCount(CacheKind kind) const { + return GetCount(get_count_by_kind_, kind); + } + int64_t SupplierCallCount() const { return supplier_call_count_; } + int64_t SupplierCallCount(CacheKind kind) const { + return GetCount(supplier_call_count_by_kind_, kind); + } + CacheKind LastKind() const { return last_kind_; } private: + static int64_t GetCount(const std::map& counts, CacheKind kind) { + auto iter = counts.find(kind); + if (iter == counts.end()) { + return 0; + } + return iter->second; + } + Result> GetCache(const std::shared_ptr& key) const { auto iter = caches_.find(key->GetKind()); if (iter == caches_.end()) { @@ -109,6 +127,8 @@ class CountingRoutingCache : public Cache { } std::map> caches_; + std::map get_count_by_kind_; + std::map supplier_call_count_by_kind_; int64_t get_count_ = 0; int64_t supplier_call_count_ = 0; CacheKind last_kind_ = CacheKind::DEFAULT; diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index e75b8091..0716fd5f 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -7,17 +7,21 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ +#include + #include "arrow/type.h" #include "gtest/gtest.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" @@ -37,9 +41,11 @@ #include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +using DataEvolutionTableParam = std::tuple; + // This is a sdk end-to-end test for data evolution class DataEvolutionTableTest : public ::testing::Test, - public ::testing::WithParamInterface { + public ::testing::WithParamInterface { void SetUp() override { dir_ = UniqueTestDirectory::Create("local"); int64_t seed = DateTimeUtils::GetCurrentUTCTimeUs(); @@ -70,7 +76,7 @@ class DataEvolutionTableTest : public ::testing::Test, void CreateTable(const std::vector& partition_keys) const { std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -144,7 +150,7 @@ class DataEvolutionTableTest : public ::testing::Test, auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - PAIMON_ASSIGN_OR_RAISE(auto scan_context, scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto scan_context, FinishScanContext(scan_context_builder)); PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); PAIMON_ASSIGN_OR_RAISE(auto result_plan, table_scan->CreatePlan()); if (!expected_array && check_scan_plan_when_empty_result) { @@ -210,7 +216,7 @@ class DataEvolutionTableTest : public ::testing::Test, auto global_index_result = BitmapGlobalIndexResult::FromRanges(row_ranges); scan_context_builder.SetGlobalIndexResult(global_index_result); } - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); const auto& result_splits = result_plan->Splits(); @@ -236,6 +242,26 @@ class DataEvolutionTableTest : public ::testing::Test, ASSERT_EQ(result_row_counts, expected_row_counts); } + Result> FinishScanContext(ScanContextBuilder& builder) const { + if (EnableSnapshotLiveManifestCache()) { + if (!snapshot_live_manifest_cache_) { + snapshot_live_manifest_cache_ = + std::make_shared(/*max_weight=*/64 * 1024 * 1024); + } + builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") + .WithCache(snapshot_live_manifest_cache_); + } + return builder.Finish(); + } + + std::string FileFormat() const { + return std::get<0>(GetParam()); + } + + bool EnableSnapshotLiveManifestCache() const { + return std::get<1>(GetParam()); + } + std::shared_ptr PrepareBulkData( int32_t write_batch_size, std::function data_generator, const arrow::FieldVector& fields) const { @@ -255,6 +281,7 @@ class DataEvolutionTableTest : public ::testing::Test, private: std::unique_ptr dir_; + mutable std::shared_ptr snapshot_live_manifest_cache_; arrow::FieldVector fields_ = { arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), @@ -520,7 +547,7 @@ TEST_P(DataEvolutionTableTest, TestOnlySomeColumns) { } TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) { - if (GetParam() != "parquet" && GetParam() != "orc") { + if (FileFormat() != "parquet" && FileFormat() != "orc") { return; } @@ -532,7 +559,7 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) }; std::map options = { {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, @@ -610,7 +637,7 @@ TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -864,7 +891,7 @@ TEST_P(DataEvolutionTableTest, TestMoreData) { TEST_P(DataEvolutionTableTest, TestOnlyRowTrackingEnabled) { std::map options = { {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "false"}, @@ -910,7 +937,7 @@ TEST_P(DataEvolutionTableTest, TestExternalPath) { std::map options = { {Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, @@ -1161,13 +1188,13 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionWithoutPartitionFieldsInFile) { } TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } std::vector partition_keys = {"f1"}; std::map options = { - {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, {"parquet.write.max-row-group-length", "1"}}; @@ -1334,7 +1361,7 @@ TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) { } TEST_P(DataEvolutionTableTest, TestAlterTable) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } @@ -1431,7 +1458,7 @@ TEST_P(DataEvolutionTableTest, TestAlterTable) { } TEST_P(DataEvolutionTableTest, TestReadCompactFiles) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } @@ -1461,7 +1488,7 @@ TEST_P(DataEvolutionTableTest, TestReadCompactFiles) { } TEST_P(DataEvolutionTableTest, TestReadTableWithDenseStats) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } @@ -1542,7 +1569,7 @@ TEST_P(DataEvolutionTableTest, TestReadTableWithDenseStats) { } TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) { - auto file_format = GetParam(); + auto file_format = FileFormat(); if (file_format == "avro") { return; } @@ -1683,7 +1710,7 @@ TEST_P(DataEvolutionTableTest, TestScanAndReadWithIndex) { } TEST_P(DataEvolutionTableTest, TestPredicate) { - if (GetParam() == "avro") { + if (FileFormat() == "avro") { // Avro does not have stats. return; } @@ -1824,7 +1851,7 @@ TEST_P(DataEvolutionTableTest, TestIOException) { TEST_P(DataEvolutionTableTest, TestWithRowIds) { std::map options = {{Options::MANIFEST_FORMAT, "orc"}, - {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_FORMAT, FileFormat()}, {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}}; @@ -1986,7 +2013,7 @@ TEST_P(DataEvolutionTableTest, TestWithRowIds) { /*predicate=*/nullptr, /*row_ranges=*/row_ranges)); } - if (GetParam() == "avro") { + if (FileFormat() == "avro") { // Avro does not support stats. return; } @@ -2046,15 +2073,17 @@ TEST_P(DataEvolutionTableTest, TestWithRowIds) { } } -std::vector GetTestValuesForDataEvolutionTableTest() { - std::vector values; - values.emplace_back("parquet"); +std::vector GetTestValuesForDataEvolutionTableTest() { + std::vector values; + for (bool enable_snapshot_live_manifest_cache : {false, true}) { + values.emplace_back("parquet", enable_snapshot_live_manifest_cache); #ifdef PAIMON_ENABLE_ORC - values.emplace_back("orc"); + values.emplace_back("orc", enable_snapshot_live_manifest_cache); #endif #ifdef PAIMON_ENABLE_AVRO - values.emplace_back("avro"); + values.emplace_back("avro", enable_snapshot_live_manifest_cache); #endif + } return values; } diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index bd0d5cb6..a9294ef0 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,7 @@ #include "arrow/ipc/json_simple.h" #include "gtest/gtest.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/io/cache/lru_cache.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" @@ -68,8 +70,10 @@ class DataSplit; } // namespace paimon namespace paimon::test { +using ScanAndReadParam = std::tuple; + class ScanAndReadInteTest : public testing::Test, - public ::testing::WithParamInterface> { + public ::testing::WithParamInterface { public: void CheckStreamScanResult( const std::unique_ptr& table_scan, const std::unique_ptr& table_read, @@ -132,13 +136,36 @@ class ScanAndReadInteTest : public testing::Test, } void AddReadOptionsForPrefetch(ReadContextBuilder* read_context_builder) { - auto [file_format, enable_prefetch] = GetParam(); read_context_builder->AddOption("test.enable-adaptive-prefetch-strategy", "false"); - if (enable_prefetch) { + if (EnablePrefetch()) { read_context_builder->EnablePrefetch(true).SetPrefetchBatchCount(3); } } + Result> FinishScanContext(ScanContextBuilder& builder) { + if (EnableSnapshotLiveManifestCache()) { + if (!snapshot_live_manifest_cache_) { + snapshot_live_manifest_cache_ = + std::make_shared(/*max_weight=*/64 * 1024 * 1024); + } + builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") + .WithCache(snapshot_live_manifest_cache_); + } + return builder.Finish(); + } + + std::string FileFormat() const { + return std::get<0>(GetParam()); + } + + bool EnablePrefetch() const { + return std::get<1>(GetParam()); + } + + bool EnableSnapshotLiveManifestCache() const { + return std::get<2>(GetParam()); + } + void AdjustSplitWithExternalPath(const std::string& src_path, const std::string& target_path, bool adjust_index, std::vector>* splits_ptr) { @@ -168,6 +195,8 @@ class ScanAndReadInteTest : public testing::Test, } private: + std::shared_ptr snapshot_live_manifest_cache_; + std::shared_ptr arrow_data_type_ = std::dynamic_pointer_cast(DataField::ConvertDataFieldsToArrowStructType( {SpecialFields::ValueKind(), DataField(0, arrow::field("f0", arrow::utf8())), @@ -176,21 +205,24 @@ class ScanAndReadInteTest : public testing::Test, DataField(3, arrow::field("f3", arrow::float64()))})); }; -std::vector> GetTestValuesForScanAndReadInteTest() { - std::vector> values = {{"parquet", false}, {"parquet", true}}; +std::vector GetTestValuesForScanAndReadInteTest() { + std::vector values; + for (bool enable_snapshot_live_manifest_cache : {false, true}) { + values.emplace_back("parquet", false, enable_snapshot_live_manifest_cache); + values.emplace_back("parquet", true, enable_snapshot_live_manifest_cache); #ifdef PAIMON_ENABLE_ORC - values.emplace_back("orc", false); - values.emplace_back("orc", true); + values.emplace_back("orc", false, enable_snapshot_live_manifest_cache); + values.emplace_back("orc", true, enable_snapshot_live_manifest_cache); #endif + } return values; } INSTANTIATE_TEST_SUITE_P(FileFormatAndEnablePaimonPrefetch, ScanAndReadInteTest, - ::testing::ValuesIn(std::vector>( - GetTestValuesForScanAndReadInteTest()))); + ::testing::ValuesIn(GetTestValuesForScanAndReadInteTest())); TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; bool run_complete = false; @@ -201,7 +233,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - Result> scan_context = scan_context_builder.Finish(); + Result> scan_context = FinishScanContext(scan_context_builder); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = TableScan::Create(std::move(scan_context).value()); @@ -245,7 +277,7 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotIOException) { } TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -257,7 +289,7 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - Result> scan_context = scan_context_builder.Finish(); + Result> scan_context = FinishScanContext(scan_context_builder); CHECK_HOOK_STATUS(scan_context.status(), i); Result> table_scan = TableScan::Create(std::move(scan_context).value()); @@ -303,13 +335,13 @@ TEST_P(ScanAndReadInteTest, TestWithPkSnapshotIOException) { } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -344,13 +376,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot1) { } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 3); @@ -386,13 +418,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot3) { } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 5); @@ -433,13 +465,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshot5) { } TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_09.db/append_09"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -482,13 +514,13 @@ TEST_P(ScanAndReadInteTest, TestWithAppendSnapshotWithStreamWithDefaultMode) { } TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_10.db/append_10"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -519,14 +551,14 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshot1) { } TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/append_complex_build_in_fieldid.db/" "append_complex_build_in_fieldid/"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -572,30 +604,29 @@ TEST_P(ScanAndReadInteTest, TestJavaPaimon1WithAppendSnapshotOfNestedType) { } // test pk with dv -TEST_F(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { - auto check_result = [&](const std::string& file_format) { - std::string table_path = GetDataDir() + "/" + file_format + - "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; +TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { + auto file_format = FileFormat(); + std::string table_path = GetDataDir() + "/" + file_format + + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; - // normal batch scan case for pk+dv, all data in level 0 is filtered out - ScanContextBuilder scan_context_builder(table_path); - scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + // normal batch scan case for pk+dv, all data in level 0 is filtered out + ScanContextBuilder scan_context_builder(table_path); + scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); - ReadContextBuilder read_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); - ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ReadContextBuilder read_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); - ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); - ASSERT_EQ(result_plan->SnapshotId().value(), 6); - ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); - ASSERT_OK_AND_ASSIGN(auto read_result, - ReadResultCollector::CollectResult(batch_reader.get())); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_EQ(result_plan->SnapshotId().value(), 6); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(auto read_result, ReadResultCollector::CollectResult(batch_reader.get())); - // check result - auto expected = std::make_shared( - arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([ + // check result + auto expected = std::make_shared( + arrow::ipc::internal::json::ArrayFromJSON(arrow_data_type_, R"([ [0, "Two roads diverged in a wood, and I took the one less traveled by, And that has made all the difference.", 10, 1, 11.0], [0, "Alice", 10, 1, 19.1], [0, "Alex", 10, 0, 16.1], @@ -605,29 +636,23 @@ TEST_F(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6) { [0, "Lucy", 20, 1, 14.1], [0, "Paul", 20, 1, 18.1] ])") - .ValueOrDie()); - ASSERT_TRUE(expected); - ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); + .ValueOrDie()); + ASSERT_TRUE(expected); + ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); - // CountRows should match the number of visible rows returned by CreateReader. - ASSERT_OK_AND_ASSIGN(auto count_reader, - table_read->CreateCountReader(result_plan->Splits())); - ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); - ASSERT_EQ(count, read_result->length()); - }; - for (auto [file_format, enable_prefetch] : GetTestValuesForScanAndReadInteTest()) { - check_result(file_format); - } - check_result("avro"); + // CountRows should match the number of visible rows returned by CreateReader. + ASSERT_OK_AND_ASSIGN(auto count_reader, table_read->CreateCountReader(result_plan->Splits())); + ASSERT_OK_AND_ASSIGN(int64_t count, count_reader->CountRows()); + ASSERT_EQ(count, read_result->length()); } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = GetDataDir() + "/" + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -643,7 +668,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot1) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBucketFilter) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); // all data in level 0 & not in partition 10, bucket 1 is filtered out std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -651,7 +676,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -678,7 +703,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPartitionAndBu } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; // predicate: f0 != "Alice" (key predicate) and f3 > 18 (value predicate) and all data in level @@ -694,7 +719,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { FieldType::DOUBLE, Literal(18.0)); ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({not_equal, greater_than})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -720,7 +745,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithPredicate) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -731,7 +756,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -740,7 +765,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot4WithPredicate) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -748,7 +773,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { // data in partition 20 is truncated ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -777,7 +802,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvBatchScanSnapshot6WithLimit) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -785,7 +810,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -829,7 +854,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot4) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -837,7 +862,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -875,13 +900,13 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot5) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -912,7 +937,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot6) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -920,7 +945,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -971,7 +996,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot1) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -979,7 +1004,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") .AddOption(Options::SCAN_MODE, "from-snapshot") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1016,12 +1041,12 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvStreamFromSnapshot2) { } TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_nested_type.db/pk_table_nested_type/"; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1068,13 +1093,13 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithNestedType) { // test pk with mor TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // normal batch scan case for pk+mor, use latest snapshot if not specified ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1113,7 +1138,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanLatestSnapshot) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1121,7 +1146,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { // with merge read ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1157,7 +1182,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot2) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndBucketFilter) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1165,7 +1190,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1195,7 +1220,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPartitionAndB } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1221,7 +1246,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { PredicateBuilder::And({not_equal, less_than, less_or_equal})); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1255,7 +1280,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithPredicate) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1267,7 +1292,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -1276,7 +1301,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot3WithPredicate) { } TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapshot3) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_dv.db/pk_table_scan_and_read_dv/"; @@ -1285,7 +1310,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho .AddOption(Options::MERGE_ENGINE, "aggregation") .AddOption("fields.f3.aggregate-function", "rbm32"); scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1315,7 +1340,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithDvWithInvalidAggregateBatchScanSnapsho } TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapshot3) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1324,7 +1349,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh .AddOption(Options::MERGE_ENGINE, "aggregation") .AddOption("fields.f3.aggregate-function", "rbm32"); scan_context_builder.SetBucketFilter(1).SetPartitionFilter({{{"f1", "10"}}}); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1342,7 +1367,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorWithInvalidAggregateBatchScanSnapsh } TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1355,7 +1380,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1385,7 +1410,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithAggregateBatchScanSnapshot3WithPredica } TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1398,7 +1423,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(20.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1428,7 +1453,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithPartialUpdateBatchScanSnapshot3WithPre } TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1436,7 +1461,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { // merging ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(6); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1469,7 +1494,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorBatchScanSnapshot5WithLimit) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1477,7 +1502,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "4") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1514,7 +1539,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot4) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1522,7 +1547,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1573,7 +1598,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot1) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1583,7 +1608,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2") .AddOption(Options::SCAN_MODE, "from-snapshot") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1620,7 +1645,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot2) { } TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -1631,7 +1656,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"f3", FieldType::DOUBLE, Literal(50.0)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1658,7 +1683,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithMorStreamFromSnapshot5WithPredicate) { // test first row merge engine TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; @@ -1666,7 +1691,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { // normal batch scan case for pk+first row, all data in level 0 is filtered out ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "5"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1698,7 +1723,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowBatchScanSnapshot5) { } TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; @@ -1707,7 +1732,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "3") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1751,14 +1776,14 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot3) { } TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_first_row.db/pk_table_scan_and_read_first_row/"; ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_MODE, "latest-full").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1791,13 +1816,13 @@ TEST_P(ScanAndReadInteTest, TestWithPKWithFirstRowStreamFromSnapshot5) { } TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_09.db/pk_09/"; // normal batch scan case for pk+dv (09 version) ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetLimit(2); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1829,7 +1854,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKWith09VersionDvBatchScanLatestSnapshot) { } TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); auto check_result = [&](const std::string& table_path, @@ -1837,7 +1862,7 @@ TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { const std::shared_ptr& expected) { ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1896,7 +1921,7 @@ TEST_P(ScanAndReadInteTest, TestWithEmptyPartitionValue) { } TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_empty_partition_with_empty_value.db/" "append_with_empty_partition_with_empty_value/"; @@ -1906,7 +1931,7 @@ TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { const std::shared_ptr& expected) { ScanContextBuilder scan_context_builder(table_path); scan_context_builder.SetPartitionFilter(partition_filters); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -1949,13 +1974,13 @@ TEST_P(ScanAndReadInteTest, TestWithMultipleEmptyPartitionValue) { } TEST_P(ScanAndReadInteTest, TestMemoryUse) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_09.db/append_09/"; // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -1994,7 +2019,7 @@ TEST_P(ScanAndReadInteTest, TestMemoryUse) { } TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); auto test_dir = UniqueTestDirectory::Create("local"); arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), @@ -2057,7 +2082,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { // batch scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.WithStreamingMode(false); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -2067,7 +2092,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { // stream scan: from snapshot 1 ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -2101,7 +2126,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { // stream scan: from snapshot 2 ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2").WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -2129,7 +2154,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "1") .AddOption(Options::SCAN_MODE, "from-snapshot-full") .WithStreamingMode(true); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -2161,7 +2186,7 @@ TEST_P(ScanAndReadInteTest, TestPkScanWithPostponeBucket) { } TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForParquet) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); if (file_format != "parquet") { return; } @@ -2172,7 +2197,7 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar auto predicate = PredicateBuilder::LessThan( /*field_index=*/3, /*field_name=*/"f4", FieldType::INT, Literal(300006)); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -2211,7 +2236,7 @@ TEST_P(ScanAndReadInteTest, TestScanWithPredicateAndReadWithUnorderedFieldForPar } TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_with_alter_table.db/pk_table_with_alter_table/"; @@ -2220,7 +2245,7 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 6); @@ -2261,7 +2286,7 @@ TEST_P(ScanAndReadInteTest, TestPkSchemaEvolutionScanWithRenamedPkPredicate) { } TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); if (file_format != "parquet") { return; } @@ -2271,7 +2296,7 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { // scan ScanContextBuilder scan_context_builder(table_path); scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "2"); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 2); @@ -2303,12 +2328,12 @@ TEST_P(ScanAndReadInteTest, TestAppendTableWithMultipleFileFormat) { } TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_no_external.db/pk_dv_index_in_data_no_external"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -2341,13 +2366,13 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndNoExternalPath) { } TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_no_external.db/pk_dv_index_not_in_data_no_external"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -2380,7 +2405,7 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndNoExternalPath) { } TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_not_in_data_with_external.db/pk_dv_index_not_in_data_with_external"; @@ -2388,7 +2413,7 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { "/pk_dv_index_not_in_data_with_external.db/external"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -2423,7 +2448,7 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexNotInDataAndWithExternalPath) { } TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_bitmap.db/append_with_bitmap"; auto predicate = @@ -2468,7 +2493,7 @@ TEST_P(ScanAndReadInteTest, TestScanAndReadWithDisableIndex) { } TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_with_external.db/pk_dv_index_in_data_with_external"; @@ -2476,7 +2501,7 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { paimon::test::GetDataDir() + file_format + "/pk_dv_index_in_data_with_external.db/external"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 4); @@ -2511,12 +2536,12 @@ TEST_P(ScanAndReadInteTest, TestPkDvTableIndexInDataAndWithExternalPath) { } TEST_P(ScanAndReadInteTest, TestTimestampType) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_multiple_ts_precision_and_timezone.db" "/append_with_multiple_ts_precision_and_timezone/"; ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2557,13 +2582,13 @@ TEST_P(ScanAndReadInteTest, TestTimestampType) { TEST_P(ScanAndReadInteTest, TestCastTimestampType) { TimezoneGuard tz_guard("Asia/Shanghai"); - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/append_with_cast_timestamp.db" "/append_with_cast_timestamp/"; // scan ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); ASSERT_EQ(result_plan->SnapshotId().value(), 1); @@ -2742,7 +2767,7 @@ TEST_F(ScanAndReadInteTest, TestAvroWithPkTable) { } TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); // Verify BucketSelectConverter: an EQUAL predicate on bucket key f2 should automatically // derive the target bucket, without explicitly calling SetBucketFilter. // From the existing test f2=0 maps to bucket 1, f2=1 maps to bucket 0. @@ -2757,7 +2782,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { scan_context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, "6"); scan_context_builder.SetPartitionFilter({{{"f1", "10"}}}); scan_context_builder.SetPredicate(predicate); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ReadContextBuilder read_context_builder(table_path); @@ -2795,7 +2820,7 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { } TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; @@ -2812,13 +2837,13 @@ TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { } TEST_P(ScanAndReadInteTest, TestCountRowsConsistencyWithCreateReader) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Scan latest snapshot ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2845,13 +2870,13 @@ TEST_P(ScanAndReadInteTest, TestCountRowsConsistencyWithCreateReader) { } TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithPredicateNotSupported) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); @@ -2869,13 +2894,13 @@ TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithPredicateNotSupported) { } TEST_P(ScanAndReadInteTest, TestCreateCountReaderWithForceKeepDeleteNotSupported) { - auto [file_format, enable_prefetch] = GetParam(); + auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + "/pk_table_scan_and_read_mor.db/pk_table_scan_and_read_mor/"; // Create splits from latest snapshot. ScanContextBuilder scan_context_builder(table_path); - ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto scan_context, FinishScanContext(scan_context_builder)); ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); diff --git a/test/inte/scan_inte_test.cpp b/test/inte/scan_inte_test.cpp index 3a0531a9..fdc38ea9 100644 --- a/test/inte/scan_inte_test.cpp +++ b/test/inte/scan_inte_test.cpp @@ -67,10 +67,12 @@ class ScanInteTest : public testing::TestWithParam { Result> FinishScanContext(ScanContextBuilder& builder) { if (GetParam() == ManifestCacheMode::Cache) { if (!cache_) { - cache_ = - std::make_shared(CacheKind::MANIFEST, 64 * 1024 * 1024); + cache_ = std::make_shared(std::map{ + {CacheKind::MANIFEST, 64 * 1024 * 1024}, + {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); } - builder.WithCache(cache_); + builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "3") + .WithCache(cache_); } return builder.Finish(); } @@ -288,6 +290,110 @@ TEST(ScanInteManifestCacheTest, TestRepeatedScanReusesManifestCache) { ASSERT_EQ(supplier_calls_after_first_scan, manifest_cache->SupplierCallCount()); } +Result>> RunBucketSnapshotScan( + const std::string& table_path, int64_t snapshot_id, int32_t bucket, + const std::shared_ptr& cache, int32_t max_snapshot_live_manifest_versions) { + ScanContextBuilder context_builder(table_path); + context_builder.AddOption(Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)) + .SetBucketFilter(bucket); + if (max_snapshot_live_manifest_versions > 0) { + context_builder.AddOption(Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, + std::to_string(max_snapshot_live_manifest_versions)); + } + if (cache) { + context_builder.WithCache(cache); + } + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_scan, TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(auto plan, table_scan->CreatePlan()); + + std::vector> data_splits; + for (const auto& split : plan->Splits()) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("expected DataSplitImpl from bucket snapshot scan"); + } + data_splits.push_back(data_split); + } + return data_splits; +} + +void AssertDataSplitsEqual(const std::vector>& expected, + const std::vector>& actual) { + ASSERT_EQ(actual.size(), expected.size()); + for (size_t i = 0; i < actual.size(); ++i) { + ASSERT_EQ(*actual[i], *expected[i]) << actual[i]->ToString() << std::endl + << expected[i]->ToString(); + } +} + +std::shared_ptr CreateSnapshotLiveManifestTestCache() { + return std::make_shared( + std::map{{CacheKind::MANIFEST, 64 * 1024 * 1024}, + {CacheKind::SNAPSHOT_LIVE_MANIFEST, 64 * 1024 * 1024}}); +} + +TEST(ScanInteManifestCacheTest, TestRepeatedBucketSnapshotScanReusesSnapshotLiveManifestCache) { + std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + auto cache = CreateSnapshotLiveManifestTestCache(); + + ASSERT_OK_AND_ASSIGN(auto expected, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, + /*bucket=*/1, nullptr, + /*max_snapshot_live_manifest_versions=*/0)); + ASSERT_OK_AND_ASSIGN(auto first, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, + /*bucket=*/1, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected, first); + ASSERT_GT(cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST), 0); + int64_t get_count_after_first_scan = cache->GetCount(CacheKind::SNAPSHOT_LIVE_MANIFEST); + int64_t supplier_calls_after_first_scan = + cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST); + + ASSERT_OK_AND_ASSIGN(auto second, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, + /*bucket=*/1, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected, second); + ASSERT_EQ(get_count_after_first_scan + 1, cache->GetCount(CacheKind::SNAPSHOT_LIVE_MANIFEST)); + ASSERT_EQ(supplier_calls_after_first_scan, + cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST)); +} + +TEST(ScanInteManifestCacheTest, TestSnapshotLiveManifestCacheRetainsSnapshotsPerBucketValue) { + std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; + auto cache = CreateSnapshotLiveManifestTestCache(); + + ASSERT_OK_AND_ASSIGN(auto expected_snapshot3, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/3, /*bucket=*/0, nullptr, + /*max_snapshot_live_manifest_versions=*/0)); + ASSERT_OK_AND_ASSIGN(auto cached_snapshot3, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/3, /*bucket=*/0, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected_snapshot3, cached_snapshot3); + + ASSERT_OK_AND_ASSIGN(auto expected_snapshot5, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, /*bucket=*/0, nullptr, + /*max_snapshot_live_manifest_versions=*/0)); + ASSERT_OK_AND_ASSIGN(auto cached_snapshot5, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/5, /*bucket=*/0, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected_snapshot5, cached_snapshot5); + int64_t get_count_after_snapshot5 = cache->GetCount(CacheKind::SNAPSHOT_LIVE_MANIFEST); + int64_t supplier_calls_after_snapshot5 = + cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST); + + ASSERT_OK_AND_ASSIGN(auto cached_snapshot3_again, + RunBucketSnapshotScan(table_path, /*snapshot_id=*/3, /*bucket=*/0, cache, + /*max_snapshot_live_manifest_versions=*/3)); + AssertDataSplitsEqual(expected_snapshot3, cached_snapshot3_again); + ASSERT_EQ(get_count_after_snapshot5 + 1, cache->GetCount(CacheKind::SNAPSHOT_LIVE_MANIFEST)); + ASSERT_EQ(supplier_calls_after_snapshot5, + cache->SupplierCallCount(CacheKind::SNAPSHOT_LIVE_MANIFEST)); +} + TEST_P(ScanInteTest, TestScanAppendWithSnapshot1) { std::string table_path = paimon::test::GetDataDir() + "orc/append_09.db/append_09"; ScanContextBuilder context_builder(table_path); From c2fd3e1d045be06a939d4f6255d7b29697f0f3d0 Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:23:46 +0800 Subject: [PATCH 087/138] feat(parquet): support pushing bitmaps down to page-level filtering (except for nested columns) --- include/paimon/utils/roaring_bitmap32.h | 5 + src/paimon/common/utils/roaring_bitmap32.cpp | 29 ++ .../common/utils/roaring_bitmap32_test.cpp | 19 + .../format/parquet/file_reader_wrapper.cpp | 44 +- .../format/parquet/file_reader_wrapper.h | 1 + .../page_filtered_row_group_reader.cpp | 8 +- .../parquet/page_filtered_row_group_reader.h | 1 + .../page_filtered_row_group_reader_test.cpp | 392 +++++++++++++++++- .../parquet/parquet_file_batch_reader.cpp | 219 +++++++--- .../parquet/parquet_file_batch_reader.h | 38 +- .../parquet_file_batch_reader_test.cpp | 117 +++++- src/paimon/format/parquet/row_ranges.h | 17 - src/paimon/format/parquet/target_row_group.h | 115 +++++ 13 files changed, 873 insertions(+), 132 deletions(-) create mode 100644 src/paimon/format/parquet/target_row_group.h diff --git a/include/paimon/utils/roaring_bitmap32.h b/include/paimon/utils/roaring_bitmap32.h index 24b5c98e..de77ce53 100644 --- a/include/paimon/utils/roaring_bitmap32.h +++ b/include/paimon/utils/roaring_bitmap32.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -137,6 +138,10 @@ class PAIMON_EXPORT RoaringBitmap32 { Iterator End() const; /// @return the iterator moved to the value which is equal or larger than key Iterator EqualOrLarger(int32_t key) const; + /// @return the first value which is equal or larger than x + std::optional NextValue(int32_t x) const; + /// @return the largest value which is smaller than x + std::optional PreviousValue(int32_t x) const; /// Computes the intersection between two bitmaps and returns new bitmap. /// The current bitmap and the provided bitmap are unchanged. diff --git a/src/paimon/common/utils/roaring_bitmap32.cpp b/src/paimon/common/utils/roaring_bitmap32.cpp index af5214b4..8a90952a 100644 --- a/src/paimon/common/utils/roaring_bitmap32.cpp +++ b/src/paimon/common/utils/roaring_bitmap32.cpp @@ -312,4 +312,33 @@ RoaringBitmap32::Iterator RoaringBitmap32::EqualOrLarger(int32_t key) const { return iter; } +std::optional RoaringBitmap32::NextValue(int32_t x) const { + auto iter = EqualOrLarger(x); + if (iter == End()) { + return std::nullopt; + } + return *iter; +} + +std::optional RoaringBitmap32::PreviousValue(int32_t x) const { + if (IsEmpty()) { + return std::nullopt; + } + + auto& bitmap = GetRoaringBitmap(roaring_bitmap_); + if (x <= static_cast(bitmap.minimum())) { + return std::nullopt; + } + + const uint64_t rank = bitmap.rank(static_cast(x - 1)); + if (rank == 0) { + return std::nullopt; + } + + uint32_t value = 0; + [[maybe_unused]] bool found = bitmap.select(static_cast(rank - 1), &value); + assert(found); + return static_cast(value); +} + } // namespace paimon diff --git a/src/paimon/common/utils/roaring_bitmap32_test.cpp b/src/paimon/common/utils/roaring_bitmap32_test.cpp index 008ae259..239381f2 100644 --- a/src/paimon/common/utils/roaring_bitmap32_test.cpp +++ b/src/paimon/common/utils/roaring_bitmap32_test.cpp @@ -256,6 +256,25 @@ TEST(RoaringBitmap32Test, TestIterator) { ASSERT_EQ(iter, roaring.End()); } +TEST(RoaringBitmap32Test, TestNextAndPreviousValue) { + RoaringBitmap32 roaring = RoaringBitmap32::From({10, 20, 30}); + + ASSERT_EQ(roaring.NextValue(5), std::optional(10)); + ASSERT_EQ(roaring.NextValue(10), std::optional(10)); + ASSERT_EQ(roaring.NextValue(25), std::optional(30)); + ASSERT_EQ(roaring.NextValue(31), std::nullopt); + + ASSERT_EQ(roaring.PreviousValue(10), std::nullopt); + ASSERT_EQ(roaring.PreviousValue(11), std::optional(10)); + ASSERT_EQ(roaring.PreviousValue(20), std::optional(10)); + ASSERT_EQ(roaring.PreviousValue(21), std::optional(20)); + ASSERT_EQ(roaring.PreviousValue(100), std::optional(30)); + + RoaringBitmap32 empty; + ASSERT_EQ(empty.NextValue(0), std::nullopt); + ASSERT_EQ(empty.PreviousValue(0), std::nullopt); +} + TEST(RoaringBitmap32Test, TestIteratorAssignAndMove) { RoaringBitmap32 roaring = RoaringBitmap32::From({10, 100, 200}); diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 11222963..82515696 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -166,14 +166,15 @@ void FileReaderWrapper::AdvanceToNextRowGroup() { current_row_group_idx_++; // Skip row groups excluded by read range. while (current_row_group_idx_ < target_row_groups_.size() && - target_row_groups_[current_row_group_idx_].excluded_by_read_range) { + target_row_groups_[current_row_group_idx_].IsExcludedByReadRange()) { current_row_group_idx_++; } if (current_row_group_idx_ >= target_row_groups_.size()) { next_row_to_read_ = num_rows_; } else { next_row_to_read_ = - all_row_group_ranges_[target_row_groups_[current_row_group_idx_].row_group_index].first; + all_row_group_ranges_[target_row_groups_[current_row_group_idx_].GetRowGroupIndex()] + .first; } } @@ -183,10 +184,10 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { filtered_global_offset_ = 0; for (uint64_t i = 0; i < target_row_groups_.size(); i++) { - if (target_row_groups_[i].excluded_by_read_range) { + if (target_row_groups_[i].IsExcludedByReadRange()) { continue; } - int32_t rg_id = target_row_groups_[i].row_group_index; + int32_t rg_id = target_row_groups_[i].GetRowGroupIndex(); uint64_t rg_start = all_row_group_ranges_[rg_id].first; uint64_t rg_end = all_row_group_ranges_[rg_id].second; if (row_number > rg_start && row_number < rg_end) { @@ -202,9 +203,9 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { // Rebuild batch_reader_ for non-page-filtered RGs at/after seek position. std::vector fully_matched_indices; for (uint64_t j = i; j < target_row_groups_.size(); j++) { - if (!target_row_groups_[j].excluded_by_read_range && - !target_row_groups_[j].is_partially_matched) { - fully_matched_indices.push_back(target_row_groups_[j].row_group_index); + if (!target_row_groups_[j].IsExcludedByReadRange() && + !target_row_groups_[j].IsPartiallyMatched()) { + fully_matched_indices.push_back(target_row_groups_[j].GetRowGroupIndex()); } } if (!fully_matched_indices.empty()) { @@ -224,7 +225,7 @@ Status FileReaderWrapper::SeekToRow(uint64_t row_number) { } Result> FileReaderWrapper::NextPageFiltered() { - int32_t rg_id = target_row_groups_[current_row_group_idx_].row_group_index; + int32_t rg_id = target_row_groups_[current_row_group_idx_].GetRowGroupIndex(); // Construct the per-RG streaming reader on demand. if (!current_page_filtered_reader_) { @@ -239,7 +240,7 @@ Result> FileReaderWrapper::NextPageFiltered( file_reader_->parquet_reader(), target_rg, target_column_indices_, page_filtered_read_schema_, file_reader_->properties().cache_options(), pre_buffered, page_ranges, max_chunksize, pool_)); - current_filtered_row_ranges_ = target_rg.row_ranges; + current_filtered_row_ranges_ = target_rg.GetRowRanges(); current_filtered_rg_start_ = all_row_group_ranges_[rg_id].first; filtered_global_offset_ = 0; } @@ -275,7 +276,7 @@ Result> FileReaderWrapper::NextFullyMatched( return std::shared_ptr(); } - int32_t rg_id = target_row_groups_[current_row_group_idx_].row_group_index; + int32_t rg_id = target_row_groups_[current_row_group_idx_].GetRowGroupIndex(); uint64_t rg_end = all_row_group_ranges_[rg_id].second; int64_t num_rows = record_batch->num_rows(); @@ -300,7 +301,7 @@ Result> FileReaderWrapper::Next() { while (current_row_group_idx_ < target_row_groups_.size()) { bool is_partially_matched = - target_row_groups_[current_row_group_idx_].is_partially_matched; + target_row_groups_[current_row_group_idx_].IsPartiallyMatched(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr batch, is_partially_matched ? NextPageFiltered() : NextFullyMatched()); if (batch) { @@ -370,9 +371,9 @@ std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges( auto file_metadata = file_reader_->parquet_reader()->metadata(); for (const auto& trg : target_row_groups_) { - if (trg.excluded_by_read_range) continue; + if (trg.IsExcludedByReadRange()) continue; - if (trg.is_partially_matched) { + if (trg.IsPartiallyMatched()) { // Page-filtered RGs: only matching page byte ranges. auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges( file_reader_->parquet_reader(), trg, column_indices); @@ -380,7 +381,7 @@ std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges( std::make_move_iterator(page_ranges.end())); } else { // Fully-matched RGs: entire column chunk ranges. - auto rg_metadata = file_metadata->RowGroup(trg.row_group_index); + auto rg_metadata = file_metadata->RowGroup(trg.GetRowGroupIndex()); for (int32_t col_idx : column_indices) { auto col_chunk = rg_metadata->ColumnChunk(col_idx); int64_t offset = col_chunk->data_page_offset(); @@ -418,12 +419,12 @@ Status FileReaderWrapper::PrepareForReading(const std::vector& t std::vector fully_matched_row_groups; uint64_t active_count = 0; for (const auto& trg : target_row_groups_) { - if (trg.excluded_by_read_range) { + if (trg.IsExcludedByReadRange()) { continue; } active_count++; - if (!trg.is_partially_matched) { - fully_matched_row_groups.push_back(trg.row_group_index); + if (!trg.IsPartiallyMatched()) { + fully_matched_row_groups.push_back(trg.GetRowGroupIndex()); } } @@ -457,14 +458,15 @@ Status FileReaderWrapper::PrepareForReading(const std::vector& t // Reset read state. Find the first non-excluded row group. uint64_t first_active_idx = 0; while (first_active_idx < target_row_groups_.size() && - target_row_groups_[first_active_idx].excluded_by_read_range) { + target_row_groups_[first_active_idx].IsExcludedByReadRange()) { first_active_idx++; } if (first_active_idx >= target_row_groups_.size()) { next_row_to_read_ = num_rows_; } else { next_row_to_read_ = - all_row_group_ranges_[target_row_groups_[first_active_idx].row_group_index].first; + all_row_group_ranges_[target_row_groups_[first_active_idx].GetRowGroupIndex()] + .first; } previous_first_row_ = std::numeric_limits::max(); current_row_group_idx_ = first_active_idx; @@ -478,7 +480,7 @@ Status FileReaderWrapper::ApplyReadRanges( const std::vector>& read_ranges) { if (read_ranges.empty()) { for (auto& trg : target_row_groups_) { - trg.excluded_by_read_range = true; + trg.SetExcludedByReadRange(true); } reader_initialized_ = false; return Status::OK(); @@ -494,7 +496,7 @@ Status FileReaderWrapper::ApplyReadRanges( } // Mark each target row group as excluded or not based on the matching set. for (auto& trg : target_row_groups_) { - trg.excluded_by_read_range = matching_rg_indices.count(trg.row_group_index) == 0; + trg.SetExcludedByReadRange(matching_rg_indices.count(trg.GetRowGroupIndex()) == 0); } reader_initialized_ = false; return Status::OK(); diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h index 6cc5464f..ddde5835 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.h +++ b/src/paimon/format/parquet/file_reader_wrapper.h @@ -35,6 +35,7 @@ #include "arrow/type_fwd.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/parquet/row_ranges.h" +#include "paimon/format/parquet/target_row_group.h" #include "paimon/result.h" #include "paimon/status.h" #include "parquet/arrow/reader.h" diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index 080ba300..e7ca9d3b 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -237,8 +237,8 @@ Result> PageFilteredRowGroupReader::Re const ::arrow::io::CacheOptions& cache_options, bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize, std::shared_ptr<::arrow::MemoryPool> pool) { - const auto& row_ranges = target_row_group.row_ranges; - int32_t row_group_index = target_row_group.row_group_index; + const auto& row_ranges = target_row_group.GetRowRanges(); + int32_t row_group_index = target_row_group.GetRowGroupIndex(); if (row_ranges.IsEmpty()) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr empty_table, @@ -292,8 +292,8 @@ Result> PageFilteredRowGroupReader::Re std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRanges( ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group, const std::vector& column_indices) { - int32_t row_group_index = target_row_group.row_group_index; - const auto& row_ranges = target_row_group.row_ranges; + int32_t row_group_index = target_row_group.GetRowGroupIndex(); + const auto& row_ranges = target_row_group.GetRowRanges(); std::vector<::arrow::io::ReadRange> ranges; auto file_metadata = parquet_reader->metadata(); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h index 7ff46c5a..6b8faf3d 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.h +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h @@ -30,6 +30,7 @@ #include "arrow/record_batch.h" #include "arrow/type.h" #include "paimon/format/parquet/row_ranges.h" +#include "paimon/format/parquet/target_row_group.h" #include "paimon/result.h" #include "parquet/column_reader.h" #include "parquet/file_reader.h" diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 0186f309..35cffc12 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -47,6 +47,7 @@ #include "paimon/status.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/utils/roaring_bitmap32.h" #include "parquet/arrow/reader.h" #include "parquet/file_reader.h" #include "parquet/properties.h" @@ -132,6 +133,31 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { paimon::test::ReadResultCollector::CollectResult(batch_reader.get())); } + /// Read back a Parquet file with a predicate, a bitmap, and page index filter enabled. + void ReadWithPredicateAndBitmapImpl(const std::string& file_name, + const std::shared_ptr& read_schema, + const std::shared_ptr& predicate, + const RoaringBitmap32& bitmap, + std::shared_ptr* out, + int32_t batch_size = 1024, + bool enable_page_level_filter = true) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + + std::map options; + options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = + enable_page_level_filter ? "true" : "false"; + ASSERT_OK_AND_ASSIGN(auto batch_reader, + ParquetFileBatchReader::Create(std::move(in_stream), options, + batch_size, nullptr, arrow_pool_)); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate, bitmap)); + ASSERT_OK_AND_ASSIGN(*out, + paimon::test::ReadResultCollector::CollectResult(batch_reader.get())); + } + protected: std::shared_ptr arrow_pool_; std::shared_ptr pool_; @@ -942,14 +968,14 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) { ASSERT_TRUE(expected->Equals(result->chunk(0))); } -/// Test: Page-level filtering reading the nested struct column along with the predicate column. +/// Test: Page-level filtering reading only the predicate column (no nested column in read schema). /// -/// This verifies that when reading a subset of columns that includes a nested column -/// and the predicate column, the schema mapping and column assembly work correctly. +/// This verifies that when reading only the "id" column (without the nested struct), +/// page-level filtering works correctly since the read schema contains no nested types. /// /// Schema: { id: int32, info: struct } -/// Read schema: { id: int32, info: struct } -/// Predicate on "id": id >= 70. +/// Read schema: { id: int32 } +/// Predicate on "id": id >= 70. Page-level filtering active → rows 70-99 (30 rows). TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) { std::string file_name = dir_->Str() + "/nested_struct_only_nested.parquet"; auto data = MakeNestedStructData(100); @@ -1090,6 +1116,95 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) { ASSERT_TRUE(expected->Equals(result->chunk(0))); } +/// Test: nested map projection falls back to row-group-level filtering when page index filter is +/// unavailable for nested read schemas. +/// +/// Schema: { id: int32, props: map } +/// 100 rows, 10 per page, 2 row group. +/// Bitmap: {70..99} hits the second row group (50..99). +/// Because nested schema disables page-level filtering, the entire row group 1 (50..99) is read, +/// so rows [50, 99] should all be returned. +TEST_F(PageFilteredRowGroupReaderTest, NestedMapBitmapFallback) { + std::string file_name = dir_->Str() + "/nested_map_projection_fallback.parquet"; + auto data = MakeMapColumnData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32())); + auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), field_props}); + + RoaringBitmap32 bitmap; + bitmap.AddRange(70, 100); + + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); + + ASSERT_TRUE(result); + // Because page-level filtering is skipped for nested schemas, we read full row groups. + ASSERT_EQ(50, result->length()); + + auto expected = data->Slice(50, 50); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} + +/// Test: nested list projection falls back to row-group-level filtering when page index filter is +/// unavailable for nested read schemas. +/// +/// Schema: { id: int32, tags: list } +/// 100 rows, 10 per page, 2 row group. +/// Bitmap: {70..99} hits the second row group (50..99). +/// Because nested schema disables page-level filtering, the entire row group 1 (50..99) is read, +/// so rows [50, 99] should all be returned. +TEST_F(PageFilteredRowGroupReaderTest, NestedListBitmapFallback) { + std::string file_name = dir_->Str() + "/nested_list_projection_fallback.parquet"; + auto data = MakeListColumnData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))); + auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), field_tags}); + + RoaringBitmap32 bitmap; + bitmap.AddRange(70, 100); + + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + auto expected = data->Slice(50, 50); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} + +/// Test: nested struct projection falls back to row-group-level filtering when page index filter is +/// unavailable for nested read schemas. +/// +/// Schema: { id: int32, info: struct } +/// Bitmap: {70..99} hits the second row group (50..99). +/// Because nested schema disables page-level filtering, the entire second row group (50..99) is +/// read. +TEST_F(PageFilteredRowGroupReaderTest, NestedStructBitmapFallback) { + std::string file_name = dir_->Str() + "/nested_struct_projection_fallback.parquet"; + auto data = MakeNestedStructData(100); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50); + + auto field_x = arrow::field("x", arrow::int32()); + auto field_y = arrow::field("y", arrow::int32()); + auto field_info = arrow::field("info", arrow::struct_({field_x, field_y})); + auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), field_info}); + + RoaringBitmap32 bitmap; + bitmap.AddRange(70, 100); + + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result); + + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + auto expected = data->Slice(50, 50); + ASSERT_TRUE(expected->Equals(result->chunk(0))); +} + /// Test: rowgroup-level filtering with multiple adjacent nested columns (struct + list). /// /// Schema: { id: int32, info: struct, tags: list } @@ -1146,5 +1261,272 @@ TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) { auto expected = data->Slice(50, 50); ASSERT_TRUE(expected->Equals(result->chunk(0))); } +/// Test: bitmap hits all pages of a subset of row groups (no predicate). +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// RG0: rows 0-99, RG1: rows 100-199. +/// Bitmap: {0..99} hits all pages of RG0, RG1 is excluded entirely. +/// Expected: 100 rows (0-99). +TEST_F(PageFilteredRowGroupReaderTest, BitmapAllPagesSomeRowGroups) { + std::string file_name = dir_->Str() + "/bitmap_all_pages_rg.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 100); // hits all of RG0 + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(100, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 100; ++i) { + ASSERT_EQ(i, val_arr->Value(i)); + } +} + +/// Test: bitmap hits partial pages of a row group (no predicate). +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {30..59} hits pages 3-5 of RG0 (rows 30-59), RG1 excluded. +/// Expected: 30 rows (30-59). +TEST_F(PageFilteredRowGroupReaderTest, BitmapPartialPagesSingleRowGroup) { + std::string file_name = dir_->Str() + "/bitmap_partial_pages_rg.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(90, 110); // hits pages 3-5 of RG0 + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(20, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 20; ++i) { + ASSERT_EQ(90 + i, val_arr->Value(i)); + } +} + +/// Test: bitmap hits all pages of some row groups and partial pages of others. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {0..99} hits all of RG0 + {120..149} hits pages 2-4 of RG1. +/// Expected: 100 (RG0) + 30 (RG1 partial) = 130 rows. +TEST_F(PageFilteredRowGroupReaderTest, BitmapAllAndPartialPagesMixed) { + std::string file_name = dir_->Str() + "/bitmap_all_and_partial.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 100); // all of RG0 + bitmap.AddRange(120, 150); // pages 2-4 of RG1 + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(130, result->length()); + + // Verify: rows 0-99 + 120-149 + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 100; ++i) { + ASSERT_EQ(i, val_arr->Value(i)); + } + for (int32_t i = 0; i < 30; ++i) { + ASSERT_EQ(120 + i, val_arr->Value(100 + i)); + } +} + +/// Test: bitmap hits partial pages of a row group, with page-filtered option disabled. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {120..149} hits pages 2-4 of RG1. +/// Expected: 100 rows (100-199) because page-filtered option is disabled, so page-level bitmap is +/// ignored. +TEST_F(PageFilteredRowGroupReaderTest, BitmapWithPageFilteredOptionDisabled) { + std::string file_name = dir_->Str() + "/bitmap_all_and_partial.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(120, 150); // pages 2-4 of RG1 + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, + 1024, false); + ASSERT_TRUE(result); + ASSERT_EQ(100, result->length()); + + // Verify: 100-199 + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 100; ++i) { + ASSERT_EQ(100 + i, val_arr->Value(i)); + } +} + +/// Test: bitmap + predicate both applied, bitmap hits all pages of some row groups. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {0..99} hits all of RG0. +/// Predicate: val >= 50. Page-level filtering on RG0: pages 5-9. +/// Expected: 50 rows (50-99). +TEST_F(PageFilteredRowGroupReaderTest, BitmapAllPagesWithPredicate) { + std::string file_name = dir_->Str() + "/bitmap_all_predicate.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 100); // hits all of RG0 + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(50)); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, predicate, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(50, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(50 + i, val_arr->Value(i)); + } +} + +/// Test: bitmap + predicate both applied, bitmap hits partial pages of a row group. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {30..59} hits pages 3-5 of RG0 (rows 30-59). +/// Predicate: val >= 40. Page-level filtering further narrows to pages 4-5 (rows 40-59). +/// Expected: 20 rows (40-59). +TEST_F(PageFilteredRowGroupReaderTest, BitmapPartialPagesWithPredicate) { + std::string file_name = dir_->Str() + "/bitmap_partial_predicate.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(30, 60); // hits pages 3-5 of RG0 + + auto predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(40)); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, predicate, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(20, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 20; ++i) { + ASSERT_EQ(40 + i, val_arr->Value(i)); + } +} + +/// Test: bitmap + predicate both applied, bitmap hits all pages of some RG and +/// partial pages of another. +/// +/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: {0..99} (all of RG0) + {120..149} (pages 2-4 of RG1). +/// Predicate: val >= 50 AND val < 160. +/// RG0: all pages → page-filtered to val>=50 → rows 50-99 (50 rows) +/// RG1: pages 2-4 (120-149) → page-filtered to val>=50 AND val<160 → all match (30 rows) +/// Expected: 80 rows (50-99 + 120-149). +TEST_F(PageFilteredRowGroupReaderTest, BitmapMixedWithPredicate) { + std::string file_name = dir_->Str() + "/bitmap_mixed_predicate.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 100); // all of RG0 + bitmap.AddRange(120, 150); // pages 2-4 of RG1 + + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::And( + {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val", + FieldType::INT, Literal(50)), + PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT, + Literal(160))})); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, predicate, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(80, result->length()); + + // Verify: rows 50-99 + 120-149 + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(50 + i, val_arr->Value(i)); + } + for (int32_t i = 0; i < 30; ++i) { + ASSERT_EQ(120 + i, val_arr->Value(50 + i)); + } +} + +/// Test: read parquet with scattered bitmap +/// +/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: [20,30), [35, 40), [125, 126), [130, 131), [150, 200) +/// To test if unneeded row at the start and end of pages are filtered out. +/// Expected: 76 rows ([20, 40) + [125, 131) + [150, 200). +TEST_F(PageFilteredRowGroupReaderTest, ScatteredBitmapTest) { + std::string file_name = dir_->Str() + "/scattered_bitmap.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(20, 30); + bitmap.AddRange(35, 40); + bitmap.Add(125); + bitmap.Add(130); + bitmap.AddRange(150, 200); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); + ASSERT_TRUE(result); + ASSERT_EQ(76, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + for (int32_t i = 0; i < 20; ++i) { + ASSERT_EQ(20 + i, val_arr->Value(i)); + } + for (int32_t i = 0; i < 6; ++i) { + ASSERT_EQ(125 + i, val_arr->Value(20 + i)); + } + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(150 + i, val_arr->Value(26 + i)); + } +} } // namespace paimon::parquet::test diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 83ae593c..bbff55c1 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -47,6 +47,7 @@ #include "paimon/core/utils/nested_projection_utils.h" #include "paimon/format/parquet/parquet_field_id_converter.h" #include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_schema_util.h" #include "paimon/format/parquet/parquet_timestamp_converter.h" #include "paimon/format/parquet/predicate_converter.h" #include "paimon/reader/batch_reader.h" @@ -159,26 +160,35 @@ Status ParquetFileBatchReader::SetReadSchema( field_index_map[field->name()] = leaf_indices; } - std::vector row_groups = arrow::internal::Iota(reader_->GetNumberOfRowGroups()); + TargetRowGroups target_row_groups = + TargetRowGroup::MakeForAllRowGroups(reader_->GetAllRowGroupRanges()); + PAIMON_ASSIGN_OR_RAISE( + bool enable_page_index_filter, + OptionsUtils::GetValueFromMap(options_, PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, + DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER)); + if (predicate) { - PAIMON_ASSIGN_OR_RAISE(row_groups, - FilterRowGroupsByPredicate(predicate, file_schema, row_groups)); + PAIMON_ASSIGN_OR_RAISE( + target_row_groups, + FilterRowGroupsByPredicate(predicate, file_schema, target_row_groups)); } if (selection_bitmap) { - PAIMON_ASSIGN_OR_RAISE(row_groups, - FilterRowGroupsByBitmap(selection_bitmap.value(), row_groups)); + PAIMON_ASSIGN_OR_RAISE( + target_row_groups, + FilterRowGroupsByBitmap(selection_bitmap.value(), target_row_groups)); + // workaround: page index filter does not support nested fields for now, skip page index + // bitmap pushdown if there is any nested field in the schema + if (!has_nested_field && enable_page_index_filter) { + PAIMON_ASSIGN_OR_RAISE(target_row_groups, + FilterPagesByBitmap(selection_bitmap.value(), + target_row_groups, column_indices)); + } } // Apply page-level filtering after bitmap pruning so we don't read page index // pages for row groups that the bitmap already excluded. - // If no predicate is provided, skip page-level filtering, row_group_row_ranges will be - // empty - std::map row_group_row_ranges; - if (predicate && !row_groups.empty()) { - PAIMON_ASSIGN_OR_RAISE( - bool enable_page_index_filter, - OptionsUtils::GetValueFromMap(options_, PARQUET_READ_ENABLE_PAGE_INDEX_FILTER, - DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER)); - // walkaround: page index filter does not support nested fields for now, skip page index + // If no predicate is provided, skip page-level filtering + if (predicate && !target_row_groups.empty()) { + // workaround: page index filter does not support nested fields for now, skip page index // filter if there is any nested field in the schema if (enable_page_index_filter && !has_nested_field) { // Build column name to index map for page-level filtering. @@ -192,13 +202,9 @@ Status ParquetFileBatchReader::SetReadSchema( column_name_to_index[name] = indices[0]; } } - - std::pair, std::map> page_filter_result; PAIMON_ASSIGN_OR_RAISE( - page_filter_result, - FilterRowGroupsByPageIndex(predicate, column_name_to_index, row_groups)); - row_groups = std::move(page_filter_result.first); - row_group_row_ranges = std::move(page_filter_result.second); + target_row_groups, + FilterRowGroupsByPageIndex(predicate, column_name_to_index, target_row_groups)); } } @@ -206,22 +212,9 @@ Status ParquetFileBatchReader::SetReadSchema( metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_TOTAL, reader_->GetNumberOfRowGroups()); - metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER, row_groups.size()); - - // Build TargetRowGroup list with page-filter info in one shot. - std::vector target_row_groups; - for (int32_t rg_id : row_groups) { - auto it = row_group_row_ranges.find(rg_id); - if (it != row_group_row_ranges.end()) { - target_row_groups.emplace_back(/*rg_index=*/rg_id, /*is_partially_matched=*/true, - /*ranges=*/it->second); - } else { - target_row_groups.emplace_back( - /*rg_index=*/rg_id, /*is_partially_matched=*/false, /*ranges=*/ - RowRanges(Range(0, reader_->GetAllRowGroupRanges()[rg_id].second - - reader_->GetAllRowGroupRanges()[rg_id].first - 1))); - } - } + metrics_->SetCounter(ParquetMetrics::READ_ROW_GROUPS_AFTER_FILTER, + target_row_groups.size()); + PAIMON_RETURN_NOT_OK(UpdateAllTargetRowRanges(target_row_groups)); PAIMON_RETURN_NOT_OK(reader_->PrepareForReadingLazy(target_row_groups, column_indices)); } @@ -229,9 +222,9 @@ Status ParquetFileBatchReader::SetReadSchema( return Status::OK(); } -Result> ParquetFileBatchReader::FilterRowGroupsByPredicate( +Result ParquetFileBatchReader::FilterRowGroupsByPredicate( const std::shared_ptr& predicate, const std::shared_ptr file_schema, - const std::vector& src_row_groups) const { + const TargetRowGroups& src_row_groups) const { if (!predicate) { return Status::Invalid("cannot pushdown an empty predicate"); } @@ -254,58 +247,140 @@ Result> ParquetFileBatchReader::FilterRowGroupsByPredicate( std::shared_ptr file_fragment, parquet_file_format->MakeFragment( file_source, /*partition_expression=*/PredicateConverter::AlwaysTrue(), - /*physical_schema=*/nullptr, /*row_groups=*/src_row_groups)); + /*physical_schema=*/nullptr, + /*row_groups=*/TargetRowGroup::GetRowGroupIndices(src_row_groups))); PAIMON_RETURN_NOT_OK_FROM_ARROW( file_fragment->EnsureCompleteMetadata(reader_->GetFileReader())); PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow::dataset::FragmentVector target_fragments, file_fragment->SplitByRowGroup(bind_expr)); - std::vector target_row_groups; + TargetRowGroups target_row_groups; target_row_groups.reserve(src_row_groups.size()); for (const auto& fragment : target_fragments) { auto parquet_fragment = dynamic_cast(fragment.get()); if (!parquet_fragment) { return Status::Invalid("cannot cast to ParquetFileFragment in ParquetFileBatchReader"); } - target_row_groups.insert(target_row_groups.end(), parquet_fragment->row_groups().begin(), - parquet_fragment->row_groups().end()); + for (auto rg_index : parquet_fragment->row_groups()) { + for (const auto& row_group : src_row_groups) { + if (row_group.GetRowGroupIndex() == rg_index) { + target_row_groups.emplace_back(row_group); + break; + } + } + } } return target_row_groups; } -Result> ParquetFileBatchReader::FilterRowGroupsByBitmap( - const RoaringBitmap32& bitmap, const std::vector& src_row_groups) const { +Result ParquetFileBatchReader::FilterRowGroupsByBitmap( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups) const { if (bitmap.IsEmpty()) { return Status::Invalid("cannot push down an empty bitmap to ParquetFileBatchReader"); } + const auto& all_row_group_ranges = reader_->GetAllRowGroupRanges(); - // filter row groups by row range - std::vector target_row_groups; - for (const auto& row_group_idx : src_row_groups) { + + TargetRowGroups target_row_groups; + for (const auto& row_group : src_row_groups) { + int32_t row_group_idx = row_group.GetRowGroupIndex(); if (static_cast(row_group_idx) >= all_row_group_ranges.size()) { return Status::Invalid( fmt::format("src row group {} not in row group meta", row_group_idx)); } + // half open interval [start_row_idx, end_row_idx) const auto& [start_row_idx, end_row_idx] = all_row_group_ranges[row_group_idx]; - if (bitmap.ContainsAny(start_row_idx, end_row_idx)) { - target_row_groups.push_back(row_group_idx); + if (!bitmap.ContainsAny(start_row_idx, end_row_idx)) { + continue; } + target_row_groups.emplace_back(row_group); } return target_row_groups; } +Result ParquetFileBatchReader::FilterPagesByBitmap( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups, + const std::vector& column_indices) const { + auto page_index_reader = reader_->GetPageIndexReader(); + if (!page_index_reader) { + return src_row_groups; + } + + TargetRowGroups target_row_groups; + target_row_groups.reserve(src_row_groups.size()); + for (const auto& row_group : src_row_groups) { + target_row_groups.emplace_back( + FilterRowGroupPagesByBitmap(bitmap, row_group, column_indices, page_index_reader)); + } + return target_row_groups; +} + +TargetRowGroup ParquetFileBatchReader::FilterRowGroupPagesByBitmap( + const RoaringBitmap32& bitmap, const TargetRowGroup& row_group, + const std::vector& column_indices, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader) const { + int32_t row_group_idx = row_group.GetRowGroupIndex(); + auto rg_page_index_reader = page_index_reader->RowGroup(row_group_idx); + if (!rg_page_index_reader) { + return row_group; + } + + const auto& all_row_group_ranges = reader_->GetAllRowGroupRanges(); + uint64_t rg_start_row = all_row_group_ranges[row_group_idx].first; + uint64_t rg_row_count = all_row_group_ranges[row_group_idx].second - rg_start_row; + + RowRanges row_ranges = row_group.GetRowRanges(); + for (int32_t col_index : column_indices) { + auto offset_index = rg_page_index_reader->GetOffsetIndex(col_index); + if (!offset_index) { + continue; + } + auto page_ranges = ComputeColumnPageRanges(bitmap, offset_index->page_locations(), + rg_start_row, rg_row_count); + row_ranges = RowRanges::Intersection(row_ranges, page_ranges); + } + if (row_ranges.RowCount() == static_cast(rg_row_count)) { + return row_group; + } else { + return TargetRowGroup(row_group_idx, true, std::move(row_ranges)); + } +} + +RowRanges ParquetFileBatchReader::ComputeColumnPageRanges( + const RoaringBitmap32& bitmap, const std::vector<::parquet::PageLocation>& page_locations, + uint64_t rg_start_row, uint64_t rg_row_count) { + RowRanges page_row_ranges; + for (size_t page_idx = 0; page_idx < page_locations.size(); ++page_idx) { + // half open interval [first_row, last_row) + auto first_row = page_locations[page_idx].first_row_index; + auto last_row = page_idx + 1 < page_locations.size() + ? page_locations[page_idx + 1].first_row_index + : rg_row_count; + + if (!bitmap.ContainsAny(rg_start_row + first_row, rg_start_row + last_row)) { + continue; + } + // closed interval [range_start_row, range_end_row] + auto range_start_row = bitmap.NextValue(rg_start_row + first_row); + auto range_end_row = bitmap.PreviousValue(rg_start_row + last_row); + if (!range_start_row.has_value() || !range_end_row.has_value()) { + continue; + } + page_row_ranges.Add( + Range(range_start_row.value() - rg_start_row, range_end_row.value() - rg_start_row)); + } + return page_row_ranges; +} + // Uses page-level column index statistics to filter row groups and store per-row-group // RowRanges for true page-level skipping. A row group is excluded if ALL its pages are // determined to not match the predicate. For partially matched row groups, RowRanges // are stored for page-level filtering during reading. -Result, std::map>> -ParquetFileBatchReader::FilterRowGroupsByPageIndex( +Result ParquetFileBatchReader::FilterRowGroupsByPageIndex( const std::shared_ptr& predicate, const std::map& column_name_to_index, - const std::vector& src_row_groups) { - std::map rg_row_ranges; - + const TargetRowGroups& src_row_groups) const { if (!predicate) { - return std::make_pair(src_row_groups, rg_row_ranges); + return src_row_groups; } auto page_index_reader = reader_->GetPageIndexReader(); @@ -313,35 +388,41 @@ ParquetFileBatchReader::FilterRowGroupsByPageIndex( PAIMON_LOG_DEBUG(logger_, "Page index not available in file, skipping page-level filtering (%s)", PARQUET_WRITE_ENABLE_PAGE_INDEX); - return std::make_pair(src_row_groups, rg_row_ranges); + return src_row_groups; } auto file_metadata = reader_->GetFileReader()->parquet_reader()->metadata(); - std::vector target_row_groups; - target_row_groups.reserve(src_row_groups.size()); + TargetRowGroups target_row_groups; - for (int32_t row_group_idx : src_row_groups) { + for (const auto& row_group : src_row_groups) { + int32_t row_group_idx = row_group.GetRowGroupIndex(); auto result = reader_->CalculateFilteredRowRanges(row_group_idx, predicate, column_name_to_index); if (!result.ok()) { - target_row_groups.push_back(row_group_idx); + target_row_groups.emplace_back(row_group); continue; } const auto& row_ranges = result.value(); if (!row_ranges.IsEmpty()) { - target_row_groups.push_back(row_group_idx); - int64_t rg_row_count = file_metadata->RowGroup(row_group_idx)->num_rows(); - if (row_ranges.RowCount() < rg_row_count) { - rg_row_ranges[row_group_idx] = row_ranges; + auto intersection = row_group.IsPartiallyMatched() + ? RowRanges::Intersection(row_group.GetRowRanges(), row_ranges) + : row_ranges; + if (intersection.IsEmpty()) { + continue; + } + if (intersection.RowCount() < rg_row_count) { + target_row_groups.emplace_back(row_group_idx, true, intersection); + } else { + target_row_groups.emplace_back(row_group); } } } - return std::make_pair(std::move(target_row_groups), std::move(rg_row_ranges)); + return target_row_groups; } Result ParquetFileBatchReader::NextBatch() { @@ -539,10 +620,10 @@ Status ParquetFileBatchReader::UpdateAllTargetRowRanges( auto all_row_group_ranges = reader_->GetAllRowGroupRanges(); RowRanges all_ranges; for (const auto& target_row_group : target_row_groups) { - for (const auto& range : target_row_group.row_ranges.GetRanges()) { - all_ranges.Add( - Range(range.from + all_row_group_ranges[target_row_group.row_group_index].first, - range.to + all_row_group_ranges[target_row_group.row_group_index].first)); + auto row_group_idx = target_row_group.GetRowGroupIndex(); + for (const auto& range : target_row_group.GetRowRanges().GetRanges()) { + all_ranges.Add(Range(all_row_group_ranges[row_group_idx].first + range.from, + all_row_group_ranges[row_group_idx].first + range.to)); } } all_row_ranges_ = std::move(all_ranges); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 4ba2aca3..7f6cab1f 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -41,6 +41,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/parquet/file_reader_wrapper.h" #include "paimon/format/parquet/row_ranges.h" +#include "paimon/format/parquet/target_row_group.h" #include "paimon/logging.h" #include "paimon/reader/prefetch_file_batch_reader.h" #include "paimon/result.h" @@ -193,20 +194,39 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { Status UpdateAllTargetRowRanges(const std::vector& target_row_groups); // precondition: predicate supposed not be empty - Result> FilterRowGroupsByPredicate( + Result FilterRowGroupsByPredicate( const std::shared_ptr& predicate, const std::shared_ptr file_schema, - const std::vector& src_row_groups) const; - - Result> FilterRowGroupsByBitmap( - const RoaringBitmap32& bitmap, const std::vector& src_row_groups) const; + const TargetRowGroups& src_row_groups) const; + + Result FilterRowGroupsByBitmap(const RoaringBitmap32& bitmap, + const TargetRowGroups& src_row_groups) const; + + Result FilterPagesByBitmap(const RoaringBitmap32& bitmap, + const TargetRowGroups& src_row_groups, + const std::vector& column_indices) const; + + // Apply page-level bitmap filtering to a single row group across all + // requested columns. Intersects the row group's existing ranges with the + // per-column page ranges derived from the bitmap. + TargetRowGroup FilterRowGroupPagesByBitmap( + const RoaringBitmap32& bitmap, const TargetRowGroup& row_group, + const std::vector& column_indices, + const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader) const; + + // Compute the set of row ranges within a single column's pages that + // overlap with the given bitmap. For each page, the bitmap is queried to + // find the first/last matching row in each page, used to trim the page head/tail + static RowRanges ComputeColumnPageRanges( + const RoaringBitmap32& bitmap, const std::vector<::parquet::PageLocation>& page_locations, + uint64_t rg_start_row, uint64_t rg_row_count); // Apply page-level filtering using column index. // Returns (filtered row groups, per-row-group RowRanges for partial matches). - Result, std::map>> - FilterRowGroupsByPageIndex(const std::shared_ptr& predicate, - const std::map& column_name_to_index, - const std::vector& src_row_groups); + Result FilterRowGroupsByPageIndex( + const std::shared_ptr& predicate, + const std::map& column_name_to_index, + const TargetRowGroups& src_row_groups) const; Status GenerateRowMapping(int64_t batch_length); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index 417c2aea..9ca4157f 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -197,12 +197,15 @@ class ParquetFileBatchReaderTest : public ::testing::Test, std::unique_ptr PrepareParquetFileBatchReader( const std::string& file_name, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, - const std::optional& selection_bitmap, int32_t batch_size) const { + const std::optional& selection_bitmap, int32_t batch_size, + bool enable_page_level_filter = false) const { EXPECT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_name)); auto length = fs_->GetFileStatus(file_name).value()->GetLen(); auto in_stream = std::make_unique(std::move(input_stream), pool_, length); - std::map options = {}; + std::map options; + options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = + enable_page_level_filter ? "true" : "false"; return PrepareParquetFileBatchReader(std::move(in_stream), options, read_schema, predicate, selection_bitmap, batch_size); } @@ -717,7 +720,7 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) { } } -TEST_F(ParquetFileBatchReaderTest, TestBitmapPushDownWithMultiRowGroups) { +TEST_F(ParquetFileBatchReaderTest, TestBitmapRowGroupPushDownWithMultiRowGroups) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; auto arrow_type = arrow::struct_(fields); auto src_array = std::dynamic_pointer_cast( @@ -755,8 +758,47 @@ TEST_F(ParquetFileBatchReaderTest, TestBitmapPushDownWithMultiRowGroups) { auto expected_array = arrow::ChunkedArray::Make({src_array->Slice(0, 6)}).ValueOrDie(); ASSERT_TRUE(result_array->Equals(expected_array)) << result_array->ToString(); } +TEST_F(ParquetFileBatchReaderTest, TestBitmapPagePushDownWithMultiRowGroups) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto arrow_type = arrow::struct_(fields); + auto src_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow_type, R"([ + [0], + [1], + [2], + [3], + [4], + [5], + [6], + [7], + [8], + [9], + [10], + [11] + ])") + .ValueOrDie()); + auto src_schema = arrow::schema(fields); + std::optional bitmap = RoaringBitmap32::From({3, 5}); + // data in file rowGroup0:[0, 1, 2, 3, 4, 5] | rowGroup1:[6, 7, 8, 9, 10, 11] + + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/12, + /*enable_dictionary=*/true, + /*max_row_group_length=*/6); + + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, arrow_schema, /*predicate=*/nullptr, bitmap, + /*batch_size=*/12, /*enable_page_level_filter=*/true); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result_array, + paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get())); + + auto expected_array = arrow::ChunkedArray(src_array->Slice(3, 3)); + ASSERT_TRUE(result_array->Equals(expected_array)) << result_array->ToString(); +} -TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPushDown) { +TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapRowGroupPushDown) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; auto arrow_type = arrow::struct_(fields); arrow::StructBuilder struct_builder(arrow_type, arrow::default_memory_pool(), @@ -813,6 +855,64 @@ TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPushDown) { ASSERT_FALSE(result_array); } } +TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPagePushDown) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32())}; + auto arrow_type = arrow::struct_(fields); + arrow::StructBuilder struct_builder(arrow_type, arrow::default_memory_pool(), + {std::make_shared()}); + auto int_builder = static_cast(struct_builder.field_builder(0)); + int32_t length = 1024; + for (int32_t i = 0; i < length; ++i) { + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(int_builder->Append(i).ok()); + } + // data file: + // rowGroup0: [0, 256) + // rowGroup1: [256, 512) + // rowGroup2: [512, 768) + // rowGroup3: [768, 1024) + std::shared_ptr src_array; + ASSERT_TRUE(struct_builder.Finish(&src_array).ok()); + auto src_schema = arrow::schema(fields); + auto arrow_schema = arrow::schema(fields); + WriteArray(file_path_, src_array, arrow_schema, /*write_batch_size=*/1024, + /*enable_dictionary=*/true, + /*max_row_group_length=*/256); + { + // simple case + std::optional bitmap = RoaringBitmap32::From({100, 400, 600}); + ASSERT_OK_AND_ASSIGN( + auto predicate, + PredicateBuilder::Or( + {PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"f0", FieldType::INT, + Literal(255)), + PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(600))})); + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, arrow_schema, predicate, bitmap, + /*batch_size=*/length, /*enable_page_level_filter=*/true); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result_array, + paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get())); + + auto expected_array = + arrow::ChunkedArray::Make({src_array->Slice(100, 1), src_array->Slice(600, 1)}) + .ValueOrDie(); + ASSERT_TRUE(result_array->Equals(expected_array)) << result_array->ToString(); + } + { + // test all data has been filtered out with predicate and bitmap pushdown + std::optional bitmap = RoaringBitmap32::From({100, 400, 600}); + auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", + FieldType::INT, Literal(800)); + auto parquet_batch_reader = PrepareParquetFileBatchReader( + file_path_, arrow_schema, predicate, bitmap, /*batch_size=*/length); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr result_array, + paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get())); + ASSERT_FALSE(result_array); + } +} TEST_F(ParquetFileBatchReaderTest, TestReadNoField) { // if only read partition fields, format reader will set empty read schema @@ -1043,7 +1143,8 @@ TEST_F(ParquetFileBatchReaderTest, TestRowMappingSimple) { FieldType::INT, Literal(5), Literal(6))})); auto parquet_batch_reader = PrepareParquetFileBatchReader( - file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/2); + file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/2, + /*enable_page_level_filter=*/true); ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN( @@ -1109,7 +1210,8 @@ TEST_F(ParquetFileBatchReaderTest, TestRowMappingFullyAndPartially) { FieldType::INT, Literal(8))})); auto parquet_batch_reader = PrepareParquetFileBatchReader( - file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3); + file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3, + /*enable_page_level_filter=*/true); ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN( @@ -1143,7 +1245,8 @@ TEST_F(ParquetFileBatchReaderTest, TestRowMappingSetReadSchemaTwice) { FieldType::INT, Literal(6), Literal(7))})); auto parquet_batch_reader = PrepareParquetFileBatchReader( - file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3); + file_path_, arrow_schema, /*predicate=*/predicate, std::nullopt, /*batch_size=*/3, + /*enable_page_level_filter=*/true); ASSERT_NOK(parquet_batch_reader->GetPreviousBatchFileRowId(0)); ASSERT_OK_AND_ASSIGN( diff --git a/src/paimon/format/parquet/row_ranges.h b/src/paimon/format/parquet/row_ranges.h index 9a8547fe..51fea472 100644 --- a/src/paimon/format/parquet/row_ranges.h +++ b/src/paimon/format/parquet/row_ranges.h @@ -108,21 +108,4 @@ class RowRanges { private: std::vector ranges_; }; - -struct TargetRowGroup { - int32_t row_group_index{-1}; - bool is_partially_matched{false}; - - RowRanges row_ranges; - // Whether this row group has been excluded by ApplyReadRanges. - // When true, this row group is logically skipped during iteration - // but retained so that a subsequent wider ApplyReadRanges can restore it. - bool excluded_by_read_range{false}; - - TargetRowGroup() = default; - TargetRowGroup(int32_t rg_index, bool is_partially_matched, RowRanges ranges) - : row_group_index(rg_index), - is_partially_matched(is_partially_matched), - row_ranges(std::move(ranges)) {} -}; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/target_row_group.h b/src/paimon/format/parquet/target_row_group.h new file mode 100644 index 00000000..b86e29df --- /dev/null +++ b/src/paimon/format/parquet/target_row_group.h @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/format/parquet/row_ranges.h" + +namespace paimon::parquet { +class TargetRowGroup; +using TargetRowGroups = std::vector; +class TargetRowGroup { + public: + TargetRowGroup(int32_t rg_index, bool is_partially_matched, RowRanges ranges) + : row_group_index_(rg_index), + is_partially_matched_(is_partially_matched), + row_ranges_(std::move(ranges)) {} + + TargetRowGroup(const TargetRowGroup& other) = default; + TargetRowGroup& operator=(const TargetRowGroup& other) = default; + + bool IsExcludedByReadRange() const { + return excluded_by_read_range_; + } + + void SetExcludedByReadRange(bool excluded) { + excluded_by_read_range_ = excluded; + } + + int32_t GetRowGroupIndex() const { + return row_group_index_; + } + + bool IsPartiallyMatched() const { + return is_partially_matched_; + } + + const RowRanges& GetRowRanges() const { + return row_ranges_; + } + + // Create a list of TargetRowGroups for serial (non-filtered) reading. + // + // Each element in 'ranges' is a (start, end) pair describing the row + // range of a single row group. The vector index 'i' is used as the row + // group index, so the caller must ensure that 'ranges' is ordered to + // match the physical row-group order in the file. + // + // For each valid range (start < end), a TargetRowGroup is created with: + // - row_group_index = i + // - is_partially_matched = false (the entire row group is read) + // - row_ranges = [0, end - start - 1] (local row indices covering the + // full group; converted from absolute offsets to 0-based local offsets) + // + // Ranges where start >= end are treated as empty and skipped. + static TargetRowGroups MakeForAllRowGroups( + const std::vector>& ranges) { + TargetRowGroups target_row_groups; + target_row_groups.reserve(ranges.size()); + for (size_t i = 0; i < ranges.size(); ++i) { + // Skip empty or invalid ranges. + if (ranges[i].first >= ranges[i].second) { + continue; + } + // Convert the absolute [start, end) pair into a 0-based local + // row range [0, row_count - 1] for this row group. + target_row_groups.emplace_back( + static_cast(i), false, + RowRanges(Range(0, ranges[i].second - ranges[i].first - 1))); + } + return target_row_groups; + } + + static std::vector GetRowGroupIndices(const TargetRowGroups& target_row_groups) { + std::vector indices; + indices.reserve(target_row_groups.size()); + for (const auto& rg : target_row_groups) { + indices.push_back(rg.GetRowGroupIndex()); + } + return indices; + } + + private: + int32_t row_group_index_{-1}; + bool is_partially_matched_{false}; + // Local row ranges + RowRanges row_ranges_; + // Whether this row group has been excluded by ApplyReadRanges. + // When true, this row group is logically skipped during iteration + // but retained so that a subsequent wider ApplyReadRanges can restore it. + bool excluded_by_read_range_{false}; +}; + +} // namespace paimon::parquet From 9a97f4fcceaacb7a18bd72b05df32df94dc74f86 Mon Sep 17 00:00:00 2001 From: lszskye <57179283+lszskye@users.noreply.github.com> Date: Fri, 10 Jul 2026 01:09:23 -0700 Subject: [PATCH 088/138] chore: adjust set read_ranges_freshed_ in SetReadRanges --- src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp index 973ac888..3e8737a3 100644 --- a/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp +++ b/src/paimon/common/reader/prefetch_file_batch_reader_impl.cpp @@ -189,8 +189,6 @@ Status PrefetchFileBatchReaderImpl::RefreshReadRangesAfterCleanUp() { need_prefetch_ = need_prefetch; PAIMON_RETURN_NOT_OK(SetReadRanges(FilterReadRanges(read_ranges, selection_bitmap_))); - read_ranges_freshed_ = true; - return Status::OK(); } @@ -237,6 +235,7 @@ Status PrefetchFileBatchReaderImpl::SetReadRanges( for (auto& read_ranges : read_ranges_in_group_) { read_ranges.push_back(eof_range); } + read_ranges_freshed_ = true; return Status::OK(); } From 6c11487cb7a6e47bd6f829760599f83091a6a945 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:33:06 +0800 Subject: [PATCH 089/138] docs: add coding agent guidelines --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- AGENTS.md | 126 +++++++++++++++++++++++++++++++ CLAUDE.md | 1 + CONTRIBUTING.md | 23 ++++++ 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 AGENTS.md create mode 120000 CLAUDE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1320da73..50d87f7f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,4 +1,4 @@ - + ### Purpose diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..9511a133 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,126 @@ + + +# AGENTS.md + +This file provides repository-specific instructions for coding agents working on Paimon C++. +For contributor-facing setup and the complete coding conventions, also read [`CONTRIBUTING.md`](CONTRIBUTING.md) and [`docs/code-style.md`](docs/code-style.md). + +## Scope + +These instructions apply to the entire repository. More deeply nested `AGENTS.md` files, if added later, may provide additional or more specific instructions for their directory trees. + +Keep the requested scope exact. Do not include unrelated refactors, formatting changes, API redesigns, dependency updates, or generated files in a focused change. + +## Repository Layout + +- `include/paimon/`: public C++ API headers. +- `src/paimon/`: core implementation and most unit tests. +- `test/inte/`: end-to-end integration tests. +- `benchmark/`: benchmarks and benchmark-specific tests. +- `examples/`: example programs. +- `docs/` and `apidoc/`: user documentation and API documentation. +- `cmake_modules/` and `build_support/`: CMake helpers and build infrastructure. +- `ci/`: scripts used by continuous integration. +- `test/test_data/`: checked-in test fixtures. +- `third_party/`: third-party sources and patches. +- `build/`, `build-release/`, and `output/`: generated or local build output. + +Do not edit `third_party/`, checked-in fixtures, generated output, or Git LFS objects unless the task explicitly requires it. Never add files from local build directories to a change. + +## Working Rules + +1. Inspect the current implementation, nearby tests, and relevant CMake target before editing. +2. Search for an existing helper or established pattern before adding a new abstraction. +3. Preserve user changes and untracked local files. Do not discard, overwrite, or reformat unrelated work. +4. Prefer the smallest change that fully implements or fixes the requested behavior. +5. Add or update a focused regression test for behavior changes when practical. +6. Do not claim a build or test passed unless the corresponding command completed successfully. +7. Do not commit, push, amend commits, or create a pull request unless explicitly requested. + +When changing a public API, check the declaration under `include/paimon/`, its implementation, symbol visibility, documentation, callers, and tests together. + +## C++ Requirements + +The full rules are in [`docs/code-style.md`](docs/code-style.md). In particular: + +- Use C++17; do not introduce C++20 or later features. +- Use `Status` and `Result` for fallible operations. Do not use exceptions for production error propagation. +- Propagate errors with the project macros, including `PAIMON_RETURN_NOT_OK` and `PAIMON_ASSIGN_OR_RAISE`. +- Use an explicit type, not `auto`, as the declaration in `PAIMON_ASSIGN_OR_RAISE` and `PAIMON_ASSIGN_OR_RAISE_FROM_ARROW`. +- Prefer `std::unique_ptr` for sole ownership and use `std::shared_ptr` only for genuine shared ownership. +- Use `static Create()` plus a private constructor when object initialization can fail. +- Mark new public API symbols with `PAIMON_EXPORT`. +- Reuse helpers under `src/paimon/common/utils/` instead of duplicating utility code. +- Follow `.clang-format`; do not manually restyle unrelated code. +- Add the repository's Apache 2.0 license header to every new source or documentation file. + +## Build and Test + +Use an existing configured build directory when it is compatible with the change. To configure a new debug build with tests: + +```bash +cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DPAIMON_BUILD_TESTS=ON +``` + +Start with the narrowest relevant validation: + +```bash +cmake --build build --target -j "$(nproc)" +./build/debug/ --gtest_filter='' +``` + +Then broaden validation in proportion to the change: + +```bash +# All unit tests +cmake --build build --target unittest -j "$(nproc)" + +# Formatting, lint, and repository checks for changed files +pre-commit run --files +git diff --check +``` + +For changes to build configuration, public APIs, shared infrastructure, or cross-module behavior, run the relevant wider test suite. The CI-equivalent build entry point is `ci/scripts/build_paimon.sh`; it may rebuild all dependencies and take substantially longer than a focused local target. + +If a required check cannot be run because of missing dependencies, unsupported hardware, or time constraints, report exactly what was and was not run. + +## Testing Conventions + +- Use GoogleTest and name test files `*_test.cpp`. +- Place unit tests next to the corresponding implementation unless an existing target establishes another location. +- Extend an existing test target when appropriate instead of creating a new executable for one small test. +- Test externally observable behavior and failure cases; avoid coupling tests to incidental implementation details. +- Use existing test utilities and temporary-directory helpers. Do not write tests that depend on developer-specific absolute paths. +- Keep fixtures deterministic and small. Do not rewrite existing fixture data unless the task explicitly calls for it. +- Use `ASSERT_*` when later assertions depend on the condition succeeding. + +## Delivery + +Before handing off a change: + +1. Review `git diff` for accidental or unrelated edits. +2. Run `git diff --check`. +3. Run the narrowest relevant build and test, plus any wider checks justified by the risk. +4. Summarize the changed behavior and list the exact validation commands that ran. +5. Call out skipped validation, remaining risks, or follow-up work explicitly. +6. When explicitly asked to commit, follow the Conventional Commits requirements in [`CONTRIBUTING.md`](CONTRIBUTING.md#commit-messages-and-pull-request-titles). +7. When explicitly asked to open a pull request, follow [`CONTRIBUTING.md`](CONTRIBUTING.md#commit-messages-and-pull-request-titles), use a Conventional Commits title, and complete every applicable section of [`.github/PULL_REQUEST_TEMPLATE.md`](.github/PULL_REQUEST_TEMPLATE.md). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 086f4a65..cabeca59 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,27 @@ If you find a bug or want to request a feature, please open an [issue](https://g --- +## Commit Messages and Pull Request Titles + +Use the [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages and pull request titles: + +```text +(): +``` + +Examples: + +```text +feat(parquet): support page-level bitmap filtering +fix: handle non-contiguous row ranges +test(executor): add shutdown coverage +docs: update the build instructions +``` + +Choose a type and optional scope that accurately describe the change. Keep the description concise and write it in the imperative mood. A pull request title should summarize the complete change and use the same format. + +--- + ## Submitting Pull Requests 1. **Fork** the repository and create a feature branch from `main`. @@ -37,6 +58,8 @@ If you find a bug or want to request a feature, please open an [issue](https://g 4. Ensure all checks pass. 5. Open a pull request against `main`. Fill in the [PR template](.github/PULL_REQUEST_TEMPLATE.md). +When addressing review feedback or adding follow-up changes to an open pull request, prefer a separate commit instead of amending and force-pushing existing commits. This makes the incremental diff easier for reviewers to inspect. Rewrite existing commits only when a maintainer explicitly requests it. + ### PR Checklist Before submitting, please verify: From 9e8c2933fa8432aa98162e077c91cf8fafabe189 Mon Sep 17 00:00:00 2001 From: Xiaoguang Zhu Date: Fri, 10 Jul 2026 23:49:47 +0800 Subject: [PATCH 090/138] fix: compile with correct flags with AppleClang --- cmake_modules/BuildUtils.cmake | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index 7377fb99..bf93c77f 100644 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -94,7 +94,8 @@ function(add_paimon_lib LIB_NAME) add_library(${LIB_NAME}_objlib OBJECT ${ARG_SOURCES}) target_link_libraries(${LIB_NAME}_objlib PRIVATE "$") - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL + "Clang") target_compile_options(${LIB_NAME}_objlib PRIVATE -Wno-global-constructors) endif() # Necessary to make static linking into other shared libraries work properly @@ -340,7 +341,8 @@ function(add_test_case REL_TEST_NAME) add_dependencies(${TEST_NAME} ${ARG_EXTRA_DEPENDENCIES}) endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL + "Clang") target_compile_options(${TEST_NAME} PRIVATE -Wno-global-constructors) endif() target_compile_options(${TEST_NAME} PRIVATE -fno-access-control) @@ -472,7 +474,8 @@ function(add_benchmark_case REL_BENCHMARK_NAME) target_include_directories(${BENCHMARK_NAME} SYSTEM PUBLIC ${ARG_EXTRA_INCLUDES}) endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL + "Clang") target_compile_options(${BENCHMARK_NAME} PRIVATE -Wno-global-constructors) endif() target_compile_options(${BENCHMARK_NAME} PRIVATE -fno-access-control) From 3965d83abb651f96ea6c85943fdcadd82da4c103 Mon Sep 17 00:00:00 2001 From: dalingmeng <49717204+dalingmeng@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:22:06 +0800 Subject: [PATCH 091/138] feat(shared-shredding): Expose shared-shredding utilities --- .../map_shared_shredding_schema_utils.h | 98 +++++++ src/paimon/CMakeLists.txt | 2 + .../map_shared_shredding_batch_converter.cpp | 5 +- .../map_shared_shredding_file_reader_test.cpp | 70 +++-- .../map_shared_shredding_schema_utils.cpp | 99 +++++++ ...map_shared_shredding_schema_utils_test.cpp | 269 ++++++++++++++++++ .../shredding/map_shared_shredding_utils.cpp | 39 ++- .../shredding/map_shared_shredding_utils.h | 19 +- .../map_shared_shredding_utils_test.cpp | 20 +- .../data/shredding/map_shredding_defs.h | 28 +- .../core/append/append_only_writer_test.cpp | 63 ++-- .../core/mergetree/merge_tree_writer_test.cpp | 16 +- .../append_only_file_store_write_test.cpp | 17 +- .../key_value_file_store_write_test.cpp | 5 +- .../postpone/postpone_bucket_writer_test.cpp | 4 +- 15 files changed, 616 insertions(+), 138 deletions(-) create mode 100644 include/paimon/data/shredding/map_shared_shredding_schema_utils.h create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp diff --git a/include/paimon/data/shredding/map_shared_shredding_schema_utils.h b/include/paimon/data/shredding/map_shared_shredding_schema_utils.h new file mode 100644 index 00000000..393b625e --- /dev/null +++ b/include/paimon/data/shredding/map_shared_shredding_schema_utils.h @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/arrow/abi.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// Parsed file-level meta for one shared-shredding MAP column. +struct PAIMON_EXPORT MapSharedShreddingFieldMeta { + /// field_name -> field_id + std::map name_to_id; + /// field_id -> ordered physical column indices + std::map> field_to_columns; + /// Set of field_ids that ever spilled into __overflow + std::set overflow_field_set; + /// Number of physical columns K in this file + int32_t num_columns = 0; + /// Maximum row width observed in this file + int32_t max_row_width = 0; + + bool operator==(const MapSharedShreddingFieldMeta& other) const { + if (this == &other) { + return true; + } + return name_to_id == other.name_to_id && field_to_columns == other.field_to_columns && + overflow_field_set == other.overflow_field_set && num_columns == other.num_columns && + max_row_width == other.max_row_width; + } +}; + +class PAIMON_EXPORT MapSharedShreddingSchemaUtils { + public: + MapSharedShreddingSchemaUtils() = delete; + ~MapSharedShreddingSchemaUtils() = delete; + + /// Converts a logical schema to a physical schema by replacing shredding MAP columns + /// with their physical Struct representation. + /// @param logical_schema The original Arrow C schema with MAP columns. + /// Ownership of schema resources is transferred to this method. + /// @param field_to_num_columns Map from field name to its physical column count K. + /// Each shredding column can have its own width. + /// @return The exported Arrow C schema for file writing. + static Result> LogicalToPhysicalSchema( + std::unique_ptr<::ArrowSchema> logical_schema, + const std::map& field_to_num_columns); + + /// Attaches shared-shredding metadata to fields in a physical schema. + /// @param physical_schema The Arrow C physical schema whose fields should receive metadata. + /// Ownership of schema resources is transferred to this method. + /// @param field_name_to_meta Map from physical field name to its shared-shredding metadata. + /// Existing shared-shredding metadata keys on matching fields are overwritten. + /// @param compression Compression codec name for field_dict serialization. + /// @return A new Arrow C schema with shared-shredding metadata attached to matching fields. + static Result> AttachMetadataToSchema( + std::unique_ptr<::ArrowSchema> physical_schema, + const std::map& field_name_to_meta, + const std::string& compression); + + /// Extracts shared-shredding metadata from a named field in a physical schema. + /// @param physical_schema The Arrow C physical schema that contains the target field. + /// Ownership of schema resources is transferred to this method. + /// @param field_name The physical field name whose metadata should be extracted. + /// @param compression Compression codec name for field_dict deserialization. + /// @return Parsed shared-shredding metadata for the field. + static Result ExtractMetadataFromField( + std::unique_ptr<::ArrowSchema> physical_schema, const std::string& field_name, + const std::string& compression); +}; + +} // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 4a1e8bc4..73f11f83 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -141,6 +141,7 @@ set(PAIMON_COMMON_SRCS common/utils/crc32c.cpp common/utils/decimal_utils.cpp common/data/shredding/map_shared_shredding_utils.cpp + common/data/shredding/map_shared_shredding_schema_utils.cpp common/data/shredding/map_shared_shredding_context.cpp common/data/shredding/map_shared_shredding_batch_converter.cpp common/data/shredding/map_shared_shredding_column_allocator.cpp @@ -549,6 +550,7 @@ if(PAIMON_BUILD_TESTS) common/utils/threadsafe_queue_test.cpp common/utils/generic_lru_cache_test.cpp common/data/shredding/map_shared_shredding_utils_test.cpp + common/data/shredding/map_shared_shredding_schema_utils_test.cpp common/data/shredding/map_shared_shredding_batch_converter_test.cpp common/data/shredding/lru_map_shared_shredding_column_allocator_test.cpp common/data/shredding/plain_map_shared_shredding_column_allocator_test.cpp diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp index 73960fe1..6869a5b2 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp @@ -68,8 +68,9 @@ Result> MapSharedShreddingBatc const std::shared_ptr& context, const CoreOptions& options, const std::shared_ptr& pool) { std::map field_to_num_columns = context->ComputeNextK(); - std::shared_ptr physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, field_to_num_columns); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, field_to_num_columns)); std::vector contexts; std::vector shredding_field_names; contexts.reserve(field_to_num_columns.size()); diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp index 2f0b3269..9fbdd9ba 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp @@ -66,17 +66,18 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test { return meta; } - std::shared_ptr PhysicalSchemaWithMetadata() const { + Result> PhysicalSchemaWithMetadata() const { return PhysicalSchemaWithMetadata(TagsMeta()); } - std::shared_ptr PhysicalSchemaWithMetadata( + Result> PhysicalSchemaWithMetadata( const MapSharedShreddingFieldMeta& meta) const { std::map field_to_num_columns = {{"tags", 2}}; - auto physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema_, field_to_num_columns); + PAIMON_ASSIGN_OR_RAISE(auto physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema_, field_to_num_columns)); auto metadata = std::make_shared(); - EXPECT_OK(MapSharedShreddingUtils::SerializeMetadata( + PAIMON_RETURN_NOT_OK(MapSharedShreddingUtils::SerializeMetadata( meta, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get())); arrow::FieldVector fields = physical_schema->fields(); @@ -84,8 +85,9 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test { return arrow::schema(std::move(fields)); } - std::shared_ptr PhysicalArray() const { - std::shared_ptr physical_schema = PhysicalSchemaWithMetadata(); + Result> PhysicalArray() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_schema, + PhysicalSchemaWithMetadata()); std::string json = R"([ [1, [[0, 1], 10, 20, null]], [2, [[2, 0], 30, 40, null]], @@ -143,15 +145,15 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test { std::move(reader), std::move(shared_shredding_name_to_context), pool_); } - std::unique_ptr CreateReader( + Result> CreateReader( std::shared_ptr physical_array = nullptr, std::shared_ptr physical_schema = nullptr, const std::optional& selected_keys = std::nullopt) const { if (!physical_schema) { - physical_schema = PhysicalSchemaWithMetadata(); + PAIMON_ASSIGN_OR_RAISE(physical_schema, PhysicalSchemaWithMetadata()); } if (!physical_array) { - physical_array = PhysicalArray(); + PAIMON_ASSIGN_OR_RAISE(physical_array, PhysicalArray()); } auto mock_reader = std::make_unique( physical_array, arrow::struct_(physical_schema->fields()), /*read_batch_size=*/10); @@ -246,7 +248,7 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test { }; TEST_F(MapSharedShreddingFileReaderTest, TestGetFileSchemaReturnsLogicalMapSchema) { - auto reader = CreateReader(); + ASSERT_OK_AND_ASSIGN(auto reader, CreateReader()); ASSERT_OK_AND_ASSIGN(auto c_schema, reader->GetFileSchema()); auto schema = arrow::ImportSchema(c_schema.get()).ValueOrDie(); @@ -259,8 +261,9 @@ TEST_F(MapSharedShreddingFileReaderTest, TestGetFileSchemaReturnsLogicalMapSchem } TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithoutOverflow) { - auto reader = CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, - /*selected_keys=*/"b"); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"b")); auto read_schema = ExportSchema(ReadSchema("b")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -280,8 +283,9 @@ TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithoutOverflow } TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithOverflow) { - auto reader = CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, - /*selected_keys=*/"a,c"); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"a,c")); auto read_schema = ExportSchema(ReadSchema("a,c")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -301,8 +305,9 @@ TEST_F(MapSharedShreddingFileReaderTest, TestAllExistSelectedKeysWithOverflow) { } TEST_F(MapSharedShreddingFileReaderTest, TestPartialExistSelectedKeys) { - auto reader = CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, - /*selected_keys=*/"a,c,missing"); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"a,c,missing")); auto read_schema = ExportSchema(ReadSchema("a,c,missing")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -323,7 +328,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestPartialExistSelectedKeys) { } TEST_F(MapSharedShreddingFileReaderTest, TestMissingSelectedKeysReadsWholeMap) { - auto reader = CreateReader(); + ASSERT_OK_AND_ASSIGN(auto reader, CreateReader()); auto read_schema = ExportSchema(ReadSchema(std::nullopt)); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -348,7 +353,7 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSpecialSelectedKeys) { meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {0}}, {3, {1}}}; meta.num_columns = 2; meta.max_row_width = 2; - auto physical_schema = PhysicalSchemaWithMetadata(meta); + ASSERT_OK_AND_ASSIGN(auto physical_schema, PhysicalSchemaWithMetadata(meta)); std::string json = R"([ [1, [[0, 1], 10, 20, null]], [2, [[2, 3], 30, 40, null]], @@ -359,7 +364,8 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSpecialSelectedKeys) { .ValueOrDie(); auto assert_read = [&](const std::string& selected_keys, const std::string& expected_json) { - auto reader = CreateReader(physical_array, physical_schema, selected_keys); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(physical_array, physical_schema, selected_keys)); auto read_schema = ExportSchema(ReadSchema(selected_keys)); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -395,8 +401,9 @@ TEST_F(MapSharedShreddingFileReaderTest, TestSpecialSelectedKeys) { } TEST_F(MapSharedShreddingFileReaderTest, TestUnknownSelectedKeyReturnsEmptyMap) { - auto reader = CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, - /*selected_keys=*/"missing"); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(/*physical_array=*/nullptr, /*physical_schema=*/nullptr, + /*selected_keys=*/"missing")); auto read_schema = ExportSchema(ReadSchema("missing")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -417,14 +424,15 @@ TEST_F(MapSharedShreddingFileReaderTest, TestUnknownSelectedKeyReturnsEmptyMap) } TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) { - auto physical_schema = PhysicalSchemaWithMetadata(); + ASSERT_OK_AND_ASSIGN(auto physical_schema, PhysicalSchemaWithMetadata()); std::string json = R"([ [1, [null, 10, null, null]] ])"; auto physical_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) .ValueOrDie(); - auto reader = CreateReader(physical_array, physical_schema, /*selected_keys=*/"a"); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(physical_array, physical_schema, /*selected_keys=*/"a")); auto read_schema = ExportSchema(ReadSchema("a")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -433,14 +441,15 @@ TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingField) { } TEST_F(MapSharedShreddingFileReaderTest, TestInvalidNullFieldMappingFieldElement) { - auto physical_schema = PhysicalSchemaWithMetadata(); + ASSERT_OK_AND_ASSIGN(auto physical_schema, PhysicalSchemaWithMetadata()); std::string json = R"([ [1, [[0, null], 10, null, null]] ])"; auto physical_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(physical_schema->fields()), json) .ValueOrDie(); - auto reader = CreateReader(physical_array, physical_schema, /*selected_keys=*/"b"); + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(physical_array, physical_schema, /*selected_keys=*/"b")); auto read_schema = ExportSchema(ReadSchema("b")); ASSERT_OK(reader->SetReadSchema(read_schema.get(), /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt)); @@ -461,8 +470,8 @@ TEST_F(MapSharedShreddingFileReaderTest, TestListValue) { meta.max_row_width = 3; std::map field_to_num_columns = {{"tags", 2}}; - auto physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, field_to_num_columns); + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, field_to_num_columns)); auto metadata = std::make_shared(); ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata( meta, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get())); @@ -478,8 +487,9 @@ TEST_F(MapSharedShreddingFileReaderTest, TestListValue) { [4, [[1, 0], [8], [9, 10], [[2, [null]]]]] ])") .ValueOrDie(); - auto reader = CreateReader(physical_array, physical_schema, - /*selected_keys=*/"a,c"); // NOLINT(whitespace/comma) + ASSERT_OK_AND_ASSIGN(auto reader, + CreateReader(physical_array, physical_schema, + /*selected_keys=*/"a,c")); // NOLINT(whitespace/comma) auto read_metadata = std::make_shared(); read_metadata->Append("paimon.map.selected-keys", "a,c"); diff --git a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp new file mode 100644 index 00000000..11a8f2f0 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/data/shredding/map_shared_shredding_schema_utils.h" + +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "fmt/format.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +Result> MapSharedShreddingSchemaUtils::LogicalToPhysicalSchema( + std::unique_ptr<::ArrowSchema> logical_schema, + const std::map& field_to_num_columns) { + if (!logical_schema) { + return Status::Invalid("logical schema is null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_logical_schema, + arrow::ImportSchema(logical_schema.get())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(arrow_logical_schema, + field_to_num_columns)); + auto c_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*physical_schema, c_schema.get())); + return c_schema; +} + +Result> MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::unique_ptr<::ArrowSchema> physical_schema, + const std::map& field_name_to_meta, + const std::string& compression) { + if (!physical_schema) { + return Status::Invalid("physical schema is null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_physical_schema, + arrow::ImportSchema(physical_schema.get())); + + arrow::FieldVector updated_fields = arrow_physical_schema->fields(); + for (const auto& [field_name, field_meta] : field_name_to_meta) { + int32_t field_index = arrow_physical_schema->GetFieldIndex(field_name); + if (field_index < 0) { + return Status::Invalid(fmt::format( + "Shared-shredding field '{}' not found in physical schema.", field_name)); + } + + const auto& field = arrow_physical_schema->field(field_index); + auto metadata = field->metadata() ? field->metadata()->Copy() + : std::make_shared(); + PAIMON_RETURN_NOT_OK( + MapSharedShreddingUtils::SerializeMetadata(field_meta, compression, metadata.get())); + updated_fields[field_index] = field->WithMetadata(metadata); + } + + auto updated_schema = + arrow::schema(std::move(updated_fields), arrow_physical_schema->metadata()); + auto c_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*updated_schema, c_schema.get())); + return c_schema; +} + +Result MapSharedShreddingSchemaUtils::ExtractMetadataFromField( + std::unique_ptr<::ArrowSchema> physical_schema, const std::string& field_name, + const std::string& compression) { + if (!physical_schema) { + return Status::Invalid("physical schema is null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_physical_schema, + arrow::ImportSchema(physical_schema.get())); + const auto& field = arrow_physical_schema->GetFieldByName(field_name); + if (!field) { + return Status::Invalid( + fmt::format("Shared-shredding field '{}' not found in physical schema.", field_name)); + } + + auto metadata = + field->metadata() ? field->metadata()->Copy() : std::shared_ptr(); + return MapSharedShreddingUtils::DeserializeMetadata(metadata, compression); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp new file mode 100644 index 00000000..e6e5cf63 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/data/shredding/map_shared_shredding_schema_utils.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "arrow/util/key_value_metadata.h" +#include "gtest/gtest.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaBasic) { + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}, {"region", 1}}; + tags_meta.field_to_columns = {{0, {0}}, {1, {1}}}; + tags_meta.num_columns = 2; + tags_meta.max_row_width = 2; + + auto id_metadata = std::make_shared(); + id_metadata->Append("paimon.field.id", "1"); + auto schema_metadata = std::make_shared(); + schema_metadata->Append("schema.key", "schema.value"); + auto schema = arrow::schema( + {arrow::field("id", arrow::int32(), true, id_metadata), + arrow::field("tags", arrow::struct_({arrow::field("__field_mapping", + arrow::list(arrow::int32()), true), + arrow::field("__col_0", arrow::utf8(), true), + arrow::field("__col_1", arrow::utf8(), true)}))}, + schema_metadata); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto c_updated_schema, + MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::move(c_schema), {{"tags", tags_meta}}, "none")); + ASSERT_TRUE(c_updated_schema); + ASSERT_TRUE(c_updated_schema->release); + auto updated_schema = arrow::ImportSchema(c_updated_schema.get()).ValueOrDie(); + + ASSERT_TRUE(updated_schema->metadata()->Equals(*schema_metadata)); + ASSERT_TRUE(updated_schema->field(0)->metadata()->Equals(*id_metadata)); + + auto tags_metadata = updated_schema->GetFieldByName("tags")->metadata()->Copy(); + ASSERT_TRUE(MapSharedShreddingUtils::HasShreddingMetadata(tags_metadata)); + ASSERT_OK_AND_ASSIGN(auto deserialized, + MapSharedShreddingUtils::DeserializeMetadata(tags_metadata, "none")); + ASSERT_EQ(deserialized, tags_meta); +} + +TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaInvalidInput) { + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::unique_ptr<::ArrowSchema>(), {}, "none"), + "physical schema is null"); + + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}}; + tags_meta.field_to_columns = {{0, {0}}}; + tags_meta.num_columns = 1; + tags_meta.max_row_width = 1; + + auto schema = arrow::schema({arrow::field("id", arrow::int32())}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::move(c_schema), {{"tags", tags_meta}}, "none"), + "Shared-shredding field 'tags' not found in physical schema."); +} + +TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaPreservesExistingFieldMetadata) { + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}}; + tags_meta.field_to_columns = {{0, {0}}}; + tags_meta.num_columns = 1; + tags_meta.max_row_width = 1; + + auto tags_metadata = std::make_shared(); + tags_metadata->Append("paimon.field.id", "7"); + tags_metadata->Append("description", "original tags field"); + auto schema = arrow::schema({arrow::field( + "tags", + arrow::struct_({arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", arrow::utf8(), true)}), + true, tags_metadata)}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto c_updated_schema, + MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::move(c_schema), {{"tags", tags_meta}}, "none")); + ASSERT_TRUE(c_updated_schema); + ASSERT_TRUE(c_updated_schema->release); + auto updated_schema = arrow::ImportSchema(c_updated_schema.get()).ValueOrDie(); + + auto updated_metadata = updated_schema->field(0)->metadata()->Copy(); + ASSERT_EQ(updated_metadata->value(updated_metadata->FindKey("paimon.field.id")), "7"); + ASSERT_EQ(updated_metadata->value(updated_metadata->FindKey("description")), + "original tags field"); + ASSERT_TRUE(MapSharedShreddingUtils::HasShreddingMetadata(updated_metadata)); +} + +TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaOverwritesExistingShreddingMetadata) { + MapSharedShreddingFieldMeta old_meta; + old_meta.name_to_id = {{"old", 0}}; + old_meta.field_to_columns = {{0, {0}}}; + old_meta.num_columns = 1; + old_meta.max_row_width = 1; + + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}, {"region", 1}}; + tags_meta.field_to_columns = {{0, {0}}, {1, {1}}}; + tags_meta.overflow_field_set = {1}; + tags_meta.num_columns = 2; + tags_meta.max_row_width = 2; + + auto tags_metadata = std::make_shared(); + tags_metadata->Append("paimon.field.id", "7"); + ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata(old_meta, "none", tags_metadata.get())); + auto schema = arrow::schema({arrow::field( + "tags", + arrow::struct_({arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", arrow::utf8(), true), + arrow::field("__col_1", arrow::utf8(), true)}), + true, tags_metadata)}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto c_updated_schema, + MapSharedShreddingSchemaUtils::AttachMetadataToSchema( + std::move(c_schema), {{"tags", tags_meta}}, "none")); + auto updated_schema = arrow::ImportSchema(c_updated_schema.get()).ValueOrDie(); + auto updated_metadata = updated_schema->field(0)->metadata()->Copy(); + + ASSERT_EQ(updated_metadata->value(updated_metadata->FindKey("paimon.field.id")), "7"); + int32_t storage_layout_key_count = 0; + for (const auto& key : updated_metadata->keys()) { + if (key == MapShreddingDefine::kStorageLayout) { + ++storage_layout_key_count; + } + } + ASSERT_EQ(storage_layout_key_count, 1); + ASSERT_OK_AND_ASSIGN(auto deserialized, + MapSharedShreddingUtils::DeserializeMetadata(updated_metadata, "none")); + ASSERT_EQ(deserialized, tags_meta); +} + +TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromField) { + MapSharedShreddingFieldMeta tags_meta; + tags_meta.name_to_id = {{"host", 0}, {"region", 1}}; + tags_meta.field_to_columns = {{0, {0}}, {1, {1}}}; + tags_meta.num_columns = 2; + tags_meta.max_row_width = 2; + + auto metadata = std::make_shared(); + ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata(tags_meta, "none", metadata.get())); + auto field = arrow::field( + "tags", + arrow::struct_({arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", arrow::utf8(), true), + arrow::field("__col_1", arrow::utf8(), true)}), + true, metadata); + auto schema = arrow::schema({arrow::field("id", arrow::int32()), field}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto parsed_meta, MapSharedShreddingSchemaUtils::ExtractMetadataFromField( + std::move(c_schema), "tags", "none")); + ASSERT_EQ(parsed_meta, tags_meta); +} + +TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromFieldNoShreddingMetadata) { + auto schema = + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("tags", arrow::utf8())}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::ExtractMetadataFromField(std::move(c_schema), + "tags", "none"), + "metadata is null or storage layout is not shared-shredding"); +} + +TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromFieldInvalidInput) { + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::ExtractMetadataFromField( + std::unique_ptr<::ArrowSchema>(), "tags", "none"), + "physical schema is null"); + + auto schema = arrow::schema({arrow::field("id", arrow::int32())}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::ExtractMetadataFromField(std::move(c_schema), + "tags", "none"), + "Shared-shredding field 'tags' not found in physical schema."); +} + +TEST(MapSharedShreddingSchemaUtilsTest, LogicalToPhysicalSchemaInvalidInput) { + std::map field_to_num_columns = {{"tags", 2}}; + + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::LogicalToPhysicalSchema( + std::unique_ptr<::ArrowSchema>(), field_to_num_columns), + "logical schema is null"); + + auto schema = + arrow::schema({arrow::field("ts", arrow::int64()), arrow::field("tags", arrow::int32())}); + + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + + ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::LogicalToPhysicalSchema( + std::move(c_schema), field_to_num_columns), + "Field 'tags' is expected to be MAP type"); +} + +TEST(MapSharedShreddingSchemaUtilsTest, LogicalToPhysicalSchemaNestedListValue) { + // MAP, b: array, c: array>> + auto nested_value = arrow::struct_({arrow::field("a", arrow::list(arrow::int32())), + arrow::field("b", arrow::list(arrow::int32())), + arrow::field("c", arrow::list(arrow::int32()))}); + auto map_type = arrow::map(arrow::utf8(), nested_value); + auto schema = + arrow::schema({arrow::field("ts", arrow::int64()), arrow::field("data", map_type)}); + + std::map field_to_num_columns = {{"data", 3}}; + auto c_schema = std::make_unique<::ArrowSchema>(); + ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto c_physical_schema, + MapSharedShreddingSchemaUtils::LogicalToPhysicalSchema( + std::move(c_schema), field_to_num_columns)); + ASSERT_TRUE(c_physical_schema); + ASSERT_TRUE(c_physical_schema->release); + auto physical_schema = arrow::ImportSchema(c_physical_schema.get()).ValueOrDie(); + + auto expected_struct = arrow::struct_({ + arrow::field("__field_mapping", arrow::list(arrow::int32()), true), + arrow::field("__col_0", nested_value, true), + arrow::field("__col_1", nested_value, true), + arrow::field("__col_2", nested_value, true), + arrow::field("__overflow", arrow::map(arrow::int32(), nested_value), true), + }); + auto expected_schema = arrow::schema( + {arrow::field("ts", arrow::int64()), arrow::field("data", expected_struct, true)}); + ASSERT_TRUE(physical_schema->Equals(expected_schema)); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp index 5a04f342..5c6f421a 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp @@ -30,6 +30,7 @@ #include "paimon/common/compression/block_decompressor.h" #include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" #include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/options/map_storage_layout.h" @@ -128,7 +129,7 @@ std::shared_ptr MapSharedShreddingUtils::InnerBuildSpecificPhys return arrow::struct_(std::move(struct_fields)); } -std::shared_ptr MapSharedShreddingUtils::LogicalToPhysicalSchema( +Result> MapSharedShreddingUtils::LogicalToPhysicalSchema( const std::shared_ptr& logical_schema, const std::map& field_to_num_columns) { arrow::FieldVector physical_fields; @@ -138,6 +139,11 @@ std::shared_ptr MapSharedShreddingUtils::LogicalToPhysicalSchema( const auto& field = logical_schema->field(i); auto it = field_to_num_columns.find(field->name()); if (it != field_to_num_columns.end()) { + if (field->type()->id() != arrow::Type::MAP) { + return Status::Invalid( + fmt::format("Field '{}' is expected to be MAP type, but got '{}'.", + field->name(), field->type()->name())); + } auto map_type = std::static_pointer_cast(field->type()); auto value_type = map_type->item_type(); bool value_nullable = map_type->item_field()->nullable(); @@ -355,23 +361,28 @@ Result> DeserializeOverflowSet(const std::string& json_str) { Status MapSharedShreddingUtils::SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta, const std::string& compression, arrow::KeyValueMetadata* metadata) { - metadata->Append(MapShreddingDefine::kStorageLayout, - MapShreddingDefine::kStorageLayoutSharedShredding); - metadata->Append(MapSharedShreddingDefine::kVersion, - std::to_string(MapSharedShreddingDefine::kCurrentVersion)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set( + MapShreddingDefine::kStorageLayout, MapShreddingDefine::kStorageLayoutSharedShredding)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + metadata->Set(MapSharedShreddingDefine::kVersion, + std::to_string(MapSharedShreddingDefine::kCurrentVersion))); std::string field_dict_json = SerializeFieldDict(field_meta); - metadata->Append(MapSharedShreddingDefine::kFieldDictOriginalSize, - std::to_string(field_dict_json.size())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set(MapSharedShreddingDefine::kFieldDictOriginalSize, + std::to_string(field_dict_json.size()))); PAIMON_ASSIGN_OR_RAISE(std::string compressed_dict, CompressString(field_dict_json, compression)); - metadata->Append(MapSharedShreddingDefine::kFieldDict, std::move(compressed_dict)); - - metadata->Append(MapSharedShreddingDefine::kFieldColumns, SerializeFieldColumns(field_meta)); - metadata->Append(MapSharedShreddingDefine::kOverflowSet, SerializeOverflowSet(field_meta)); - metadata->Append(MapSharedShreddingDefine::kNumColumns, std::to_string(field_meta.num_columns)); - metadata->Append(MapSharedShreddingDefine::kMaxRowWidth, - std::to_string(field_meta.max_row_width)); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + metadata->Set(MapSharedShreddingDefine::kFieldDict, std::move(compressed_dict))); + + PAIMON_RETURN_NOT_OK_FROM_ARROW( + metadata->Set(MapSharedShreddingDefine::kFieldColumns, SerializeFieldColumns(field_meta))); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + metadata->Set(MapSharedShreddingDefine::kOverflowSet, SerializeOverflowSet(field_meta))); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set(MapSharedShreddingDefine::kNumColumns, + std::to_string(field_meta.num_columns))); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set(MapSharedShreddingDefine::kMaxRowWidth, + std::to_string(field_meta.max_row_width))); return Status::OK(); } diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.h b/src/paimon/common/data/shredding/map_shared_shredding_utils.h index eef19098..97d57b80 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.h @@ -29,6 +29,7 @@ #include "arrow/type.h" #include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/data/shredding/map_shared_shredding_schema_utils.h" #include "paimon/result.h" #include "paimon/status.h" @@ -71,7 +72,7 @@ class MapSharedShreddingUtils { /// @param field_to_num_columns Map from field name to its physical column count K. /// Each shredding column can have its own width. /// @return The physical schema for file writing. - static std::shared_ptr LogicalToPhysicalSchema( + static Result> LogicalToPhysicalSchema( const std::shared_ptr& logical_schema, const std::map& field_to_num_columns); @@ -86,6 +87,14 @@ class MapSharedShreddingUtils { // ---- Metadata serialization ---- + /// Serializes shredding metadata and appends entries to an existing KeyValueMetadata. + /// @param field_meta The field-level shredding metadata to serialize. + /// @param compression Compression codec name for field_dict compression. + /// @param[out] metadata The KeyValueMetadata to append entries to. + static Status SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta, + const std::string& compression, + arrow::KeyValueMetadata* metadata); + /// Deserializes shredding metadata from file footer KeyValueMetadata (per field). /// @param metadata The KeyValueMetadata from file footer. /// @param compression Compression codec name. @@ -142,14 +151,6 @@ class MapSharedShreddingUtils { static Result> BuildColumnToNumColumns( const std::vector& shredding_field_names, const CoreOptions& options); - /// Serializes shredding metadata and appends entries to an existing KeyValueMetadata. - /// @param field_meta The field-level shredding metadata to serialize. - /// @param compression Compression codec name for field_dict compression. - /// @param[out] metadata The KeyValueMetadata to append entries to. - static Status SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta, - const std::string& compression, - arrow::KeyValueMetadata* metadata); - /// Builds the physical Arrow type for one shredding MAP column. /// @param value_type The value type of the original MAP. /// @param num_columns Number of physical columns K. diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp index a7dddfa4..fe85708e 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp +++ b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp @@ -91,8 +91,8 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaBasic) { }); std::map field_to_num_columns = {{"tags", 4}}; - auto physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, field_to_num_columns); + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + schema, field_to_num_columns)); // Build expected schema for comparison auto expected_struct = arrow::struct_({ @@ -119,8 +119,8 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNestedValue) { auto schema = arrow::schema({arrow::field("data", map_type)}); std::map field_to_num_columns = {{"data", 2}}; - auto physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, field_to_num_columns); + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + schema, field_to_num_columns)); auto expected_struct = arrow::struct_({ arrow::field("__field_mapping", arrow::list(arrow::int32()), true), @@ -138,7 +138,8 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNullable) { auto schema_nullable = arrow::schema({arrow::field("m", nullable_map)}); std::map col_map = {{"m", 2}}; - auto physical = MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_nullable, col_map); + ASSERT_OK_AND_ASSIGN( + auto physical, MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_nullable, col_map)); auto struct_type = physical->field(0)->type(); ASSERT_TRUE(struct_type->field(0)->nullable()); ASSERT_TRUE(struct_type->field(1)->nullable()); @@ -148,7 +149,8 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNullable) { auto non_nullable_map = arrow::map(arrow::utf8(), arrow::field("item", arrow::int64(), false)); auto schema_non_nullable = arrow::schema({arrow::field("m", non_nullable_map)}); - auto physical2 = MapSharedShreddingUtils::LogicalToPhysicalSchema(schema_non_nullable, col_map); + ASSERT_OK_AND_ASSIGN(auto physical2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + schema_non_nullable, col_map)); auto struct_type2 = physical2->field(0)->type(); ASSERT_FALSE(struct_type2->field(1)->nullable()); ASSERT_FALSE(struct_type2->field(2)->nullable()); @@ -162,7 +164,8 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaPreservesFieldMetadata) auto schema = arrow::schema({arrow::field("m", map_type, false, metadata)}); std::map col_map = {{"m", 2}}; - auto physical_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, col_map); + ASSERT_OK_AND_ASSIGN(auto physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, col_map)); ASSERT_FALSE(physical_schema->field(0)->nullable()); ASSERT_TRUE(physical_schema->field(0)->metadata()->Equals(*metadata)); @@ -175,7 +178,8 @@ TEST(MapSharedShreddingUtilsTest, LogicalToPhysicalSchemaNoShreddingColumns) { }); std::map empty_map; - auto physical_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, empty_map); + ASSERT_OK_AND_ASSIGN(auto physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(schema, empty_map)); ASSERT_TRUE(physical_schema->Equals(schema)); } diff --git a/src/paimon/common/data/shredding/map_shredding_defs.h b/src/paimon/common/data/shredding/map_shredding_defs.h index ecc993d1..1ada7a7c 100644 --- a/src/paimon/common/data/shredding/map_shredding_defs.h +++ b/src/paimon/common/data/shredding/map_shredding_defs.h @@ -20,10 +20,9 @@ #pragma once #include -#include -#include #include -#include + +#include "paimon/data/shredding/map_shared_shredding_schema_utils.h" namespace paimon { @@ -74,27 +73,4 @@ struct MapSharedShreddingDefine { } }; -/// Parsed file-level meta for one shared-shredding MAP column. -struct MapSharedShreddingFieldMeta { - /// field_name -> field_id - std::map name_to_id; - /// field_id -> set of physical column indices S - std::map> field_to_columns; - /// Set of field_ids that ever spilled into __overflow - std::set overflow_field_set; - /// Number of physical columns K in this file - int32_t num_columns = 0; - /// Maximum row width observed in this file - int32_t max_row_width = 0; - - bool operator==(const MapSharedShreddingFieldMeta& other) const { - if (this == &other) { - return true; - } - return name_to_id == other.name_to_id && field_to_columns == other.field_to_columns && - overflow_field_set == other.overflow_field_set && num_columns == other.num_columns && - max_row_width == other.max_row_width; - } -}; - } // namespace paimon diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp index be53042e..0363286e 100644 --- a/src/paimon/core/append/append_only_writer_test.cpp +++ b/src/paimon/core/append/append_only_writer_test.cpp @@ -973,8 +973,9 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) { // Check shared-shredding map metadata: a=0, b=1, c=2; K=3, max_row_width=3, no overflow. std::map column_to_k = {{"tags", 3}}; - auto expected_physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k); + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); MapSharedShreddingFieldMeta expected_meta; expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; @@ -1036,8 +1037,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllEmptyFirstFile) { path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map first_file_k = {{"tags", 3}}; - auto first_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, first_file_k); + ASSERT_OK_AND_ASSIGN(auto first_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, first_file_k)); MapSharedShreddingFieldMeta empty_meta; empty_meta.num_columns = 3; empty_meta.max_row_width = 0; @@ -1092,8 +1093,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF path_factory->ToPath(null_inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map first_file_k = {{"tags", 3}}; - auto first_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, first_file_k); + ASSERT_OK_AND_ASSIGN(auto first_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, first_file_k)); MapSharedShreddingFieldMeta empty_meta; empty_meta.num_columns = 3; empty_meta.max_row_width = 0; @@ -1124,8 +1125,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF // Previous file observed max_row_width=0, but the next file must still keep at least one // physical value column so shared-shredding never produces a K=0 schema. std::map second_file_k = {{"tags", 1}}; - auto second_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, second_file_k); + ASSERT_OK_AND_ASSIGN(auto second_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, second_file_k)); empty_meta.num_columns = 1; CheckShreddingFileSchema(empty_file_path, format, second_schema, /*field_index=*/1, empty_meta, options.GetFileCompression()); @@ -1218,8 +1219,9 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithOverflow) { path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k = {{"tags", 2}}; - auto expected_physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k); + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); std::string compression = options.GetFileCompression(); // Verify metadata: a=0,b=1,c=2,d=3,e=4,f=5; K=2, max_row_width=4 @@ -1287,8 +1289,9 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithLruPlacemen path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k = {{"tags", 3}}; - auto expected_physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k); + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k)); MapSharedShreddingFieldMeta expected_meta; expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}}; @@ -1352,8 +1355,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil // File 1 should have K=10 (first file uses K_max). std::map column_to_k_file1 = {{"tags", 10}}; - auto phys_schema1 = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k_file1); + ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k_file1)); // Verify file1 physical schema has 10 columns. auto struct_type1 = std::static_pointer_cast(phys_schema1->field(1)->type()); ASSERT_EQ(12, struct_type1->num_fields()); // mapping + 10 cols + overflow @@ -1381,8 +1384,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil // File 2 should have K=3 (adapted from file1's max_row_width=3). std::map column_to_k_file2 = {{"tags", 3}}; - auto phys_schema2 = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k_file2); + ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k_file2)); auto struct_type2 = std::static_pointer_cast(phys_schema2->field(1)->type()); ASSERT_EQ(5, struct_type2->num_fields()); // mapping + 3 cols + overflow @@ -1420,8 +1423,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil // File 3 should have K=5 (window max grew from file2's max_row_width=5). std::map column_to_k_file3 = {{"tags", 5}}; - auto phys_schema3 = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k_file3); + ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k_file3)); auto struct_type3 = std::static_pointer_cast(phys_schema3->field(1)->type()); ASSERT_EQ(7, struct_type3->num_fields()); // mapping + 5 cols + overflow @@ -1484,8 +1487,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapUsesInitialContextFo path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k = {{"tags", 2}}; - auto physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, column_to_k); + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, column_to_k)); MapSharedShreddingFieldMeta expected_meta; expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; expected_meta.field_to_columns = {{0, {0}}, {1, {1}}}; @@ -1554,8 +1557,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA // Verify file1: tags K=8, attrs K=4 (first file uses K_max). std::map col_to_k_file1 = {{"tags", 8}, {"attrs", 4}}; - auto phys_schema1 = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, col_to_k_file1); + ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k_file1)); MapSharedShreddingFieldMeta meta1_tags; meta1_tags.name_to_id = {{"a", 0}, {"b", 1}}; @@ -1586,8 +1589,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA path_factory->ToPath(inc2.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map col_to_k_file2 = {{"tags", 2}, {"attrs", 1}}; - auto phys_schema2 = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, col_to_k_file2); + ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k_file2)); MapSharedShreddingFieldMeta meta2_tags; meta2_tags.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}}; @@ -1620,8 +1623,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA path_factory->ToPath(inc3.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map col_to_k_file3 = {{"tags", 3}, {"attrs", 3}}; - auto phys_schema3 = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, col_to_k_file3); + ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k_file3)); MapSharedShreddingFieldMeta meta3_tags; meta3_tags.name_to_id = {{"f", 0}, {"g", 1}}; @@ -1703,7 +1706,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapDataFileMetaInfo) { // Verify the written file has correct shared-shredding map content. std::string file_path = path_factory->ToPath(actual_meta->file_name); std::map col_to_k = {{"tags", 3}}; - auto phys_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, col_to_k); + ASSERT_OK_AND_ASSIGN(auto phys_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, col_to_k)); auto physical_type = arrow::struct_(phys_schema->fields()); std::shared_ptr expected_array; @@ -1790,8 +1794,9 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapWithBlobSeparation) arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())), }); std::map col_to_k = {{"tags", 3}}; - auto expected_physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(main_logical_schema, col_to_k); + ASSERT_OK_AND_ASSIGN( + auto expected_physical_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(main_logical_schema, col_to_k)); MapSharedShreddingFieldMeta expected_meta; expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp index d2a5cc51..ddcba958 100644 --- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp @@ -450,8 +450,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) { options.GetFileSystem()->GetFileStatus(expected_data_file_path)); std::map column_to_k = {{"tags", 3}}; - auto physical_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, column_to_k); + ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k)); auto physical_type = arrow::struct_(physical_schema->fields()); std::shared_ptr expected_array; @@ -543,8 +543,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) path_factory->ToPath(commit_increment1.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k_file1 = {{"tags", 8}, {"attrs", 4}}; - auto physical_schema1 = - MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, column_to_k_file1); + ASSERT_OK_AND_ASSIGN(auto physical_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k_file1)); MapSharedShreddingFieldMeta tags_meta1; tags_meta1.name_to_id = {{"a", 0}, {"b", 1}}; tags_meta1.field_to_columns = {{0, {0}}, {1, {1}}}; @@ -571,8 +571,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) path_factory->ToPath(commit_increment2.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k_file2 = {{"tags", 2}, {"attrs", 1}}; - auto physical_schema2 = - MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, column_to_k_file2); + ASSERT_OK_AND_ASSIGN(auto physical_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k_file2)); MapSharedShreddingFieldMeta tags_meta2; tags_meta2.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}}; tags_meta2.field_to_columns = {{0, {0}}, {1, {1}}}; @@ -601,8 +601,8 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) path_factory->ToPath(commit_increment3.GetNewFilesIncrement().NewFiles()[0]->file_name); std::map column_to_k_file3 = {{"tags", 3}, {"attrs", 3}}; - auto physical_schema3 = - MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, column_to_k_file3); + ASSERT_OK_AND_ASSIGN(auto physical_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema( + write_schema, column_to_k_file3)); MapSharedShreddingFieldMeta tags_meta3; tags_meta3.name_to_id = {{"f", 0}, {"g", 1}}; tags_meta3.field_to_columns = {{0, {0}}, {1, {1}}}; diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp index 1a0eb92c..4f330985 100644 --- a/src/paimon/core/operation/append_only_file_store_write_test.cpp +++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp @@ -324,8 +324,9 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNex ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); - auto expected_second_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 2}}); + ASSERT_OK_AND_ASSIGN( + auto expected_second_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 2}})); ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false)); ASSERT_EQ(2, second_meta.num_columns); ASSERT_EQ(3, second_meta.max_row_width); @@ -368,8 +369,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreIgnoresAvroFileWi ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), shredding_options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1); - auto expected_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 10}}); + ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 10}})); ASSERT_TRUE(second_file_schema->Equals(*expected_schema, /*check_metadata=*/false)); ASSERT_EQ(10, second_meta.num_columns); ASSERT_EQ(3, second_meta.max_row_width); @@ -429,8 +430,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreMultipleMapColumn auto tags_meta = ShreddingMeta(full_file_schema, /*field_index=*/1); auto attrs_meta = ShreddingMeta(full_file_schema, /*field_index=*/2); - auto expected_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, {{"tags", 2}, {"attrs", 4}}); + ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 2}, {"attrs", 4}})); ASSERT_TRUE(full_file_schema->Equals(*expected_schema, /*check_metadata=*/false)); ASSERT_EQ(2, tags_meta.num_columns); ASSERT_EQ(3, tags_meta.max_row_width); @@ -481,8 +482,8 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreUsesDefaultForMis auto tags_meta = ShreddingMeta(full_file_schema, /*field_index=*/1); auto attrs_meta = ShreddingMeta(full_file_schema, /*field_index=*/2); - auto expected_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema( - logical_schema, {{"tags", 2}, {"attrs", 10}}); + ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + logical_schema, {{"tags", 2}, {"attrs", 10}})); ASSERT_TRUE(full_file_schema->Equals(*expected_schema, /*check_metadata=*/false)); ASSERT_EQ(2, tags_meta.num_columns); ASSERT_EQ(3, tags_meta.max_row_width); diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index f939fb46..9448a1fe 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -354,8 +354,9 @@ TEST_F(KeyValueFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNextW ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), options); auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/3); - auto expected_second_schema = - MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 2}}); + ASSERT_OK_AND_ASSIGN( + auto expected_second_schema, + MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 2}})); ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false)); ASSERT_EQ(2, second_meta.num_columns); ASSERT_EQ(3, second_meta.max_row_width); diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp index 641f2831..8b466a3e 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp @@ -368,8 +368,8 @@ TEST_F(PostponeBucketWriterTest, TestSharedShreddingMap) { arrow::FieldVector write_fields = {arrow::field("_SEQUENCE_NUMBER", arrow::int64()), arrow::field("_VALUE_KIND", arrow::int8())}; write_fields.insert(write_fields.end(), fields.begin(), fields.end()); - auto expected_schema = MapSharedShreddingUtils::LogicalToPhysicalSchema( - arrow::schema(write_fields), {{"tags", 3}}); + ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema( + arrow::schema(write_fields), {{"tags", 3}})); MapSharedShreddingFieldMeta expected_meta; expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}}; From 505cda0b460998a929696d1943cdd36dc8f6e840 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Mon, 13 Jul 2026 16:30:06 +0800 Subject: [PATCH 092/138] feat(blob view): support blob-view.resolve.enabled to preserve blob view references at read time --- include/paimon/defs.h | 4 + src/paimon/common/defs.cpp | 1 + src/paimon/core/core_options.cpp | 8 + src/paimon/core/core_options.h | 1 + src/paimon/core/core_options_test.cpp | 3 + .../operation/data_evolution_split_read.cpp | 5 + test/inte/blob_table_inte_test.cpp | 160 ++++++++++++++++++ 7 files changed, 182 insertions(+) diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 5d91b959..fb25e99a 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -412,6 +412,10 @@ struct PAIMON_EXPORT Options { /// serialized BlobViewStruct bytes inline in data files and resolve from upstream tables at /// read time. No default value. static const char BLOB_VIEW_FIELD[]; + /// "blob-view.resolve.enabled" - Whether to resolve blob-view-field values from upstream + /// tables at read time. Set to false to preserve serialized BlobViewStruct bytes when + /// forwarding blob view values to another blob-view table. Default value is "true". + static const char BLOB_VIEW_RESOLVE_ENABLED[]; /// "blob-view-upstream-warehouse" - Since the catalog capabilities are partially missing, when /// Blob View is enabled, cpp paimon cannot automatically obtain the upstream table warehouse /// path and requires manual configuration by the user. No default value. diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 4e68916f..19851c54 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -105,6 +105,7 @@ const char Options::BLOB_FIELD[] = "blob-field"; const char Options::BLOB_DESCRIPTOR_FIELD[] = "blob-descriptor-field"; const char Options::FALLBACK_BLOB_DESCRIPTOR_FIELD[] = "blob.stored-descriptor-fields"; const char Options::BLOB_VIEW_FIELD[] = "blob-view-field"; +const char Options::BLOB_VIEW_RESOLVE_ENABLED[] = "blob-view.resolve.enabled"; const char Options::BLOB_VIEW_UPSTREAM_WAREHOUSE[] = "blob-view-upstream-warehouse"; const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled"; const char Options::GLOBAL_INDEX_THREAD_NUM[] = "global-index.thread-num"; diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index dd21c147..a4ccb67c 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -444,6 +444,7 @@ struct CoreOptions::Impl { bool row_tracking_enabled = false; bool row_tracking_partition_group_on_commit = true; bool data_evolution_enabled = false; + bool blob_view_resolve_enabled = true; bool legacy_partition_name_enabled = true; bool global_index_enabled = true; std::optional global_index_thread_num; @@ -565,6 +566,9 @@ struct CoreOptions::Impl { // Parse blob-view-upstream-warehouse - warehouse path for configured blob view fields PAIMON_RETURN_NOT_OK( parser.Parse(Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, &blob_view_upstream_warehouse)); + // Parse blob-view.resolve.enabled - whether to resolve blob view fields at read time + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::BLOB_VIEW_RESOLVE_ENABLED, &blob_view_resolve_enabled)); return Status::OK(); } @@ -1491,6 +1495,10 @@ std::optional CoreOptions::GetBlobViewUpstreamWarehouse() const { return impl_->blob_view_upstream_warehouse; } +bool CoreOptions::BlobViewResolveEnabled() const { + return impl_->blob_view_resolve_enabled; +} + std::vector CoreOptions::GetBlobInlineFields() const { std::vector blob_inline_fields = impl_->blob_descriptor_fields; blob_inline_fields.insert(blob_inline_fields.end(), impl_->blob_view_fields.begin(), diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index e6b08312..8dd2410a 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -199,6 +199,7 @@ class PAIMON_EXPORT CoreOptions { const std::vector& GetBlobDescriptorFields() const; const std::vector& GetBlobViewFields() const; std::optional GetBlobViewUpstreamWarehouse() const; + bool BlobViewResolveEnabled() const; std::vector GetBlobInlineFields() const; const std::map& ToMap() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index f5a977ed..e6f61e5e 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -126,6 +126,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_TRUE(core_options.GetBlobViewFields().empty()); ASSERT_TRUE(core_options.GetBlobInlineFields().empty()); ASSERT_EQ(std::nullopt, core_options.GetBlobViewUpstreamWarehouse()); + ASSERT_TRUE(core_options.BlobViewResolveEnabled()); ASSERT_TRUE(core_options.LegacyPartitionNameEnabled()); ASSERT_TRUE(core_options.GlobalIndexEnabled()); ASSERT_EQ(std::nullopt, core_options.GetGlobalIndexExternalPath()); @@ -230,6 +231,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::BLOB_DESCRIPTOR_FIELD, "blob3,blob4"}, {Options::BLOB_VIEW_FIELD, "blob5"}, {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, "FILE:///tmp/blob_view_upstream_warehouse/"}, + {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}, {Options::PARTITION_GENERATE_LEGACY_NAME, "false"}, {Options::GLOBAL_INDEX_ENABLED, "false"}, {Options::GLOBAL_INDEX_THREAD_NUM, "4"}, @@ -370,6 +372,7 @@ TEST(CoreOptionsTest, TestFromMap) { std::vector({"blob3", "blob4", "blob5"})); ASSERT_EQ(core_options.GetBlobViewUpstreamWarehouse(), std::optional("FILE:///tmp/blob_view_upstream_warehouse/")); + ASSERT_FALSE(core_options.BlobViewResolveEnabled()); ASSERT_FALSE(core_options.LegacyPartitionNameEnabled()); ASSERT_FALSE(core_options.GlobalIndexEnabled()); ASSERT_EQ(core_options.GetGlobalIndexThreadNum(), 4); diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 4da17383..a75acad0 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -169,6 +169,11 @@ Result> DataEvolutionSplitRead::CreateReader( Result> DataEvolutionSplitRead::WrapWithBlobViewResolverIfNeeded( const std::shared_ptr& data_split, std::unique_ptr&& inner_reader) const { + if (!options_.BlobViewResolveEnabled()) { + // preserve serialized BlobViewStruct bytes, e.g. for forwarding blob view values to + // another blob-view table + return std::move(inner_reader); + } std::vector read_blob_view_fields = HasBlobViewField(options_, raw_read_schema_); if (read_blob_view_fields.empty()) { return std::move(inner_reader); diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 9dcef3ad..a814192b 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -2493,6 +2493,166 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) { } } +TEST_P(BlobTableInteTest, TestForwardBlobViewReference) { + auto file_format = GetParam(); + if (file_format != "orc" && file_format != "parquet") { + return; + } + + // Forward blob view references between two blob-view tables: read the source table with + // resolve dynamically disabled, write the preserved BlobViewStruct bytes into the target + // table, then verify the target still stores the original upstream references and a + // default read resolves them to the actual upstream blob values. + const std::string upstream_db_name = "append_table_with_multi_blob"; + const std::string upstream_table_name = "append_table_with_multi_blob"; + std::string src_db_path = paimon::test::GetDataDir() + file_format + "/" + upstream_db_name + + ".db/" + upstream_table_name; + std::string dst_db_path = + PathUtil::JoinPath(dir_->Str(), upstream_db_name + ".db/" + upstream_table_name); + ASSERT_TRUE(TestUtil::CopyDirectory(src_db_path, dst_db_path)); + + // The source table has no upstream warehouse configured: only a read with resolve + // dynamically disabled can succeed on it. + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + BlobUtils::ToArrowField("view", true)}; + std::map source_options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_VIEW_FIELD, "view"}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, source_options); + std::string source_table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // The target table configures the upstream warehouse for resolving forwarded references. + std::map target_options = source_options; + target_options[Options::BLOB_VIEW_UPSTREAM_WAREHOUSE] = dir_->Str(); + auto schema = arrow::schema(fields); + ::ArrowSchema c_target_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_target_schema).ok()); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir_->Str(), {})); + ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar_forward"), &c_target_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, target_options, + /*ignore_if_exists=*/false)); + std::string target_table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar_forward"); + + // src array + Identifier upstream_identifier(upstream_db_name, upstream_table_name); + arrow::LargeBinaryBuilder view_builder; + for (int32_t i = 0; i < 8; ++i) { + if (i < 6) { + BlobViewStruct view_struct(upstream_identifier, /*field_id=*/6, + /*row_id=*/static_cast(i)); + auto serialized = view_struct.Serialize(pool_); + ASSERT_TRUE(view_builder + .Append(reinterpret_cast(serialized->data()), + serialized->size()) + .ok()); + } else { + ASSERT_TRUE(view_builder.AppendNull().ok()); + } + } + std::shared_ptr write_view_array; + ASSERT_TRUE(view_builder.Finish(&write_view_array).ok()); + auto write_f0_array = arrow::ipc::internal::json::ArrayFromJSON( + arrow::int32(), R"([100,101,102,103,104,105,106,107])") + .ValueOrDie(); + auto write_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({write_f0_array, write_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + + // write & commit into the source table + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(source_table_path, {}, schema->field_names(), {write_struct})); + ASSERT_OK(Commit(source_table_path, commit_msgs)); + + // A default read fails on the missing upstream warehouse: the pass-through below is + // enabled by the dynamic option alone. + ASSERT_OK_AND_ASSIGN(auto source_plan, ScanTable(source_table_path)); + ASSERT_NOK_WITH_MSG( + ReadTable(source_table_path, schema->field_names(), source_plan, /*predicate=*/nullptr), + "BLOB_VIEW_UPSTREAM_WAREHOUSE"); + + ASSERT_OK_AND_ASSIGN( + auto source_result, + ReadTable(source_table_path, schema->field_names(), source_plan, /*predicate=*/nullptr, + {{Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}})); + ASSERT_TRUE(source_result.chunked_array); + auto source_concat = arrow::Concatenate(source_result.chunked_array->chunks()).ValueOrDie(); + auto source_struct = std::dynamic_pointer_cast(source_concat); + ASSERT_EQ(source_struct->length(), 8); + auto forward_f0_array = source_struct->GetFieldByName("f0"); + ASSERT_TRUE(forward_f0_array); + auto forward_view_array = source_struct->GetFieldByName("view"); + ASSERT_TRUE(forward_view_array); + ASSERT_TRUE(forward_view_array->Equals(write_view_array)) + << "source view:" << forward_view_array->ToString() << std::endl + << "written view:" << write_view_array->ToString(); + + // Forward the preserved references into the target blob-view table. + auto forward_struct = std::dynamic_pointer_cast( + arrow::StructArray::Make(arrow::ArrayVector({forward_f0_array, forward_view_array}), + std::vector({"f0", "view"})) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN( + auto forward_commit_msgs, + WriteArray(target_table_path, {}, schema->field_names(), {forward_struct})); + ASSERT_OK(Commit(target_table_path, forward_commit_msgs)); + + // The target table stores the original upstream references byte-identically. + ASSERT_OK_AND_ASSIGN(auto target_plan, ScanTable(target_table_path)); + ASSERT_OK_AND_ASSIGN( + auto raw_target_result, + ReadTable(target_table_path, schema->field_names(), target_plan, /*predicate=*/nullptr, + {{Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}})); + ASSERT_TRUE(raw_target_result.chunked_array); + auto raw_target_concat = + arrow::Concatenate(raw_target_result.chunked_array->chunks()).ValueOrDie(); + auto raw_target_struct = std::dynamic_pointer_cast(raw_target_concat); + ASSERT_EQ(raw_target_struct->length(), 8); + auto raw_target_view_array = raw_target_struct->GetFieldByName("view"); + ASSERT_TRUE(raw_target_view_array); + ASSERT_TRUE(raw_target_view_array->Equals(write_view_array)) + << "target view:" << raw_target_view_array->ToString() << std::endl + << "written view:" << write_view_array->ToString(); + + // A default read of the target table resolves to the actual upstream blob values. + ASSERT_OK_AND_ASSIGN(auto resolved_result, + ReadTable(target_table_path, schema->field_names(), target_plan, + /*predicate=*/nullptr)); + ASSERT_TRUE(resolved_result.chunked_array); + auto resolved_concat = arrow::Concatenate(resolved_result.chunked_array->chunks()).ValueOrDie(); + auto resolved_struct = std::dynamic_pointer_cast(resolved_concat); + ASSERT_EQ(resolved_struct->length(), 8); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(resolved_struct, {"view"})); + + std::string padding_b(2048, 'b'); + std::string padding_d(2048, 'd'); + std::string padding_e(2048, 'e'); + std::string padding_f(2048, 'f'); + // clang-format off + std::string expected_json = R"([ +[100, null], +[101, ")" + padding_b + R"("], +[102, null], +[103, ")" + padding_d + R"("], +[104, ")" + padding_e + R"("], +[105, ")" + padding_f + R"("], +[106, null], +[107, null] +])"; + // clang-format on + auto expected_struct = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_struct)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "resolved:" << resolved->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) { auto file_format = GetParam(); if (GetParam() == "lance") { From fdca1ac34a4409c2655abf311f64a0eae17ae26d Mon Sep 17 00:00:00 2001 From: "Mr Dk." Date: Mon, 13 Jul 2026 18:51:28 +0800 Subject: [PATCH 093/138] fix(build): disable glog unwind without libunwind --- cmake_modules/ThirdpartyToolchain.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index e2063b99..1617df68 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -1890,6 +1890,8 @@ endmacro() macro(build_glog) message(STATUS "Building glog from source") + find_library(LIBUNWIND_LIBRARY NAMES unwind) + set(GLOG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/glog_ep-install") set(GLOG_INCLUDE_DIR "${GLOG_PREFIX}/include") if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") @@ -1912,6 +1914,9 @@ macro(build_glog) -DWITH_GTEST=OFF -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${GLOG_CMAKE_C_FLAGS}) + if(NOT LIBUNWIND_LIBRARY) + list(APPEND GLOG_CMAKE_ARGS -DWITH_UNWIND=none) + endif() externalproject_add(glog_ep URL ${GLOG_SOURCE_URL} @@ -1929,7 +1934,6 @@ macro(build_glog) add_dependencies(glog glog_ep) - find_library(LIBUNWIND_LIBRARY NAMES unwind) if(LIBUNWIND_LIBRARY) target_link_libraries(glog INTERFACE ${LIBUNWIND_LIBRARY}) endif() From b7f3ce3af9b31edeadb564025a259c9b7875dcb2 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Wed, 15 Jul 2026 13:37:08 +0800 Subject: [PATCH 094/138] fix(avro): include headers used by file batch reader --- src/paimon/format/avro/avro_file_batch_reader.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/paimon/format/avro/avro_file_batch_reader.h b/src/paimon/format/avro/avro_file_batch_reader.h index 1bb2452f..ccd1f770 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.h +++ b/src/paimon/format/avro/avro_file_batch_reader.h @@ -18,12 +18,14 @@ #pragma once +#include #include #include #include #include #include "avro/DataFile.hh" +#include "fmt/format.h" #include "paimon/format/avro/avro_direct_decoder.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" From c92c4d2d7c280e8295ba6d4c0fd87d05ec3dd228 Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Wed, 15 Jul 2026 16:39:46 +0800 Subject: [PATCH 095/138] fix(orc): avoid removed Arrow bitmap append API --- src/paimon/format/orc/orc_adapter.cpp | 28 ++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/paimon/format/orc/orc_adapter.cpp b/src/paimon/format/orc/orc_adapter.cpp index 3ca065f7..992387d1 100644 --- a/src/paimon/format/orc/orc_adapter.cpp +++ b/src/paimon/format/orc/orc_adapter.cpp @@ -144,7 +144,9 @@ class UnPooledBooleanBuilder : public EmptyBuilder { } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status SetData(const uint8_t* data, int64_t length) { @@ -189,7 +191,9 @@ class UnPooledPrimitiveBuilder : public arrow::NumericBuilder { } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { @@ -230,7 +234,9 @@ class UnPooledLargeBinaryBuilder : public arrow::LargeBinaryBuilder { length_ += length; } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { std::shared_ptr null_bitmap; @@ -275,7 +281,9 @@ class UnPooledBinaryBuilder : public arrow::BinaryBuilder { } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { @@ -315,7 +323,9 @@ class UnPooledListBuilder : public EmptyBuilder { } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } void SetOffsets(const std::shared_ptr& offsets) { @@ -363,7 +373,9 @@ class UnPooledStructBuilder : public EmptyBuilder { } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { @@ -489,7 +501,9 @@ class UnPooledStringDictionaryBuilder : public EmptyBuilder { indices_ = indices; } arrow::Status SetNulls(const uint8_t* valid_bytes, int64_t length) { - return arrow::ArrayBuilder::AppendToBitmap(valid_bytes, length); + ARROW_RETURN_NOT_OK(this->Reserve(length)); + this->UnsafeAppendToBitmap(valid_bytes, length); + return arrow::Status::OK(); } arrow::Status FinishInternal(std::shared_ptr* out) override { std::shared_ptr null_bitmap; From 354daf8f8ee59d9166fd1bc644c447f2c361430e Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Thu, 16 Jul 2026 09:47:47 +0800 Subject: [PATCH 096/138] feat(commit): align functionality with java paimon commit --- include/paimon/commit_context.h | 12 +- include/paimon/defs.h | 25 + include/paimon/file_store_commit.h | 19 +- include/paimon/utils/row_range_index.h | 9 +- src/paimon/CMakeLists.txt | 24 + src/paimon/common/data/binary_row.h | 13 + src/paimon/common/defs.cpp | 7 + src/paimon/common/table/special_fields.h | 12 +- .../common/table/special_fields_test.cpp | 15 +- src/paimon/common/utils/linked_hash_map.h | 9 +- src/paimon/common/utils/row_range_index.cpp | 18 +- .../common/utils/row_range_index_test.cpp | 65 +- .../catalog/commit_table_request_test.cpp | 3 +- .../catalog/renaming_snapshot_commit_test.cpp | 2 +- src/paimon/core/core_options.cpp | 68 + src/paimon/core/core_options.h | 16 + src/paimon/core/core_options_test.cpp | 47 + .../core/index/index_file_handler_test.cpp | 6 +- .../io/append_data_file_writer_factory.cpp | 4 +- .../core/io/blob_data_file_writer_factory.cpp | 4 +- ...edding_append_data_file_writer_factory.cpp | 4 +- .../core/manifest/manifest_committable.h | 21 +- .../manifest/manifest_committable_test.cpp | 32 +- .../operation/abstract_file_store_write.cpp | 3 +- .../commit/commit_changes_provider.cpp | 53 + .../commit/commit_changes_provider.h | 61 + .../commit/commit_changes_provider_test.cpp | 132 ++ .../core/operation/commit/commit_scanner.cpp | 244 ++++ .../core/operation/commit/commit_scanner.h | 118 ++ .../operation/commit/commit_scanner_test.cpp | 153 +++ .../compacted_changelog_path_resolver.cpp | 80 ++ .../compacted_changelog_path_resolver.h | 33 + ...compacted_changelog_path_resolver_test.cpp | 94 ++ .../operation/commit/conflict_detection.cpp | 613 +++++++++ .../operation/commit/conflict_detection.h | 130 ++ .../commit/conflict_detection_test.cpp | 590 +++++++++ .../commit/manifest_entry_changes.cpp | 149 +++ .../operation/commit/manifest_entry_changes.h | 75 ++ .../commit/manifest_entry_changes_test.cpp | 188 +++ .../commit/overwrite_changes_provider.cpp | 68 + .../commit/overwrite_changes_provider.h | 50 + .../overwrite_changes_provider_test.cpp | 225 ++++ .../core/operation/commit/retry_waiter.cpp | 61 + .../core/operation/commit/retry_waiter.h | 37 + .../operation/commit/retry_waiter_test.cpp | 50 + .../commit/row_id_column_conflict_checker.cpp | 222 ++++ .../commit/row_id_column_conflict_checker.h | 89 ++ .../row_id_column_conflict_checker_test.cpp | 157 +++ .../commit/row_tracking_commit_utils.cpp | 153 +++ .../commit/row_tracking_commit_utils.h | 52 + .../commit/row_tracking_commit_utils_test.cpp | 219 ++++ .../commit/sequence_snapshot_properties.cpp | 97 ++ .../commit/sequence_snapshot_properties.h | 55 + src/paimon/core/operation/commit_context.cpp | 14 +- .../core/operation/commit_context_test.cpp | 89 ++ .../core/operation/commit_metrics_test.cpp | 123 ++ .../core/operation/file_store_commit.cpp | 75 +- .../core/operation/file_store_commit_impl.cpp | 1119 ++++++++++------- .../core/operation/file_store_commit_impl.h | 107 +- .../operation/file_store_commit_impl_test.cpp | 817 ++++++++---- .../core/operation/file_store_commit_test.cpp | 69 + .../core/operation/metrics/commit_metrics.cpp | 63 + .../core/operation/metrics/commit_metrics.h | 9 + .../core/operation/metrics/commit_stats.h | 206 +++ .../operation/metrics/commit_stats_test.cpp | 160 +++ .../operation/orphan_files_cleaner_test.cpp | 2 - src/paimon/core/schema/schema_validation.cpp | 6 +- .../core/schema/schema_validation_test.cpp | 2 +- src/paimon/core/snapshot.cpp | 27 +- src/paimon/core/snapshot.h | 32 +- src/paimon/core/snapshot_test.cpp | 31 +- src/paimon/core/table/bucket_mode.cpp | 40 + src/paimon/core/table/bucket_mode.h | 5 + src/paimon/core/table/bucket_mode_test.cpp | 62 + .../table/system/metadata_system_tables.cpp | 14 +- src/paimon/core/tag/tag.cpp | 11 +- src/paimon/core/tag/tag.h | 4 +- src/paimon/core/tag/tag_test.cpp | 27 +- test/inte/append_compaction_inte_test.cpp | 28 +- test/inte/clean_inte_test.cpp | 10 +- test/inte/data_evolution_table_test.cpp | 53 + test/inte/pk_compaction_inte_test.cpp | 16 +- test/inte/read_inte_test.cpp | 2 +- test/inte/write_inte_test.cpp | 88 +- 84 files changed, 6992 insertions(+), 1005 deletions(-) create mode 100644 src/paimon/core/operation/commit/commit_changes_provider.cpp create mode 100644 src/paimon/core/operation/commit/commit_changes_provider.h create mode 100644 src/paimon/core/operation/commit/commit_changes_provider_test.cpp create mode 100644 src/paimon/core/operation/commit/commit_scanner.cpp create mode 100644 src/paimon/core/operation/commit/commit_scanner.h create mode 100644 src/paimon/core/operation/commit/commit_scanner_test.cpp create mode 100644 src/paimon/core/operation/commit/compacted_changelog_path_resolver.cpp create mode 100644 src/paimon/core/operation/commit/compacted_changelog_path_resolver.h create mode 100644 src/paimon/core/operation/commit/compacted_changelog_path_resolver_test.cpp create mode 100644 src/paimon/core/operation/commit/conflict_detection.cpp create mode 100644 src/paimon/core/operation/commit/conflict_detection.h create mode 100644 src/paimon/core/operation/commit/conflict_detection_test.cpp create mode 100644 src/paimon/core/operation/commit/manifest_entry_changes.cpp create mode 100644 src/paimon/core/operation/commit/manifest_entry_changes.h create mode 100644 src/paimon/core/operation/commit/manifest_entry_changes_test.cpp create mode 100644 src/paimon/core/operation/commit/overwrite_changes_provider.cpp create mode 100644 src/paimon/core/operation/commit/overwrite_changes_provider.h create mode 100644 src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp create mode 100644 src/paimon/core/operation/commit/retry_waiter.cpp create mode 100644 src/paimon/core/operation/commit/retry_waiter.h create mode 100644 src/paimon/core/operation/commit/retry_waiter_test.cpp create mode 100644 src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp create mode 100644 src/paimon/core/operation/commit/row_id_column_conflict_checker.h create mode 100644 src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp create mode 100644 src/paimon/core/operation/commit/row_tracking_commit_utils.cpp create mode 100644 src/paimon/core/operation/commit/row_tracking_commit_utils.h create mode 100644 src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp create mode 100644 src/paimon/core/operation/commit/sequence_snapshot_properties.cpp create mode 100644 src/paimon/core/operation/commit/sequence_snapshot_properties.h create mode 100644 src/paimon/core/operation/commit_context_test.cpp create mode 100644 src/paimon/core/operation/metrics/commit_metrics.cpp create mode 100644 src/paimon/core/operation/metrics/commit_stats.h create mode 100644 src/paimon/core/operation/metrics/commit_stats_test.cpp create mode 100644 src/paimon/core/table/bucket_mode.cpp create mode 100644 src/paimon/core/table/bucket_mode_test.cpp diff --git a/include/paimon/commit_context.h b/include/paimon/commit_context.h index b273048f..91acb1c1 100644 --- a/include/paimon/commit_context.h +++ b/include/paimon/commit_context.h @@ -40,7 +40,7 @@ class PAIMON_EXPORT CommitContext { public: CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, - const std::shared_ptr& memory_pool, + bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, const std::map& options); @@ -62,6 +62,10 @@ class PAIMON_EXPORT CommitContext { return use_rest_catalog_commit_; } + bool AppendCommitCheckConflict() const { + return append_commit_check_conflict_; + } + std::shared_ptr GetMemoryPool() const { return memory_pool_; } @@ -83,6 +87,7 @@ class PAIMON_EXPORT CommitContext { std::string commit_user_; bool ignore_empty_commit_; bool use_rest_catalog_commit_; + bool append_commit_check_conflict_; std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr specific_file_system_; @@ -128,6 +133,11 @@ class PAIMON_EXPORT CommitContextBuilder { /// @return Reference to this builder for method chaining. CommitContextBuilder& UseRESTCatalogCommit(bool use_rest_catalog_commit); + /// Sets whether append commits should perform conflict checking (default is false). + /// @param append_commit_check_conflict True to enable append conflict checks. + /// @return Reference to this builder for method chaining. + CommitContextBuilder& AppendCommitCheckConflict(bool append_commit_check_conflict); + /// Sets the memory pool to be used for memory allocation during commit operations. /// @param memory_pool Shared pointer to the memory pool instance. /// @return Reference to this builder for method chaining. diff --git a/include/paimon/defs.h b/include/paimon/defs.h index fb25e99a..dd3fc5ac 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -239,6 +239,16 @@ struct PAIMON_EXPORT Options { /// "commit.max-retries" - Maximum number of retries when commit failed. Default value is 10. static const char COMMIT_MAX_RETRIES[]; + /// "commit.min-retry-wait" - Min retry wait time when commit failed. Default value is 10ms. + static const char COMMIT_MIN_RETRY_WAIT[]; + + /// "commit.max-retry-wait" - Max retry wait time when commit failed. Default value is 10s. + static const char COMMIT_MAX_RETRY_WAIT[]; + + /// "commit.discard-duplicate-files" - Whether to discard duplicate files in commit. + /// Default value is "false". + static const char COMMIT_DISCARD_DUPLICATE_FILES[]; + /// "compaction.max-size-amplification-percent" - The size amplification is defined as the /// amount (in percentage) of additional storage needed to store a single byte of data in the /// merge tree for changelog mode table. Default value is 200. @@ -274,6 +284,15 @@ struct PAIMON_EXPORT Options { /// level 0 files in candidates. Default value is false. static const char COMPACTION_FORCE_UP_LEVEL_0[]; + /// "overwrite-upgrade" - Whether to try upgrading the data files after overwriting a + /// primary key table. Default value is true. + static const char OVERWRITE_UPGRADE[]; + + /// "dynamic-partition-overwrite" - Whether only overwrite dynamic partition when + /// overwriting a partitioned table with dynamic partition columns. Works only when + /// the table has partition keys. Default value is true. + static const char DYNAMIC_PARTITION_OVERWRITE[]; + /// "lookup-compact.max-interval" - The max interval for a gentle mode lookup compaction to be /// triggered. For every interval, a forced lookup compaction will be performed to flush L0 /// files to higher level. This option is only valid when lookup-compact mode is gentle. No @@ -457,6 +476,12 @@ struct PAIMON_EXPORT Options { /// "write-only" - If set to "true", compactions and snapshot expiration will be skipped. This /// option is used along with dedicated compact jobs. Default value is "false". static const char WRITE_ONLY[]; + /// "bucket-append-ordered" - Whether append writes in fixed bucket mode are ordered. This + /// option is used by commit conflict checks. Default value is "false". + static const char BUCKET_APPEND_ORDERED[]; + /// "write.sequence-number-init-mode" - Specify how to initialize the next sequence number for + /// primary key table writers. Values can be: "scan", "snapshot". Default value is "scan". + static const char WRITE_SEQUENCE_NUMBER_INIT_MODE[]; /// "compaction.min.file-num" - For file set [f_0,...,f_N], the minimum file number to trigger a /// compaction for append-only table. Default value is 5. static const char COMPACTION_MIN_FILE_NUM[]; diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index 66d3d390..b4cd2896 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -91,7 +91,7 @@ class PAIMON_EXPORT FileStoreCommit { /// Overwrite from manifest committable and partition. /// - /// @param partitions A single partition maps each partition key to a partition value. Depending + /// @param partition A single partition maps each partition key to a partition value. Depending /// on the user-defined statement, the partition might not include all partition keys. Also /// note that this partition does not necessarily equal to the partitions of the newly added /// key-values. This is just the partition to be cleaned up. @@ -100,7 +100,7 @@ class PAIMON_EXPORT FileStoreCommit { /// @param watermark An optional event-time watermark used to indicate the progress of data /// processing. Default is std::nullopt. /// @return Result of the operation. - virtual Status Overwrite(const std::vector>& partitions, + virtual Status Overwrite(const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) = 0; @@ -108,14 +108,14 @@ class PAIMON_EXPORT FileStoreCommit { /// This is a temporary interface for internal use. It will be removed in a future version. /// Please do not rely on it for long-term use. /// - /// @param partitions Description of the partitions. + /// @param partition Description of the partition. /// @param commit_messages Description of the commit messages. /// @param commit_identifier Unique identifier. /// @param watermark An optional event-time watermark used to indicate the progress of data /// processing. Default is std::nullopt. /// @return Result of the operation. virtual Result FilterAndOverwrite( - const std::vector>& partitions, + const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) = 0; @@ -143,6 +143,17 @@ class PAIMON_EXPORT FileStoreCommit { virtual Status DropPartition(const std::vector>& partitions, int64_t commit_identifier) = 0; + /// Configure row-id conflict checking from a specific snapshot id. + /// + /// If set to a snapshot id, commit conflict detection will additionally validate row-id + /// conflicts against snapshots after that id. Passing std::nullopt disables this behavior. + /// + /// @param row_id_check_from_snapshot Snapshot id to start row-id conflict checks from, or + /// std::nullopt to disable. + /// @return Current commit object for chaining. + virtual FileStoreCommit& RowIdCheckConflict( + std::optional row_id_check_from_snapshot) = 0; + /// Retrieve metrics related to commit operations. /// /// @return A shared pointer to a `Metrics` object containing commit metrics. diff --git a/include/paimon/utils/row_range_index.h b/include/paimon/utils/row_range_index.h index f5351a0f..4a349492 100644 --- a/include/paimon/utils/row_range_index.h +++ b/include/paimon/utils/row_range_index.h @@ -34,7 +34,8 @@ class PAIMON_EXPORT RowRangeIndex { public: /// Creates a RowRangeIndex from the given ranges. The ranges will be sorted and merged /// (overlapping and adjacent ranges are combined) before indexing. - static Result Create(const std::vector& ranges); + static Result Create(const std::vector& ranges, + bool merge_adjacent = true); /// Returns the sorted, non-overlapping ranges held by this index. const std::vector& Ranges() const; @@ -42,6 +43,12 @@ class PAIMON_EXPORT RowRangeIndex { /// Returns true if any range in this index intersects with the interval [start, end]. bool Intersects(int64_t start, int64_t end) const; + /// Returns true if one range in this index fully contains `range`. + bool Contains(const Range& range) const; + + /// Returns true if one range in this index exactly equals `range`. + bool ContainsExactly(const Range& range) const; + /// Returns the sub-ranges of this index that intersect with the interval [start, end]. /// Each returned range is clipped to lie within [start, end]. std::vector IntersectedRanges(int64_t start, int64_t end) const; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 73f11f83..2fbbb63e 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -290,6 +290,17 @@ set(PAIMON_CORE_SRCS core/operation/abstract_split_read.cpp core/operation/append_only_file_store_scan.cpp core/operation/append_only_file_store_write.cpp + core/operation/metrics/commit_metrics.cpp + core/operation/commit/conflict_detection.cpp + core/operation/commit/commit_scanner.cpp + core/operation/commit/commit_changes_provider.cpp + core/operation/commit/compacted_changelog_path_resolver.cpp + core/operation/commit/overwrite_changes_provider.cpp + core/operation/commit/row_id_column_conflict_checker.cpp + core/operation/commit/manifest_entry_changes.cpp + core/operation/commit/row_tracking_commit_utils.cpp + core/operation/commit/sequence_snapshot_properties.cpp + core/operation/commit/retry_waiter.cpp core/operation/commit_context.cpp core/operation/expire_snapshots.cpp core/operation/file_store_commit.cpp @@ -322,6 +333,7 @@ set(PAIMON_CORE_SRCS core/stats/simple_stats.cpp core/stats/simple_stats_evolution.cpp core/table/table.cpp + core/table/bucket_mode.cpp core/table/sink/commit_message.cpp core/table/sink/commit_message_impl.cpp core/table/sink/commit_message_serializer.cpp @@ -698,13 +710,24 @@ if(PAIMON_BUILD_TESTS) core/mergetree/spill_reader_writer_test.cpp core/migrate/file_meta_utils_test.cpp core/operation/metrics/compaction_metrics_test.cpp + core/operation/metrics/commit_stats_test.cpp core/operation/data_evolution_file_store_scan_test.cpp core/operation/data_evolution_split_read_test.cpp + core/operation/commit/compacted_changelog_path_resolver_test.cpp + core/operation/commit/commit_changes_provider_test.cpp + core/operation/commit/commit_scanner_test.cpp + core/operation/commit/conflict_detection_test.cpp + core/operation/commit/manifest_entry_changes_test.cpp + core/operation/commit/overwrite_changes_provider_test.cpp + core/operation/commit/row_id_column_conflict_checker_test.cpp + core/operation/commit/row_tracking_commit_utils_test.cpp + core/operation/commit/retry_waiter_test.cpp core/operation/key_value_file_store_write_test.cpp core/operation/internal_read_context_test.cpp core/operation/abstract_split_read_test.cpp core/operation/append_only_file_store_write_test.cpp core/operation/commit_metrics_test.cpp + core/operation/commit_context_test.cpp core/operation/expire_snapshots_test.cpp core/operation/file_store_commit_impl_test.cpp core/operation/file_store_commit_test.cpp @@ -733,6 +756,7 @@ if(PAIMON_BUILD_TESTS) core/stats/simple_stats_collector_test.cpp core/stats/simple_stats_test.cpp core/table/table_test.cpp + core/table/bucket_mode_test.cpp core/table/sink/commit_message_test.cpp core/table/sink/commit_message_impl_test.cpp core/table/source/fallback_data_split_test.cpp diff --git a/src/paimon/common/data/binary_row.h b/src/paimon/common/data/binary_row.h index a8640632..e3dfe8b0 100644 --- a/src/paimon/common/data/binary_row.h +++ b/src/paimon/common/data/binary_row.h @@ -159,6 +159,19 @@ struct hash> { } }; +/// for std::unordered_map, ...> +template <> +struct hash> { + size_t operator()( + const std::tuple& partition_bucket_level) const { + const auto& [partition, bucket, level] = partition_bucket_level; + size_t hash = paimon::MurmurHashUtils::HashUnsafeBytes( + reinterpret_cast(&bucket), 0, sizeof(bucket), partition.HashCode()); + return paimon::MurmurHashUtils::HashUnsafeBytes(reinterpret_cast(&level), 0, + sizeof(level), hash); + } +}; + template <> struct hash { size_t operator()(const paimon::BinaryRow& row) const { diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 19851c54..7c5beabf 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -68,6 +68,9 @@ const char Options::SNAPSHOT_CLEAN_EMPTY_DIRECTORIES[] = "snapshot.clean-empty-d const char Options::COMMIT_FORCE_COMPACT[] = "commit.force-compact"; const char Options::COMMIT_TIMEOUT[] = "commit.timeout"; const char Options::COMMIT_MAX_RETRIES[] = "commit.max-retries"; +const char Options::COMMIT_MIN_RETRY_WAIT[] = "commit.min-retry-wait"; +const char Options::COMMIT_MAX_RETRY_WAIT[] = "commit.max-retry-wait"; +const char Options::COMMIT_DISCARD_DUPLICATE_FILES[] = "commit.discard-duplicate-files"; const char Options::SEQUENCE_FIELD[] = "sequence.field"; const char Options::SEQUENCE_FIELD_SORT_ORDER[] = "sequence.field.sort-order"; const char Options::MERGE_ENGINE[] = "merge-engine"; @@ -117,6 +120,8 @@ const char Options::SCAN_TIMESTAMP_MILLIS[] = "scan.timestamp-millis"; const char Options::SCAN_TIMESTAMP[] = "scan.timestamp"; const char Options::SCAN_TAG_NAME[] = "scan.tag-name"; const char Options::WRITE_ONLY[] = "write-only"; +const char Options::BUCKET_APPEND_ORDERED[] = "bucket-append-ordered"; +const char Options::WRITE_SEQUENCE_NUMBER_INIT_MODE[] = "write.sequence-number-init-mode"; const char Options::COMPACTION_MIN_FILE_NUM[] = "compaction.min.file-num"; const char Options::COMPACTION_FORCE_REWRITE_ALL_FILES[] = "compaction.force-rewrite-all-files"; const char Options::COMPACTION_OPTIMIZATION_INTERVAL[] = "compaction.optimization-interval"; @@ -141,6 +146,8 @@ const char Options::NUM_SORTED_RUNS_COMPACTION_TRIGGER[] = "num-sorted-run.compa const char Options::NUM_SORTED_RUNS_STOP_TRIGGER[] = "num-sorted-run.stop-trigger"; const char Options::NUM_LEVELS[] = "num-levels"; const char Options::COMPACTION_FORCE_UP_LEVEL_0[] = "compaction.force-up-level-0"; +const char Options::OVERWRITE_UPGRADE[] = "overwrite-upgrade"; +const char Options::DYNAMIC_PARTITION_OVERWRITE[] = "dynamic-partition-overwrite"; const char Options::LOOKUP_WAIT[] = "lookup-wait"; const char Options::LOOKUP_COMPACT[] = "lookup-compact"; const char Options::LOOKUP_COMPACT_MAX_INTERVAL[] = "lookup-compact.max-interval"; diff --git a/src/paimon/common/table/special_fields.h b/src/paimon/common/table/special_fields.h index b4319649..74b95b19 100644 --- a/src/paimon/common/table/special_fields.h +++ b/src/paimon/common/table/special_fields.h @@ -24,6 +24,7 @@ #include "arrow/type_fwd.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/utils/special_field_ids.h" namespace paimon { @@ -65,14 +66,15 @@ struct SpecialFields { return data_field; } - static bool IsSpecialFieldName(const std::string& field_name) { - if (field_name == SequenceNumber().Name() || field_name == ValueKind().Name() || - field_name == RowKind().Name() || field_name == RowId().Name() || - field_name == IndexScore().Name()) { + static bool IsSystemField(const std::string& field_name) { + if (StringUtils::StartsWith(field_name, KEY_FIELD_PREFIX)) { return true; } - return false; + return field_name == SequenceNumber().Name() || field_name == ValueKind().Name() || + field_name == RowKind().Name() || field_name == RowId().Name() || + field_name == IndexScore().Name(); } + // TODO(xinyu.lxy): add a func to complete row-tracking fields static std::shared_ptr CompleteSequenceAndValueKindField( diff --git a/src/paimon/common/table/special_fields_test.cpp b/src/paimon/common/table/special_fields_test.cpp index 0c17918c..68e805fd 100644 --- a/src/paimon/common/table/special_fields_test.cpp +++ b/src/paimon/common/table/special_fields_test.cpp @@ -59,13 +59,14 @@ TEST(SpecialFieldsTest, TestKeyValueSpecialFieldCount) { ASSERT_EQ(SpecialFields::KEY_VALUE_SPECIAL_FIELD_COUNT, 2); } -TEST(SpecialFieldsTest, TestIsSpecialFieldName) { - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("_SEQUENCE_NUMBER")); - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("_VALUE_KIND")); - ASSERT_FALSE(SpecialFields::IsSpecialFieldName("VALUE_KIND")); - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("rowkind")); - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("_ROW_ID")); - ASSERT_TRUE(SpecialFields::IsSpecialFieldName("_INDEX_SCORE")); +TEST(SpecialFieldsTest, TestIsSystemField) { + ASSERT_TRUE(SpecialFields::IsSystemField("_SEQUENCE_NUMBER")); + ASSERT_TRUE(SpecialFields::IsSystemField("_VALUE_KIND")); + ASSERT_FALSE(SpecialFields::IsSystemField("VALUE_KIND")); + ASSERT_TRUE(SpecialFields::IsSystemField("rowkind")); + ASSERT_TRUE(SpecialFields::IsSystemField("_ROW_ID")); + ASSERT_TRUE(SpecialFields::IsSystemField("_INDEX_SCORE")); + ASSERT_TRUE(SpecialFields::IsSystemField("_KEY_0")); } } // namespace paimon::test diff --git a/src/paimon/common/utils/linked_hash_map.h b/src/paimon/common/utils/linked_hash_map.h index f831756e..02e0beef 100644 --- a/src/paimon/common/utils/linked_hash_map.h +++ b/src/paimon/common/utils/linked_hash_map.h @@ -73,12 +73,13 @@ class LinkedHashMap { } IteratorType erase(const K& key) { - if (!map_.count(key)) { + auto map_iter = map_.find(key); + if (map_iter == map_.end()) { return order_.end(); } - auto iter = order_.erase(map_[key]); - map_.erase(key); - return iter; + IteratorType order_iter = map_iter->second; + map_.erase(map_iter); + return order_.erase(order_iter); } IteratorType insert(const K& key, const V& value) { diff --git a/src/paimon/common/utils/row_range_index.cpp b/src/paimon/common/utils/row_range_index.cpp index 87bff084..c65bcf20 100644 --- a/src/paimon/common/utils/row_range_index.cpp +++ b/src/paimon/common/utils/row_range_index.cpp @@ -33,11 +33,8 @@ RowRangeIndex::RowRangeIndex(std::vector ranges) : ranges_(std::move(rang } } -Result RowRangeIndex::Create(const std::vector& ranges) { - if (ranges.empty()) { - return Status::Invalid("Ranges cannot be empty in RowRangeIndex"); - } - return RowRangeIndex(Range::SortAndMergeOverlap(ranges, /*adjacent=*/true)); +Result RowRangeIndex::Create(const std::vector& ranges, bool merge_adjacent) { + return RowRangeIndex(Range::SortAndMergeOverlap(ranges, merge_adjacent)); } const std::vector& RowRangeIndex::Ranges() const { @@ -49,6 +46,17 @@ bool RowRangeIndex::Intersects(int64_t start, int64_t end) const { return candidate < static_cast(starts_.size()) && starts_[candidate] <= end; } +bool RowRangeIndex::Contains(const Range& range) const { + int32_t candidate = LowerBound(range.from); + return candidate < static_cast(ranges_.size()) && + ranges_[candidate].from <= range.from && ranges_[candidate].to >= range.to; +} + +bool RowRangeIndex::ContainsExactly(const Range& range) const { + int32_t candidate = LowerBound(range.from); + return candidate < static_cast(ranges_.size()) && ranges_[candidate] == range; +} + std::vector RowRangeIndex::IntersectedRanges(int64_t start, int64_t end) const { int32_t left = LowerBound(start); if (left >= static_cast(ranges_.size())) { diff --git a/src/paimon/common/utils/row_range_index_test.cpp b/src/paimon/common/utils/row_range_index_test.cpp index e9095f64..d2d755a8 100644 --- a/src/paimon/common/utils/row_range_index_test.cpp +++ b/src/paimon/common/utils/row_range_index_test.cpp @@ -25,8 +25,9 @@ namespace paimon::test { // ======================== Create ======================== -TEST(RowRangeIndexTest, CreateWithEmptyRangesReturnsError) { - ASSERT_NOK_WITH_MSG(RowRangeIndex::Create({}), "Ranges cannot be empty in RowRangeIndex"); +TEST(RowRangeIndexTest, CreateWithEmptyRangesReturnsEmptyIndex) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({})); + ASSERT_TRUE(index.Ranges().empty()); } TEST(RowRangeIndexTest, CreateWithSingleRange) { @@ -165,6 +166,66 @@ TEST(RowRangeIndexTest, IntersectsSinglePointNoMatch) { ASSERT_FALSE(index.Intersects(11, 11)); } +// ======================== Contains ======================== + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyExactRange) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(10, 20), Range(30, 40)})); + Range query(10, 20); + + ASSERT_TRUE(index.Contains(query)); + ASSERT_TRUE(index.ContainsExactly(query)); +} + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyBoundarySubRanges) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(10, 20)})); + + ASSERT_TRUE(index.Contains(Range(10, 19))); + ASSERT_FALSE(index.ContainsExactly(Range(10, 19))); + + ASSERT_TRUE(index.Contains(Range(11, 20))); + ASSERT_FALSE(index.ContainsExactly(Range(11, 20))); +} + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyOutOfBoundaryRanges) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(10, 20)})); + + ASSERT_FALSE(index.Contains(Range(9, 20))); + ASSERT_FALSE(index.ContainsExactly(Range(9, 20))); + + ASSERT_FALSE(index.Contains(Range(10, 21))); + ASSERT_FALSE(index.ContainsExactly(Range(10, 21))); +} + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyRangeAcrossTwoDisjointRanges) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(0, 10), Range(11, 20)}, false)); + + ASSERT_FALSE(index.Contains(Range(10, 11))); + ASSERT_FALSE(index.ContainsExactly(Range(10, 11))); + ASSERT_FALSE(index.ContainsExactly(Range(0, 20))); +} + +TEST(RowRangeIndexTest, ContainsAndContainsExactlyAdjacentMergeFlagDifference) { + ASSERT_OK_AND_ASSIGN(auto merged_index, + RowRangeIndex::Create({Range(0, 10), Range(11, 20)}, true)); + ASSERT_OK_AND_ASSIGN(auto non_merged_index, + RowRangeIndex::Create({Range(0, 10), Range(11, 20)}, false)); + + Range query(10, 11); + ASSERT_TRUE(merged_index.Contains(query)); + ASSERT_FALSE(merged_index.ContainsExactly(query)); + + ASSERT_FALSE(non_merged_index.Contains(query)); + ASSERT_FALSE(non_merged_index.ContainsExactly(query)); +} + +// ======================== merge_adjacent ======================== + +TEST(RowRangeIndexTest, CreateDoesNotMergeAdjacentRangesWhenDisabled) { + ASSERT_OK_AND_ASSIGN(auto index, RowRangeIndex::Create({Range(0, 10), Range(11, 20)}, false)); + std::vector expected = {Range(0, 10), Range(11, 20)}; + ASSERT_EQ(index.Ranges(), expected); +} + // ======================== IntersectedRanges ======================== TEST(RowRangeIndexTest, IntersectedRangesExactMatch) { diff --git a/src/paimon/core/catalog/commit_table_request_test.cpp b/src/paimon/core/catalog/commit_table_request_test.cpp index 39b9ff95..7de6935a 100644 --- a/src/paimon/core/catalog/commit_table_request_test.cpp +++ b/src/paimon/core/catalog/commit_table_request_test.cpp @@ -39,7 +39,7 @@ TEST(CommitTableRequestTest, TestSimple) { /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt, /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807, /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597, - /*log_offsets=*/std::map(), /*total_record_count=*/5, + /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0); std::vector partition_statistics = { @@ -63,7 +63,6 @@ TEST(CommitTableRequestTest, TestSimple) { "commitIdentifier": 9223372036854775807, "commitKind": "APPEND", "timeMillis": 1758097357597, - "logOffsets": {}, "totalRecordCount": 5, "deltaRecordCount": 5, "changelogRecordCount": 0, diff --git a/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp b/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp index f00da0e6..c415390a 100644 --- a/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp +++ b/src/paimon/core/catalog/renaming_snapshot_commit_test.cpp @@ -48,7 +48,7 @@ TEST(RenamingSnapshotCommitTest, TestSimple) { /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt, /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807, /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597, - /*log_offsets=*/std::map(), /*total_record_count=*/5, + /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0); ASSERT_OK_AND_ASSIGN(bool success, commit->Commit(snapshot, /*statistics=*/{})); diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index a4ccb67c..5c434188 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -370,6 +370,8 @@ struct CoreOptions::Impl { int64_t manifest_full_compaction_file_size = 16 * 1024 * 1024; int64_t write_buffer_size = 256 * 1024 * 1024; int64_t commit_timeout = std::numeric_limits::max(); + int64_t commit_min_retry_wait = 10; + int64_t commit_max_retry_wait = 10 * 1000; std::shared_ptr file_format; std::shared_ptr file_system; @@ -430,6 +432,9 @@ struct CoreOptions::Impl { bool ignore_delete = false; bool write_buffer_spillable = true; bool write_only = false; + bool bucket_append_ordered = false; + CoreOptions::SequenceNumberInitMode write_sequence_number_init_mode = + CoreOptions::SequenceNumberInitMode::SCAN; bool deletion_vectors_enabled = false; bool deletion_vectors_bitmap64 = false; bool force_lookup = false; @@ -449,6 +454,9 @@ struct CoreOptions::Impl { bool global_index_enabled = true; std::optional global_index_thread_num; bool commit_force_compact = false; + bool commit_discard_duplicate_files = false; + bool dynamic_partition_overwrite = true; + bool overwrite_upgrade = true; bool compaction_force_rewrite_all_files = false; bool compaction_force_up_level_0 = false; std::optional global_index_external_path; @@ -523,6 +531,9 @@ struct CoreOptions::Impl { specified_file_system, &file_system)); // Parse write-only - if true, compactions and snapshot expiration will be skipped PAIMON_RETURN_NOT_OK(parser.Parse(Options::WRITE_ONLY, &write_only)); + // Parse bucket-append-ordered - append writes in fixed-bucket mode are ordered + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::BUCKET_APPEND_ORDERED, &bucket_append_ordered)); // Parse partition.legacy-name - use legacy ToString for partition names, default true PAIMON_RETURN_NOT_OK(parser.Parse(Options::PARTITION_GENERATE_LEGACY_NAME, &legacy_partition_name_enabled)); @@ -644,6 +655,22 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK(parser.ParseTimeDuration(Options::COMMIT_TIMEOUT, &commit_timeout)); // Parse commit.max-retries - maximum retries when commit failed, default 10 PAIMON_RETURN_NOT_OK(parser.Parse(Options::COMMIT_MAX_RETRIES, &commit_max_retries)); + // Parse commit.min-retry-wait - minimum retry wait when commit failed, default 10ms + PAIMON_RETURN_NOT_OK( + parser.ParseTimeDuration(Options::COMMIT_MIN_RETRY_WAIT, &commit_min_retry_wait)); + // Parse commit.max-retry-wait - maximum retry wait when commit failed, default 10s + PAIMON_RETURN_NOT_OK( + parser.ParseTimeDuration(Options::COMMIT_MAX_RETRY_WAIT, &commit_max_retry_wait)); + // Parse commit.discard-duplicate-files - whether to discard duplicate files on append + PAIMON_RETURN_NOT_OK(parser.Parse(Options::COMMIT_DISCARD_DUPLICATE_FILES, + &commit_discard_duplicate_files)); + // Parse dynamic-partition-overwrite - whether overwrite only dynamic partitions + // for partitioned table overwrite. + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::DYNAMIC_PARTITION_OVERWRITE, &dynamic_partition_overwrite)); + // Parse overwrite-upgrade - whether to try upgrading data files after overwrite on + // primary key table + PAIMON_RETURN_NOT_OK(parser.Parse(Options::OVERWRITE_UPGRADE, &overwrite_upgrade)); return Status::OK(); } @@ -654,6 +681,19 @@ struct CoreOptions::Impl { Options::SEQUENCE_FIELD, Options::FIELDS_SEPARATOR, &sequence_field)); // Parse sequence.field.sort-order - order of sequence field, default "ascending" PAIMON_RETURN_NOT_OK(parser.ParseSortOrder(&sequence_field_sort_order)); + // Parse write-sequence-number-init-mode - sequence init mode for write path + std::string write_sequence_init_mode_str = "scan"; + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, &write_sequence_init_mode_str)); + write_sequence_init_mode_str = StringUtils::ToLowerCase(write_sequence_init_mode_str); + if (write_sequence_init_mode_str == "scan") { + write_sequence_number_init_mode = CoreOptions::SequenceNumberInitMode::SCAN; + } else if (write_sequence_init_mode_str == "snapshot") { + write_sequence_number_init_mode = CoreOptions::SequenceNumberInitMode::SNAPSHOT; + } else { + return Status::Invalid(fmt::format("invalid write sequence number init mode: {}", + write_sequence_init_mode_str)); + } // Parse sort-engine - sort engine for primary key table, default "loser-tree" PAIMON_RETURN_NOT_OK(parser.ParseSortEngine(&sort_engine)); // Parse merge-engine - merge engine for primary key table, default "deduplicate" @@ -1056,6 +1096,26 @@ int32_t CoreOptions::GetCommitMaxRetries() const { return impl_->commit_max_retries; } +int64_t CoreOptions::GetCommitMinRetryWait() const { + return impl_->commit_min_retry_wait; +} + +int64_t CoreOptions::GetCommitMaxRetryWait() const { + return impl_->commit_max_retry_wait; +} + +bool CoreOptions::CommitDiscardDuplicateFiles() const { + return impl_->commit_discard_duplicate_files; +} + +bool CoreOptions::DynamicPartitionOverwrite() const { + return impl_->dynamic_partition_overwrite; +} + +bool CoreOptions::OverwriteUpgrade() const { + return impl_->overwrite_upgrade; +} + int32_t CoreOptions::GetCompactionMinFileNum() const { return impl_->compaction_min_file_num; } @@ -1152,6 +1212,14 @@ bool CoreOptions::WriteOnly() const { return impl_->write_only; } +bool CoreOptions::BucketAppendOrdered() const { + return impl_->bucket_append_ordered; +} + +CoreOptions::SequenceNumberInitMode CoreOptions::WriteSequenceNumberInitMode() const { + return impl_->write_sequence_number_init_mode; +} + std::optional CoreOptions::GetFieldsDefaultFunc() const { return impl_->field_default_func; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index 8dd2410a..ee573eb7 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -50,6 +50,15 @@ class Cache; class PAIMON_EXPORT CoreOptions { public: + /// Specifies how to initialize the next sequence number for primary key table writers. + enum class SequenceNumberInitMode { + // initialize by scanning existing file metadata. + SCAN, + // initialize from the maximum sequence number recorded in snapshot properties, + // which can avoid scanning existing file metadata in write-only mode. + SNAPSHOT, + }; + static Result FromMap( const std::map& options_map, const std::shared_ptr& specified_file_system = nullptr, @@ -103,6 +112,11 @@ class PAIMON_EXPORT CoreOptions { bool CompactionForceUpLevel0() const; int64_t GetCommitTimeout() const; int32_t GetCommitMaxRetries() const; + int64_t GetCommitMinRetryWait() const; + int64_t GetCommitMaxRetryWait() const; + bool CommitDiscardDuplicateFiles() const; + bool DynamicPartitionOverwrite() const; + bool OverwriteUpgrade() const; int32_t GetCompactionMinFileNum() const; int32_t GetCompactionMaxSizeAmplificationPercent() const; int32_t GetCompactionSizeRatio() const; @@ -118,6 +132,8 @@ class PAIMON_EXPORT CoreOptions { SortEngine GetSortEngine() const; bool IgnoreDelete() const; bool WriteOnly() const; + bool BucketAppendOrdered() const; + SequenceNumberInitMode WriteSequenceNumberInitMode() const; std::optional GetFieldsDefaultFunc() const; Result> GetFieldAggFunc(const std::string& field_name) const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index e6f61e5e..71508421 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -70,8 +70,13 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ("zstd", core_options.GetSpillCompressOptions().compress); ASSERT_EQ(1, core_options.GetSpillCompressOptions().zstd_level); ASSERT_FALSE(core_options.CommitForceCompact()); + ASSERT_TRUE(core_options.DynamicPartitionOverwrite()); + ASSERT_TRUE(core_options.OverwriteUpgrade()); ASSERT_EQ(std::numeric_limits::max(), core_options.GetCommitTimeout()); ASSERT_EQ(10, core_options.GetCommitMaxRetries()); + ASSERT_EQ(10, core_options.GetCommitMinRetryWait()); + ASSERT_EQ(10 * 1000, core_options.GetCommitMaxRetryWait()); + ASSERT_FALSE(core_options.CommitDiscardDuplicateFiles()); ExpireConfig expire_config = core_options.GetExpireConfig(); ASSERT_EQ(10, expire_config.GetSnapshotRetainMin()); ASSERT_EQ(std::numeric_limits::max(), expire_config.GetSnapshotRetainMax()); @@ -84,6 +89,9 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(SortEngine::LOSER_TREE, core_options.GetSortEngine()); ASSERT_FALSE(core_options.IgnoreDelete()); ASSERT_FALSE(core_options.WriteOnly()); + ASSERT_FALSE(core_options.BucketAppendOrdered()); + ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SCAN, + core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(5, core_options.GetCompactionMinFileNum()); ASSERT_FALSE(core_options.CompactionForceRewriteAllFiles()); ASSERT_FALSE(core_options.CompactionForceUpLevel0()); @@ -186,6 +194,11 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::COMMIT_FORCE_COMPACT, "true"}, {Options::COMMIT_TIMEOUT, "120s"}, {Options::COMMIT_MAX_RETRIES, "20"}, + {Options::COMMIT_MIN_RETRY_WAIT, "5ms"}, + {Options::COMMIT_MAX_RETRY_WAIT, "3s"}, + {Options::COMMIT_DISCARD_DUPLICATE_FILES, "true"}, + {Options::DYNAMIC_PARTITION_OVERWRITE, "false"}, + {Options::OVERWRITE_UPGRADE, "false"}, {Options::SCAN_SNAPSHOT_ID, "5"}, {Options::SCAN_MODE, "from-snapshot-full"}, {Options::SCAN_MANIFEST_ENTRY_CACHE_MAX_SNAPSHOTS, "7"}, @@ -238,6 +251,8 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::GLOBAL_INDEX_EXTERNAL_PATH, "FILE:///tmp/global_index/"}, {Options::SCAN_TAG_NAME, "test-tag"}, {Options::WRITE_ONLY, "true"}, + {Options::BUCKET_APPEND_ORDERED, "true"}, + {Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot"}, {Options::COMPACTION_MIN_FILE_NUM, "10"}, {Options::COMPACTION_FORCE_REWRITE_ALL_FILES, "true"}, {Options::COMPACTION_FORCE_UP_LEVEL_0, "true"}, @@ -307,8 +322,13 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ("lz4", core_options.GetSpillCompressOptions().compress); ASSERT_EQ(2, core_options.GetSpillCompressOptions().zstd_level); ASSERT_TRUE(core_options.CommitForceCompact()); + ASSERT_FALSE(core_options.DynamicPartitionOverwrite()); + ASSERT_FALSE(core_options.OverwriteUpgrade()); ASSERT_EQ(120 * 1000, core_options.GetCommitTimeout()); ASSERT_EQ(20, core_options.GetCommitMaxRetries()); + ASSERT_EQ(5, core_options.GetCommitMinRetryWait()); + ASSERT_EQ(3 * 1000, core_options.GetCommitMaxRetryWait()); + ASSERT_TRUE(core_options.CommitDiscardDuplicateFiles()); ASSERT_EQ(5, core_options.GetScanSnapshotId().value_or(-1)); ASSERT_EQ(7, core_options.GetScanManifestEntryCacheMaxSnapshots()); ExpireConfig expire_config = core_options.GetExpireConfig(); @@ -383,6 +403,9 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/true)); ASSERT_EQ(375809637, core_options.GetCompactionFileSize(/*has_primary_key=*/false)); ASSERT_TRUE(core_options.WriteOnly()); + ASSERT_TRUE(core_options.BucketAppendOrdered()); + ASSERT_EQ(CoreOptions::SequenceNumberInitMode::SNAPSHOT, + core_options.WriteSequenceNumberInitMode()); ASSERT_EQ(10, core_options.GetCompactionMinFileNum()); ASSERT_EQ(123, core_options.GetCompactionMaxSizeAmplificationPercent()); ASSERT_EQ(9, core_options.GetCompactionSizeRatio()); @@ -445,6 +468,9 @@ TEST(CoreOptionsTest, TestInvalidCase) { "The high priority pool ratio should in the range [0, 1), while input is 1.1"); ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::BUCKET_FUNCTION_TYPE, "invalid"}}), "invalid bucket function type: invalid"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "invalid"}}), + "invalid write sequence number init mode: invalid"); } TEST(CoreOptionsTest, TestLookupCompactMaxIntervalComputedValue) { @@ -456,6 +482,27 @@ TEST(CoreOptionsTest, TestLookupCompactMaxIntervalComputedValue) { ASSERT_EQ(13, core_options.GetLookupCompactMaxInterval()); } +TEST(CoreOptionsTest, TestDynamicPartitionOverwriteOption) { + { + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ASSERT_TRUE(core_options.DynamicPartitionOverwrite()); + } + + { + ASSERT_OK_AND_ASSIGN( + CoreOptions core_options, + CoreOptions::FromMap({{Options::DYNAMIC_PARTITION_OVERWRITE, "false"}})); + ASSERT_FALSE(core_options.DynamicPartitionOverwrite()); + } + + { + ASSERT_OK_AND_ASSIGN( + CoreOptions core_options, + CoreOptions::FromMap({{Options::DYNAMIC_PARTITION_OVERWRITE, "true"}})); + ASSERT_TRUE(core_options.DynamicPartitionOverwrite()); + } +} + TEST(CoreOptionsTest, TestNumSortedRunsStopTriggerFloorAndDefault) { { std::map options = { diff --git a/src/paimon/core/index/index_file_handler_test.cpp b/src/paimon/core/index/index_file_handler_test.cpp index 8236d184..2f839b8d 100644 --- a/src/paimon/core/index/index_file_handler_test.cpp +++ b/src/paimon/core/index/index_file_handler_test.cpp @@ -268,9 +268,9 @@ TEST_F(IndexFileHandlerTest, TestScanWithNoIndexManifest) { snapshot.DeltaManifestListSize(), snapshot.ChangelogManifestList(), snapshot.ChangelogManifestListSize(), /*index_manifest=*/std::nullopt, snapshot.CommitUser(), snapshot.CommitIdentifier(), snapshot.GetCommitKind(), - snapshot.TimeMillis(), snapshot.LogOffsets(), snapshot.TotalRecordCount(), - snapshot.DeltaRecordCount(), snapshot.ChangelogRecordCount(), snapshot.Watermark(), - snapshot.Statistics(), snapshot.Properties(), snapshot.NextRowId()); + snapshot.TimeMillis(), snapshot.TotalRecordCount(), snapshot.DeltaRecordCount(), + snapshot.ChangelogRecordCount(), snapshot.Watermark(), snapshot.Statistics(), + snapshot.Properties(), snapshot.NextRowId()); auto partition = BinaryRowGenerator::GenerateRow({10}, memory_pool_.get()); std::unordered_set partitions = {partition}; diff --git a/src/paimon/core/io/append_data_file_writer_factory.cpp b/src/paimon/core/io/append_data_file_writer_factory.cpp index 370c99e2..20e9daec 100644 --- a/src/paimon/core/io/append_data_file_writer_factory.cpp +++ b/src/paimon/core/io/append_data_file_writer_factory.cpp @@ -45,12 +45,14 @@ AppendDataFileWriterFactory::AppendDataFileWriterFactory( Result>>> AppendDataFileWriterFactory::CreateWriter() const { + std::shared_ptr seq_num_counter = + options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*options_.GetFileFormat(), write_schema_, /*create_stats_extractor=*/true)); auto writer = std::make_unique( options_.GetFileCompression(), std::function(), - schema_id_, seq_num_counter_, file_source_, resources.stats_extractor, + schema_id_, seq_num_counter, file_source_, resources.stats_extractor, path_factory_->IsExternalPath(), write_cols_, pool_); PAIMON_RETURN_NOT_OK( writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.cpp b/src/paimon/core/io/blob_data_file_writer_factory.cpp index 31dce727..78d740d7 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.cpp +++ b/src/paimon/core/io/blob_data_file_writer_factory.cpp @@ -46,6 +46,8 @@ BlobDataFileWriterFactory::BlobDataFileWriterFactory( Result>>> BlobDataFileWriterFactory::CreateWriter() const { + std::shared_ptr seq_num_counter = + options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, FileFormatFactory::Get("blob", options_.ToMap())); PAIMON_ASSIGN_OR_RAISE(WriterResources resources, @@ -53,7 +55,7 @@ BlobDataFileWriterFactory::CreateWriter() const { /*create_stats_extractor=*/true)); auto writer = std::make_unique( /*compression=*/"none", std::function(), schema_id_, - seq_num_counter_, FileSource::Append(), resources.stats_extractor, + seq_num_counter, FileSource::Append(), resources.stats_extractor, path_factory_->IsExternalPath(), write_cols_, pool_); PAIMON_RETURN_NOT_OK(writer->Init(options_.GetFileSystem(), path_factory_->NewBlobPath(), resources.writer_builder)); diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp index ceb9ca23..0622bb02 100644 --- a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp +++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp @@ -50,6 +50,8 @@ ShreddingAppendDataFileWriterFactory::CreateWriter() const { if (!shredding_context_) { return Status::Invalid("Shared-shredding append writer requires a shredding context."); } + std::shared_ptr seq_num_counter = + options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, MapSharedShreddingBatchConverter::Create( write_schema_, shredding_context_, options_, pool_)); @@ -64,7 +66,7 @@ ShreddingAppendDataFileWriterFactory::CreateWriter() const { CreateWriterResources(*options_.GetFileFormat(), file_schema, /*create_stats_extractor=*/true)); auto writer = std::make_unique( - options_.GetFileCompression(), std::move(batch_converter), schema_id_, seq_num_counter_, + options_.GetFileCompression(), std::move(batch_converter), schema_id_, seq_num_counter, file_source_, resources.stats_extractor, path_factory_->IsExternalPath(), write_cols_, pool_); PAIMON_RETURN_NOT_OK( diff --git a/src/paimon/core/manifest/manifest_committable.h b/src/paimon/core/manifest/manifest_committable.h index 8ba2b8bb..0fe72faf 100644 --- a/src/paimon/core/manifest/manifest_committable.h +++ b/src/paimon/core/manifest/manifest_committable.h @@ -42,15 +42,13 @@ class ManifestCommittable { : ManifestCommittable(identifier, std::nullopt) {} ManifestCommittable(int64_t identifier, std::optional watermark) - : ManifestCommittable(identifier, watermark, {}, {}, {}) {} + : ManifestCommittable(identifier, watermark, {}, {}) {} ManifestCommittable(int64_t identifier, std::optional watermark, - const std::map& log_offsets, const std::map& properties, const std::vector>& commit_messages) : identifier_(identifier), watermark_(watermark), - log_offsets_(log_offsets), properties_(properties), commit_messages_(commit_messages) {} @@ -62,10 +60,6 @@ class ManifestCommittable { return watermark_; } - const std::map& LogOffsets() const { - return log_offsets_; - } - const std::map& Properties() const { return properties_; } @@ -88,12 +82,6 @@ class ManifestCommittable { std::string watermark_str = watermark_ == std::nullopt ? "null" : std::to_string(watermark_.value()); - std::vector log_offsets_str; - log_offsets_str.reserve(log_offsets_.size()); - for (const auto& [key, value] : log_offsets_) { - log_offsets_str.emplace_back(fmt::format("{}: {}", key, value)); - } - std::vector properties_str; properties_str.reserve(properties_.size()); for (const auto& [key, value] : properties_) { @@ -101,16 +89,15 @@ class ManifestCommittable { } return fmt::format( - "ManifestCommittable {{identifier = {}, watermark = {}, logOffsets = {}, " + "ManifestCommittable {{identifier = {}, watermark = {}, " "commitMessages = {}, properties = {}}}", - identifier_, watermark_str, fmt::join(log_offsets_str, ", "), - fmt::join(commit_messages_str, ", "), fmt::join(properties_str, ", ")); + identifier_, watermark_str, fmt::join(commit_messages_str, ", "), + fmt::join(properties_str, ", ")); } private: int64_t identifier_; std::optional watermark_; - std::map log_offsets_; std::map properties_; std::vector> commit_messages_; }; diff --git a/src/paimon/core/manifest/manifest_committable_test.cpp b/src/paimon/core/manifest/manifest_committable_test.cpp index 886ccedf..00a8a30d 100644 --- a/src/paimon/core/manifest/manifest_committable_test.cpp +++ b/src/paimon/core/manifest/manifest_committable_test.cpp @@ -31,28 +31,6 @@ namespace paimon::test { class ManifestCommittableTest : public testing::Test { private: - bool IsEqualMap(const std::map& actual_map, - const std::map& expected_map) { - if (expected_map.size() != actual_map.size()) { - return false; - } - for (const auto& kv : expected_map) { - const auto& key = kv.first; - const auto& value = kv.second; - auto iter = actual_map.find(key); - if (iter != actual_map.end()) { - if (iter->second == value) { - continue; - } else { - return false; - } - } else { - return false; - } - } - return true; - } - std::vector> GetCommitMessages(const std::string& path, int32_t version) const { auto file_system = std::make_shared(); @@ -101,26 +79,22 @@ TEST_F(ManifestCommittableTest, TestSimple) { ASSERT_EQ(committable.Watermark().value(), 456); } { - std::map log_offsets = {{123, 444}, {234, 555}}; std::map properties = {}; std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); - ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, log_offsets, - properties, msgs); - ASSERT_TRUE(IsEqualMap(committable.LogOffsets(), log_offsets)); + ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, properties, msgs); + ASSERT_EQ(committable.Properties(), properties); ASSERT_TRUE(IsEqualMsgs(msgs, committable.FileCommittables())); } { - std::map log_offsets = {}; std::map properties = {{"key1", "value1"}, {"key2", "value2"}}; std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); - ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, - /*log_offsets=*/{}, properties, msgs); + ManifestCommittable committable(/*identifier=*/123, /*watermark=*/456, properties, msgs); ASSERT_EQ(committable.Properties(), properties); ASSERT_TRUE(IsEqualMsgs(msgs, committable.FileCommittables())); } diff --git a/src/paimon/core/operation/abstract_file_store_write.cpp b/src/paimon/core/operation/abstract_file_store_write.cpp index 2b2cda88..d7dafa18 100644 --- a/src/paimon/core/operation/abstract_file_store_write.cpp +++ b/src/paimon/core/operation/abstract_file_store_write.cpp @@ -283,7 +283,8 @@ int32_t AbstractFileStoreWrite::GetDefaultBucketNum() const { Result> AbstractFileStoreWrite::ScanExistingFileMetas( const BinaryRow& partition, int32_t bucket) const { - PAIMON_ASSIGN_OR_RAISE(auto part_values, + std::vector> part_values; + PAIMON_ASSIGN_OR_RAISE(part_values, file_store_path_factory_->GeneratePartitionVector(partition)); std::map part_values_map; for (const auto& [key, value] : part_values) { diff --git a/src/paimon/core/operation/commit/commit_changes_provider.cpp b/src/paimon/core/operation/commit/commit_changes_provider.cpp new file mode 100644 index 00000000..019b2e1d --- /dev/null +++ b/src/paimon/core/operation/commit/commit_changes_provider.cpp @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/commit_changes_provider.h" + +#include + +namespace paimon { + +namespace { + +class FixedInputCommitChangesProvider final : public CommitChangesProvider { + public: + FixedInputCommitChangesProvider(std::vector delta_files, + std::vector changelog_files, + std::vector index_entries) + : commit_changes_(std::make_shared( + std::move(delta_files), std::move(changelog_files), std::move(index_entries))) {} + + Result> Provide(const std::optional&) const override { + return commit_changes_; + } + + private: + std::shared_ptr commit_changes_; +}; + +} // namespace + +std::shared_ptr CommitChangesProvider::Provider( + std::vector delta_files, std::vector changelog_files, + std::vector index_entries) { + return std::make_shared( + std::move(delta_files), std::move(changelog_files), std::move(index_entries)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/commit_changes_provider.h b/src/paimon/core/operation/commit/commit_changes_provider.h new file mode 100644 index 00000000..ef23eb0d --- /dev/null +++ b/src/paimon/core/operation/commit/commit_changes_provider.h @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/snapshot.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +struct CommitChanges { + CommitChanges() = default; + + CommitChanges(std::vector delta, std::vector changelog, + std::vector index) + : delta_files(std::move(delta)), + changelog_files(std::move(changelog)), + index_entries(std::move(index)) {} + + std::vector delta_files; + std::vector changelog_files; + std::vector index_entries; +}; + +class CommitChangesProvider { + public: + virtual ~CommitChangesProvider() = default; + + static std::shared_ptr Provider( + std::vector delta_files, std::vector changelog_files, + std::vector index_entries); + + virtual Result> Provide( + const std::optional& latest_snapshot) const = 0; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/commit_changes_provider_test.cpp b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp new file mode 100644 index 00000000..43de67b5 --- /dev/null +++ b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/commit_changes_provider.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +BinaryRow CreateIntRow(int32_t value) { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; +} + +ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& kind, + int32_t partition_value) { + auto file_meta = std::make_shared( + file_name, /*file_size=*/1024, /*row_count=*/8, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_seq_no=*/0, + /*max_seq_no=*/0, + /*schema_id=*/0, /*level=*/0, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + + return ManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, /*total_buckets=*/1, + file_meta); +} + +IndexManifestEntry CreateIndexEntry(const std::string& file_name, int32_t partition_value) { + auto index_file = std::make_shared( + /*index_type=*/"HASH", file_name, /*file_size=*/10, /*row_count=*/1, + /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, + /*global_index_meta=*/std::nullopt); + + return IndexManifestEntry(FileKind::Add(), CreateIntRow(partition_value), /*bucket=*/0, + index_file); +} + +} // namespace + +TEST(CommitChangesProviderTest, TestProvideReturnsGivenEntries) { + std::vector delta_files = { + CreateManifestEntry("delta-1", FileKind::Add(), /*partition_value=*/1)}; + std::vector changelog_files = { + CreateManifestEntry("changelog-1", FileKind::Add(), /*partition_value=*/2)}; + std::vector index_entries = { + CreateIndexEntry("index-1", /*partition_value=*/3)}; + + std::shared_ptr provider = + CommitChangesProvider::Provider(delta_files, changelog_files, index_entries); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr provided, provider->Provide(std::nullopt)); + + const std::vector& provided_delta = provided->delta_files; + const std::vector& provided_changelog = provided->changelog_files; + const std::vector& provided_index = provided->index_entries; + + ASSERT_EQ(delta_files.size(), provided_delta.size()); + ASSERT_EQ(changelog_files.size(), provided_changelog.size()); + ASSERT_EQ(index_entries.size(), provided_index.size()); + EXPECT_EQ("delta-1", provided_delta[0].FileName()); + EXPECT_EQ("changelog-1", provided_changelog[0].FileName()); + EXPECT_EQ("index-1", provided_index[0].index_file->FileName()); +} + +TEST(CommitChangesProviderTest, TestProvideUsesCopiedInputs) { + std::vector delta_files = { + CreateManifestEntry("delta-1", FileKind::Add(), /*partition_value=*/1)}; + std::vector changelog_files; + std::vector index_entries; + + std::shared_ptr provider = + CommitChangesProvider::Provider(delta_files, changelog_files, index_entries); + + delta_files.push_back(CreateManifestEntry("delta-2", FileKind::Add(), /*partition_value=*/2)); + changelog_files.push_back( + CreateManifestEntry("changelog-2", FileKind::Add(), /*partition_value=*/3)); + index_entries.push_back(CreateIndexEntry("index-2", /*partition_value=*/4)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr provided, provider->Provide(std::nullopt)); + + ASSERT_EQ(1u, provided->delta_files.size()); + ASSERT_EQ(0u, provided->changelog_files.size()); + ASSERT_EQ(0u, provided->index_entries.size()); + EXPECT_EQ("delta-1", provided->delta_files[0].FileName()); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/commit_scanner.cpp b/src/paimon/core/operation/commit/commit_scanner.cpp new file mode 100644 index 00000000..abee7695 --- /dev/null +++ b/src/paimon/core/operation/commit/commit_scanner.cpp @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/commit_scanner.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/core/core_options.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/manifest/manifest_file_meta.h" +#include "paimon/core/operation/commit/overwrite_changes_provider.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/scan_context.h" + +namespace paimon { + +CommitScanner::CommitScanner(const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& index_manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& schema, + const CoreOptions& core_options, + const std::shared_ptr& executor, + const std::shared_ptr& pool, + const BinaryRowPartitionComputer* partition_computer, + ScanSupplier scan_supplier) + : snapshot_manager_(snapshot_manager), + schema_manager_(schema_manager), + manifest_list_(manifest_list), + manifest_file_(manifest_file), + index_manifest_file_(index_manifest_file), + table_schema_(table_schema), + schema_(schema), + core_options_(core_options), + executor_(executor), + pool_(pool), + partition_computer_(partition_computer), + scan_supplier_(std::move(scan_supplier)) {} + +Result>> CommitScanner::ToPartitionFilters( + const std::vector& changed_partitions) const { + std::vector> partition_filters; + partition_filters.reserve(changed_partitions.size()); + + for (const BinaryRow& changed_partition : changed_partitions) { + std::vector> part_values; + PAIMON_ASSIGN_OR_RAISE(part_values, + partition_computer_->GeneratePartitionVector(changed_partition)); + std::map partition_filter; + for (const auto& [key, value] : part_values) { + partition_filter[key] = value; + } + partition_filters.push_back(std::move(partition_filter)); + } + + return partition_filters; +} + +Result> CommitScanner::ReadAllEntriesFromChangedPartitions( + const Snapshot& snapshot, const std::vector& changed_partitions) const { + if (changed_partitions.empty()) { + return std::vector{}; + } + + std::vector> partition_filters; + PAIMON_ASSIGN_OR_RAISE(partition_filters, ToPartitionFilters(changed_partitions)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + NewScan(partition_filters, /*for_overwrite=*/false)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + scan->WithSnapshot(snapshot)->WithKind(ScanMode::ALL)->CreatePlan()); + return plan->Files(); +} + +Result> CommitScanner::ReadIncrementalEntries( + const Snapshot& snapshot, const std::vector& changed_partitions) const { + if (changed_partitions.empty()) { + return std::vector{}; + } + + std::unordered_set changed_partition_set(changed_partitions.begin(), + changed_partitions.end()); + std::vector delta_manifests; + PAIMON_RETURN_NOT_OK(manifest_list_->ReadDeltaManifests(snapshot, &delta_manifests)); + + std::vector incremental_entries; + for (const ManifestFileMeta& manifest_meta : delta_manifests) { + std::vector manifest_entries; + PAIMON_RETURN_NOT_OK( + manifest_file_->Read(manifest_meta.FileName(), /*filter=*/nullptr, &manifest_entries)); + for (const ManifestEntry& entry : manifest_entries) { + if (changed_partition_set.find(entry.Partition()) != changed_partition_set.end()) { + incremental_entries.push_back(entry); + } + } + } + + return incremental_entries; +} + +Result> CommitScanner::ReadAllEntriesFromPartitions( + const Snapshot& snapshot, + const std::vector>& partitions) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + NewScan(partitions, /*for_overwrite=*/false)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + scan->WithSnapshot(snapshot)->WithKind(ScanMode::ALL)->CreatePlan()); + return plan->Files(); +} + +Result> CommitScanner::NewScan( + const std::vector>& partitions, bool for_overwrite) const { + auto scan_filter = std::make_shared(/*predicate=*/nullptr, partitions, + /*bucket_filter=*/std::nullopt); + if (!scan_supplier_) { + return Status::Invalid("CommitScanner requires non-empty scan supplier."); + } + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, scan_supplier_(scan_filter)); + if (for_overwrite && core_options_.GetBucket() != BucketModeDefine::POSTPONE_BUCKET) { + scan->OnlyReadRealBuckets(); + } + return scan; +} + +Result> CommitScanner::ReadAllIndexEntriesFromPartitions( + const Snapshot& snapshot, + const std::vector>& partitions) const { + std::vector index_entries; + if (!snapshot.IndexManifest()) { + return index_entries; + } + + auto filter = [this, &partitions](const IndexManifestEntry& entry) -> Result { + if (partitions.empty()) { + return true; + } + + std::vector> part_values; + PAIMON_ASSIGN_OR_RAISE(part_values, + partition_computer_->GeneratePartitionVector(entry.partition)); + std::map partition; + for (const auto& [key, value] : part_values) { + partition[key] = value; + } + + for (const auto& partition_spec : partitions) { + bool matched = true; + for (const auto& [key, value] : partition_spec) { + auto iter = partition.find(key); + if (iter == partition.end() || iter->second != value) { + matched = false; + break; + } + } + if (matched) { + return true; + } + } + return false; + }; + + PAIMON_RETURN_NOT_OK( + index_manifest_file_->Read(snapshot.IndexManifest().value(), filter, &index_entries)); + return index_entries; +} + +std::shared_ptr CommitScanner::OverwriteChangesProvider( + const std::vector>& partitions, + const std::vector& changes, + const std::vector& index_entries) const { + return std::make_shared( + changes, index_entries, + [this, partitions](const Snapshot& snapshot) -> Result> { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + NewScan(partitions, /*for_overwrite=*/true)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr plan, + scan->WithSnapshot(snapshot)->WithKind(ScanMode::ALL)->CreatePlan()); + return plan->Files(); + }, + [this, partitions](const Snapshot& snapshot) { + return ReadAllIndexEntriesFromPartitions(snapshot, partitions); + }); +} + +Result> CommitScanner::ReadTotalBuckets( + const Snapshot& snapshot, const std::vector& changed_partitions) const { + std::unordered_map total_buckets; + if (changed_partitions.empty()) { + return total_buckets; + } + + PAIMON_ASSIGN_OR_RAISE(std::vector entries, + ReadAllEntriesFromChangedPartitions(snapshot, changed_partitions)); + + std::unordered_set remaining_partitions(changed_partitions.begin(), + changed_partitions.end()); + for (const ManifestEntry& entry : entries) { + if (remaining_partitions.empty()) { + break; + } + if (!(entry.Kind() == FileKind::Add()) || entry.TotalBuckets() <= 0) { + continue; + } + if (remaining_partitions.erase(entry.Partition()) > 0) { + total_buckets.emplace(entry.Partition(), entry.TotalBuckets()); + } + } + + return total_buckets; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/commit_scanner.h b/src/paimon/core/operation/commit/commit_scanner.h new file mode 100644 index 00000000..6704da6f --- /dev/null +++ b/src/paimon/core/operation/commit/commit_scanner.h @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/common/data/binary_row.h" +#include "paimon/core/core_options.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class BinaryRowPartitionComputer; +class CommitChangesProvider; +class Executor; +class FileStoreScan; +struct IndexManifestEntry; +class IndexManifestFile; +class ManifestEntry; +class ManifestFile; +class ManifestList; +class MemoryPool; +class ScanFilter; +class SchemaManager; +class Snapshot; +class SnapshotManager; +class TableSchema; + +// Manifest entries scanner for commit operations. +class CommitScanner { + public: + using ScanSupplier = + std::function>(const std::shared_ptr&)>; + + CommitScanner(const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& index_manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& schema, const CoreOptions& core_options, + const std::shared_ptr& executor, + const std::shared_ptr& pool, + const BinaryRowPartitionComputer* partition_computer, ScanSupplier scan_supplier); + + Result> ReadAllEntriesFromChangedPartitions( + const Snapshot& snapshot, const std::vector& changed_partitions) const; + + Result> ReadIncrementalEntries( + const Snapshot& snapshot, const std::vector& changed_partitions) const; + + Result> ReadTotalBuckets( + const Snapshot& snapshot, const std::vector& changed_partitions) const; + + Result> ReadAllEntriesFromPartitions( + const Snapshot& snapshot, + const std::vector>& partitions) const; + + Result> ReadAllIndexEntriesFromPartitions( + const Snapshot& snapshot, + const std::vector>& partitions) const; + + std::shared_ptr OverwriteChangesProvider( + const std::vector>& partitions, + const std::vector& changes, + const std::vector& index_entries) const; + + private: + Result>> ToPartitionFilters( + const std::vector& changed_partitions) const; + + Result> NewScan( + const std::vector>& partitions, + bool for_overwrite) const; + + private: + std::shared_ptr snapshot_manager_; + std::shared_ptr schema_manager_; + std::shared_ptr manifest_list_; + std::shared_ptr manifest_file_; + std::shared_ptr index_manifest_file_; + std::shared_ptr table_schema_; + std::shared_ptr schema_; + CoreOptions core_options_; + std::shared_ptr executor_; + std::shared_ptr pool_; + const BinaryRowPartitionComputer* partition_computer_; + ScanSupplier scan_supplier_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/commit_scanner_test.cpp b/src/paimon/core/operation/commit/commit_scanner_test.cpp new file mode 100644 index 00000000..4d9f69bd --- /dev/null +++ b/src/paimon/core/operation/commit/commit_scanner_test.cpp @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/commit_scanner.h" + +#include +#include +#include +#include +#include + +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/core/core_options.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/snapshot.h" +#include "paimon/scan_context.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +Snapshot MakeSnapshot() { + return Snapshot( + /*id=*/1, + /*schema_id=*/0, + /*base_manifest_list=*/"base-manifest-list", + /*base_manifest_list_size=*/std::nullopt, + /*delta_manifest_list=*/"delta-manifest-list", + /*delta_manifest_list_size=*/std::nullopt, + /*changelog_manifest_list=*/std::nullopt, + /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, + /*commit_user=*/"test-user", + /*commit_identifier=*/1, Snapshot::CommitKind::Append(), + /*time_millis=*/0, + /*total_record_count=*/0, + /*delta_record_count=*/0, + /*changelog_record_count=*/std::nullopt, + /*watermark=*/std::nullopt, + /*statistics=*/std::nullopt, + /*properties=*/std::nullopt, + /*next_row_id=*/std::nullopt); +} + +BinaryRow CreateIntPartition(int32_t value) { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; +} + +} // namespace + +class CommitScannerTest : public testing::Test { + protected: + void SetUp() override { + schema_ = arrow::schema({arrow::field("pt", arrow::int32())}); + ASSERT_OK_AND_ASSIGN(core_options_, CoreOptions::FromMap({})); + ASSERT_OK_AND_ASSIGN(partition_computer_, + BinaryRowPartitionComputer::Create( + /*partition_keys=*/{"pt"}, schema_, + /*default_part_value=*/"__DEFAULT_PARTITION__", + /*legacy_partition_name_enabled=*/true, GetDefaultPool())); + } + + CommitScanner CreateScanner(CommitScanner::ScanSupplier scan_supplier) const { + return CommitScanner( + /*snapshot_manager=*/nullptr, + /*schema_manager=*/nullptr, + /*manifest_list=*/nullptr, + /*manifest_file=*/nullptr, + /*index_manifest_file=*/nullptr, + /*table_schema=*/nullptr, schema_, core_options_, + /*executor=*/nullptr, GetDefaultPool(), partition_computer_.get(), + std::move(scan_supplier)); + } + + protected: + std::shared_ptr schema_; + CoreOptions core_options_; + std::unique_ptr partition_computer_; +}; + +TEST_F(CommitScannerTest, TestReadAllEntriesFromChangedPartitionsEmptyFastExit) { + bool supplier_called = false; + CommitScanner scanner = CreateScanner([&supplier_called](const std::shared_ptr&) + -> Result> { + supplier_called = true; + return Status::Invalid("should not be called"); + }); + + ASSERT_OK_AND_ASSIGN(std::vector entries, + scanner.ReadAllEntriesFromChangedPartitions(MakeSnapshot(), + /*changed_partitions=*/{})); + EXPECT_TRUE(entries.empty()); + EXPECT_FALSE(supplier_called); +} + +TEST_F(CommitScannerTest, TestReadAllEntriesFromPartitionsRequiresSupplier) { + CommitScanner scanner = CreateScanner(CommitScanner::ScanSupplier{}); + + std::vector> partitions = {{{"pt", "1"}}}; + ASSERT_NOK_WITH_MSG(scanner.ReadAllEntriesFromPartitions(MakeSnapshot(), partitions), + "CommitScanner requires non-empty scan supplier"); +} + +TEST_F(CommitScannerTest, TestReadAllEntriesFromChangedPartitionsBuildsScanFilterPartitions) { + std::vector> captured_partition_filters; + bool supplier_called = false; + + CommitScanner scanner = CreateScanner( + [&captured_partition_filters, &supplier_called]( + const std::shared_ptr& filter) -> Result> { + supplier_called = true; + captured_partition_filters = filter->GetPartitionFilters(); + return Status::Invalid("stop after capturing filter"); + }); + + std::vector changed_partitions = {CreateIntPartition(42)}; + ASSERT_NOK_WITH_MSG( + scanner.ReadAllEntriesFromChangedPartitions(MakeSnapshot(), changed_partitions), + "stop after capturing filter"); + + ASSERT_TRUE(supplier_called); + ASSERT_EQ(1u, captured_partition_filters.size()); + ASSERT_EQ(1u, captured_partition_filters[0].size()); + EXPECT_EQ("42", captured_partition_filters[0]["pt"]); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/compacted_changelog_path_resolver.cpp b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.cpp new file mode 100644 index 00000000..4e5b4289 --- /dev/null +++ b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.cpp @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/compacted_changelog_path_resolver.h" + +#include +#include + +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" + +namespace paimon { + +bool CompactedChangelogPathResolver::IsCompactedChangelogPath(const std::string& path) { + const std::string file_name = PathUtil::GetName(path); + return StringUtils::StartsWith(file_name, "compacted-changelog-"); +} + +std::string CompactedChangelogPathResolver::Resolve(const std::string& path) { + if (!IsCompactedChangelogPath(path)) { + return path; + } + const std::string file_name = PathUtil::GetName(path); + + const size_t dot_pos = file_name.find_last_of('.'); + if (dot_pos == std::string::npos || dot_pos + 1 >= file_name.size()) { + return path; + } + + const std::string name_without_ext = file_name.substr(0, dot_pos); + const std::string format = file_name.substr(dot_pos + 1); + const size_t dollar_pos = name_without_ext.find('$'); + if (dollar_pos == std::string::npos || dollar_pos + 1 >= name_without_ext.size()) { + return path; + } + + const std::string base_name = name_without_ext.substr(0, dollar_pos); + const std::string suffix = name_without_ext.substr(dollar_pos + 1); + const std::vector split_tokens = StringUtils::Split(suffix, "-", false); + + // Real compacted changelog path pattern: ...$bucket-len.ext + if (split_tokens.size() == 2) { + return path; + } + + // Fake compacted changelog path pattern: ...$bucket-len-offset-sliceLen.ext + if (split_tokens.size() < 4) { + return path; + } + + const std::string& bucket = split_tokens[0]; + const std::string& total_len = split_tokens[1]; + const std::string real_file_name = base_name + "$" + bucket + "-" + total_len + "." + format; + + const std::string parent = PathUtil::GetParentDirPath(path); + const std::string grand_parent = PathUtil::GetParentDirPath(parent); + if (parent.empty() || grand_parent.empty()) { + return path; + } + + return PathUtil::JoinPath(PathUtil::JoinPath(grand_parent, "bucket-" + bucket), real_file_name); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h new file mode 100644 index 00000000..b9232874 --- /dev/null +++ b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +namespace paimon { + +class CompactedChangelogPathResolver { + public: + static bool IsCompactedChangelogPath(const std::string& path); + + static std::string Resolve(const std::string& path); +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/compacted_changelog_path_resolver_test.cpp b/src/paimon/core/operation/commit/compacted_changelog_path_resolver_test.cpp new file mode 100644 index 00000000..ef9bfc5a --- /dev/null +++ b/src/paimon/core/operation/commit/compacted_changelog_path_resolver_test.cpp @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/compacted_changelog_path_resolver.h" + +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(CompactedChangelogPathResolverTest, IsCompactedChangelogPath) { + std::string regular_changelog = + "table/bucket-0/changelog-25b05ab0-6f90-4865-a984-8d9629bac735-1426.parquet"; + std::string compacted_changelog = + "table/bucket-0/" + "compacted-changelog-8e049c65-5ce4-4ce7-b1b0-78ce694ab351$0-39253.cc-parquet"; + std::string data_file = "table/bucket-0/data-file-1.parquet"; + + ASSERT_FALSE(CompactedChangelogPathResolver::IsCompactedChangelogPath(regular_changelog)); + ASSERT_TRUE(CompactedChangelogPathResolver::IsCompactedChangelogPath(compacted_changelog)); + ASSERT_FALSE(CompactedChangelogPathResolver::IsCompactedChangelogPath(data_file)); +} + +TEST(CompactedChangelogPathResolverTest, ResolveNonCompactedChangelogPath) { + std::string regular_changelog = + "table/bucket-0/changelog-25b05ab0-6f90-4865-a984-8d9629bac735-1426.parquet"; + + ASSERT_EQ(regular_changelog, CompactedChangelogPathResolver::Resolve(regular_changelog)); +} + +TEST(CompactedChangelogPathResolverTest, ResolveFakeCompactedChangelogPath) { + std::string fake_path = + "table/f1=10/bucket-1/compacted-changelog-abc$0-39253-39253-35699.cc-parquet"; + std::string expected_real_path = + "table/f1=10/bucket-0/compacted-changelog-abc$0-39253.cc-parquet"; + + ASSERT_EQ(expected_real_path, CompactedChangelogPathResolver::Resolve(fake_path)); +} + +TEST(CompactedChangelogPathResolverTest, KeepRealCompactedChangelogPath) { + std::string real_path = "table/f1=10/bucket-1/compacted-changelog-abc$1-39253.cc-parquet"; + + ASSERT_EQ(real_path, CompactedChangelogPathResolver::Resolve(real_path)); +} + +TEST(CompactedChangelogPathResolverTest, KeepNonCompactedPath) { + std::string normal_path = "table/f1=10/bucket-1/data-file.orc"; + + ASSERT_EQ(normal_path, CompactedChangelogPathResolver::Resolve(normal_path)); +} + +TEST(CompactedChangelogPathResolverTest, KeepInvalidCompactedPath) { + std::string invalid_path = "table/f1=10/bucket-1/compacted-changelog-abc$1.cc-parquet"; + + ASSERT_EQ(invalid_path, CompactedChangelogPathResolver::Resolve(invalid_path)); +} + +TEST(CompactedChangelogPathResolverTest, ResolveWithDifferentFormats) { + std::string fake_orc_path = + "table/f1=10/bucket-2/compacted-changelog-abc$0-1024-1024-512.cc-orc"; + std::string expected_orc_path = "table/f1=10/bucket-0/compacted-changelog-abc$0-1024.cc-orc"; + ASSERT_EQ(expected_orc_path, CompactedChangelogPathResolver::Resolve(fake_orc_path)); + + std::string fake_avro_path = + "table/f1=10/bucket-5/compacted-changelog-abc$2-2048-2048-1024.cc-avro"; + std::string expected_avro_path = "table/f1=10/bucket-2/compacted-changelog-abc$2-2048.cc-avro"; + ASSERT_EQ(expected_avro_path, CompactedChangelogPathResolver::Resolve(fake_avro_path)); +} + +TEST(CompactedChangelogPathResolverTest, ResolveFileWithoutExtension) { + std::string file_without_extension = "table/f1=10/bucket-0/file"; + + ASSERT_EQ(file_without_extension, + CompactedChangelogPathResolver::Resolve(file_without_extension)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/conflict_detection.cpp b/src/paimon/core/operation/commit/conflict_detection.cpp new file mode 100644 index 00000000..7ad14464 --- /dev/null +++ b/src/paimon/core/operation/commit/conflict_detection.cpp @@ -0,0 +1,613 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/conflict_detection.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/range_helper.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" +#include "paimon/core/manifest/file_entry.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_file_meta.h" +#include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/commit/manifest_entry_changes.h" +#include "paimon/core/operation/commit/row_id_column_conflict_checker.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/utils/range.h" +#include "paimon/utils/row_range_index.h" + +namespace paimon { + +namespace { + +bool IsVectorStoreFile(const std::string& file_name) { + return file_name.find(".vector.") != std::string::npos; +} + +bool IsDedicatedStorageFile(const std::string& file_name) { + return BlobUtils::IsBlobFile(file_name) || IsVectorStoreFile(file_name); +} + +struct PartitionBucketKey { + BinaryRow partition; + int32_t bucket; + + bool operator==(const PartitionBucketKey& other) const { + return partition == other.partition && bucket == other.bucket; + } +}; + +struct PartitionBucketKeyHash { + size_t operator()(const PartitionBucketKey& key) const { + return std::hash()(key.partition) ^ (std::hash()(key.bucket) << 1); + } +}; + +} // namespace + +ConflictDetection::ConflictDetection(std::shared_ptr table_schema, + const CoreOptions& options, + std::shared_ptr snapshot_manager, + std::shared_ptr manifest_list, + std::shared_ptr manifest_file, + std::shared_ptr commit_scanner) + : table_schema_(std::move(table_schema)), + options_(options), + snapshot_manager_(std::move(snapshot_manager)), + manifest_list_(std::move(manifest_list)), + manifest_file_(std::move(manifest_file)), + commit_scanner_(std::move(commit_scanner)) {} + +void ConflictDetection::SetRowIdCheckFromSnapshot( + const std::optional& row_id_check_from_snapshot) { + row_id_check_from_snapshot_ = row_id_check_from_snapshot; +} + +bool ConflictDetection::HasRowIdCheckFromSnapshot() const { + return row_id_check_from_snapshot_.has_value(); +} + +Status ConflictDetection::CheckConflicts( + const Snapshot& latest_snapshot, const std::vector& base_entries, + const std::vector& delta_entries, + const std::vector& delta_index_entries, + const std::optional>& + row_id_column_conflict_checker, + const Snapshot::CommitKind& commit_kind) const { + if (options_.DeletionVectorsEnabled() && + ResolveBucketMode(options_.GetBucket(), table_schema_) == BucketMode::BUCKET_UNAWARE) { + return Status::NotImplemented( + "check conflicts failed. not yet support dv with BUCKET_UNAWARE mode"); + } + + std::vector all_entries = base_entries; + all_entries.insert(all_entries.end(), delta_entries.begin(), delta_entries.end()); + PAIMON_RETURN_NOT_OK(CheckBucketKeepSame(all_entries, commit_kind)); + + // check the delta, it is important not to delete and add the same file. Since scan + // relies on map for deduplication, this may result in the loss of this file + std::vector merged_delta_entries; + PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(delta_entries, &merged_delta_entries)); + + std::vector merged_entries; + // merge manifest entries and also check if the files we want to delete are still there + PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(all_entries, &merged_entries)); + PAIMON_RETURN_NOT_OK(CheckDeleteInEntries(merged_entries)); + PAIMON_RETURN_NOT_OK(CheckKeyRange(merged_entries)); + if (commit_kind != Snapshot::CommitKind::Compact()) { + PAIMON_RETURN_NOT_OK( + CheckRowIdExistence(base_entries, delta_entries, latest_snapshot.NextRowId())); + } + PAIMON_RETURN_NOT_OK(CheckRowIdRangeConflicts(commit_kind, merged_entries)); + PAIMON_RETURN_NOT_OK(CheckGlobalIndexRowIdExistence(base_entries, delta_index_entries)); + PAIMON_RETURN_NOT_OK(CheckForRowIdFromSnapshot( + latest_snapshot, delta_entries, delta_index_entries, row_id_column_conflict_checker)); + return Status::OK(); +} + +bool ConflictDetection::ShouldBeOverwriteCommit( + const std::vector& append_table_files, + const std::vector& append_index_files) const { + for (const ManifestEntry& entry : append_table_files) { + if (entry.Kind() == FileKind::Delete()) { + return true; + } + } + + for (const IndexManifestEntry& entry : append_index_files) { + if (entry.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX) { + return true; + } + } + + return false; +} + +Status ConflictDetection::CheckBucketKeepSame(const std::vector& all_entries, + const Snapshot::CommitKind& commit_kind) const { + if (commit_kind == Snapshot::CommitKind::Overwrite()) { + return Status::OK(); + } + + // total buckets within the same partition should remain the same + std::unordered_map total_buckets; + for (const ManifestEntry& entry : all_entries) { + if (entry.TotalBuckets() <= 0) { + continue; + } + if (same_bucket_checked_partitions_.find(entry.Partition()) != + same_bucket_checked_partitions_.end()) { + continue; + } + + auto [iter, inserted] = total_buckets.emplace(entry.Partition(), entry.TotalBuckets()); + if (inserted || iter->second == entry.TotalBuckets()) { + continue; + } + + return BucketNumMismatch(entry.Partition(), entry.TotalBuckets(), iter->second); + } + + MarkBucketCheckedPartitions(total_buckets); + return Status::OK(); +} + +Status ConflictDetection::CollectUncheckedBucketPartitions( + const std::vector& delta_entries, + std::unordered_map* total_buckets) const { + total_buckets->clear(); + for (const ManifestEntry& entry : delta_entries) { + if (!(entry.Kind() == FileKind::Add()) || entry.TotalBuckets() <= 0 || + same_bucket_checked_partitions_.find(entry.Partition()) != + same_bucket_checked_partitions_.end()) { + continue; + } + + auto [iter, inserted] = total_buckets->emplace(entry.Partition(), entry.TotalBuckets()); + if (!inserted && iter->second != entry.TotalBuckets()) { + return BucketNumMismatch(entry.Partition(), entry.TotalBuckets(), iter->second); + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckSameBucketByTotalBuckets( + const std::unordered_map& expected_total_buckets, + const std::unordered_map& previous_total_buckets) const { + for (const auto& [partition, total_buckets] : expected_total_buckets) { + auto iter = previous_total_buckets.find(partition); + if (iter != previous_total_buckets.end() && iter->second != total_buckets) { + return BucketNumMismatch(partition, total_buckets, iter->second); + } + } + + MarkBucketCheckedPartitions(expected_total_buckets); + return Status::OK(); +} + +Status ConflictDetection::BucketNumMismatch(const BinaryRow& partition, int32_t num_buckets, + int32_t previous_num_buckets) const { + return Status::Invalid(fmt::format( + "Total buckets of partition {} changed from {} to {} without overwrite. Give up " + "committing.", + partition.ToString(), previous_num_buckets, num_buckets)); +} + +void ConflictDetection::MarkBucketCheckedPartitions( + const std::unordered_map& total_buckets) const { + if (total_buckets.empty()) { + return; + } + + for (const auto& [partition, _] : total_buckets) { + same_bucket_checked_partitions_.insert_or_assign(partition, true); + while (same_bucket_checked_partitions_.size() > kSameBucketCheckCacheMaxSize) { + same_bucket_checked_partitions_.erase(same_bucket_checked_partitions_.begin()->first); + } + } +} + +Status ConflictDetection::CheckDeleteInEntries( + const std::vector& merged_entries) const { + for (const auto& entry : merged_entries) { + if (entry.Kind() == FileKind::Delete()) { + return Status::Invalid(fmt::format( + "Trying to delete file {} which is not previously added.", entry.FileName())); + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckKeyRange(const std::vector& merged_entries) const { + if (table_schema_->PrimaryKeys().empty()) { + return Status::OK(); + } + + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + table_schema_->TrimmedPrimaryKeyFields()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options_.SequenceFieldSortOrderIsAscending())); + + // group entries by partitions, buckets and levels + std::unordered_map, std::vector> levels; + for (const auto& entry : merged_entries) { + if (!(entry.Kind() == FileKind::Add())) { + continue; + } + int32_t level = entry.Level(); + if (level < 1) { + continue; + } + + levels[std::make_tuple(entry.Partition(), entry.Bucket(), level)].push_back(entry); + } + + // check for all LSM level >= 1, key ranges of files do not intersect + for (auto& [_, entries] : levels) { + std::sort(entries.begin(), entries.end(), + [&key_comparator](const ManifestEntry& a, const ManifestEntry& b) { + return key_comparator->CompareTo(a.MinKey(), b.MinKey()) < 0; + }); + for (size_t i = 0; i + 1 < entries.size(); ++i) { + const ManifestEntry& a = entries[i]; + const ManifestEntry& b = entries[i + 1]; + if (key_comparator->CompareTo(a.MaxKey(), b.MinKey()) >= 0) { + return Status::Invalid(fmt::format( + "LSM conflicts detected! Give up committing. Conflict files are {} and {}.", + a.FileName(), b.FileName())); + } + } + } + return Status::OK(); +} + +Status ConflictDetection::CheckRowIdExistence(const std::vector& base_entries, + const std::vector& delta_entries, + const std::optional& next_row_id) const { + if (!options_.DataEvolutionEnabled()) { + return Status::OK(); + } + + std::vector files_to_check; + files_to_check.reserve(delta_entries.size()); + for (const ManifestEntry& entry : delta_entries) { + if (!(entry.Kind() == FileKind::Add()) || !entry.File()->first_row_id || !next_row_id || + entry.File()->first_row_id.value() >= next_row_id.value()) { + continue; + } + files_to_check.push_back(entry); + } + if (files_to_check.empty()) { + return Status::OK(); + } + + std::vector existing_data_ranges; + existing_data_ranges.reserve(base_entries.size()); + for (const ManifestEntry& entry : base_entries) { + if (!entry.File()->first_row_id || IsDedicatedStorageFile(entry.FileName())) { + continue; + } + int64_t range_from = entry.File()->first_row_id.value(); + int64_t range_to = range_from + entry.File()->row_count - 1; + existing_data_ranges.emplace_back(range_from, range_to); + } + + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex existing_index, + RowRangeIndex::Create(existing_data_ranges, + /*merge_adjacent=*/false)); + + for (const ManifestEntry& entry : files_to_check) { + int64_t range_from = entry.File()->first_row_id.value(); + int64_t range_to = range_from + entry.File()->row_count - 1; + Range row_range(range_from, range_to); + + bool exists = false; + if (IsDedicatedStorageFile(entry.FileName())) { + exists = existing_index.Contains(row_range); + } else { + exists = existing_index.ContainsExactly(row_range); + } + + if (!exists) { + return Status::Invalid(fmt::format( + "Row ID existence conflict: file '{}' references firstRowId={}, rowCount={} in " + "bucket {}, but no matching file exists in the current snapshot.", + entry.FileName(), entry.File()->first_row_id.value(), entry.File()->row_count, + entry.Bucket())); + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckRowIdRangeConflicts( + const Snapshot::CommitKind& commit_kind, + const std::vector& merged_entries) const { + if (!options_.DataEvolutionEnabled()) { + return Status::OK(); + } + if (!row_id_check_from_snapshot_ && !(commit_kind == Snapshot::CommitKind::Compact())) { + return Status::OK(); + } + + std::vector entries_with_ranges; + entries_with_ranges.reserve(merged_entries.size()); + for (const ManifestEntry& entry : merged_entries) { + if (entry.File()->first_row_id) { + entries_with_ranges.push_back(entry); + } + } + if (entries_with_ranges.empty()) { + return Status::OK(); + } + + RangeHelper range_helper( + [](const ManifestEntry& entry) -> Result { + return entry.File()->first_row_id.value(); + }, + [](const ManifestEntry& entry) -> Result { + return entry.File()->first_row_id.value() + entry.File()->row_count - 1; + }); + std::vector data_files; + std::vector dedicated_files; + data_files.reserve(entries_with_ranges.size()); + dedicated_files.reserve(entries_with_ranges.size()); + for (const ManifestEntry& entry : entries_with_ranges) { + if (IsDedicatedStorageFile(entry.FileName())) { + dedicated_files.push_back(entry); + } else { + data_files.push_back(entry); + } + } + + PAIMON_RETURN_NOT_OK(CheckDataFileRowIdRangeConflicts(range_helper, data_files)); + PAIMON_RETURN_NOT_OK(CheckDedicatedFileRowIdRangeConflicts(data_files, dedicated_files)); + + return Status::OK(); +} + +Status ConflictDetection::CheckDataFileRowIdRangeConflicts( + RangeHelper& range_helper, const std::vector& data_files) const { + std::vector data_files_copy = data_files; + PAIMON_ASSIGN_OR_RAISE(std::vector> data_file_groups, + range_helper.MergeOverlappingRanges(std::move(data_files_copy))); + for (const std::vector& data_file_group : data_file_groups) { + PAIMON_ASSIGN_OR_RAISE(bool all_data_ranges_same, + range_helper.AreAllRangesSame(data_file_group)); + if (!all_data_ranges_same) { + return Status::Invalid( + "For Data Evolution table, multiple MERGE INTO/COMPACT operations have " + "encountered row-id range conflicts."); + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckDedicatedFileRowIdRangeConflicts( + const std::vector& data_files, + const std::vector& dedicated_files) const { + if (dedicated_files.empty()) { + return Status::OK(); + } + + std::vector data_ranges; + data_ranges.reserve(data_files.size()); + for (const ManifestEntry& data_file : data_files) { + int64_t data_range_from = data_file.File()->first_row_id.value(); + int64_t data_range_to = data_range_from + data_file.File()->row_count - 1; + data_ranges.emplace_back(data_range_from, data_range_to); + } + + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex data_file_row_range_index, + RowRangeIndex::Create(data_ranges, /*merge_adjacent=*/false)); + + for (const ManifestEntry& dedicated_file : dedicated_files) { + int64_t dedicated_from = dedicated_file.File()->first_row_id.value(); + int64_t dedicated_to = dedicated_from + dedicated_file.File()->row_count - 1; + Range dedicated_range(dedicated_from, dedicated_to); + + std::vector intersecting_ranges = + data_file_row_range_index.IntersectedRanges(dedicated_range.from, dedicated_range.to); + bool covered_by_one_data_range = intersecting_ranges.size() == 1 && + intersecting_ranges[0].from <= dedicated_range.from && + intersecting_ranges[0].to >= dedicated_range.to; + if (!covered_by_one_data_range) { + std::string conflict_reason = intersecting_ranges.size() > 1 + ? "spans multiple data file ranges" + : "is not covered by one data file range"; + return Status::Invalid(fmt::format( + "For Data Evolution table, multiple MERGE INTO/COMPACT operations have " + "encountered row-id range conflicts, dedicated file '{}' range {} {}.", + dedicated_file.FileName(), dedicated_range.ToString(), conflict_reason)); + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckForRowIdFromSnapshot( + const Snapshot& latest_snapshot, const std::vector& delta_entries, + const std::vector& delta_index_entries, + const std::optional>& + row_id_column_conflict_checker) const { + if (!options_.DataEvolutionEnabled() || !row_id_check_from_snapshot_ || !snapshot_manager_ || + !row_id_column_conflict_checker || !row_id_column_conflict_checker.value() || + row_id_column_conflict_checker.value()->IsEmpty()) { + return Status::OK(); + } + + if (row_id_check_from_snapshot_.value() > latest_snapshot.Id()) { + return Status::OK(); + } + + PAIMON_ASSIGN_OR_RAISE(Snapshot check_snapshot, + snapshot_manager_->LoadSnapshot(row_id_check_from_snapshot_.value())); + if (!check_snapshot.NextRowId()) { + return Status::Invalid(fmt::format("Next row id cannot be null for snapshot {}.", + row_id_check_from_snapshot_.value())); + } + int64_t check_next_row_id = check_snapshot.NextRowId().value(); + + int64_t from_snapshot_id = row_id_check_from_snapshot_.value() + 1; + if (from_snapshot_id < Snapshot::FIRST_SNAPSHOT_ID) { + from_snapshot_id = Snapshot::FIRST_SNAPSHOT_ID; + } + if (from_snapshot_id > latest_snapshot.Id()) { + return Status::OK(); + } + + std::vector changed_partitions = + ManifestEntryChanges::ChangedPartitions(delta_entries, delta_index_entries); + if (changed_partitions.empty()) { + return Status::OK(); + } + std::unordered_set changed_partition_set(changed_partitions.begin(), + changed_partitions.end()); + + for (int64_t snapshot_id = from_snapshot_id; snapshot_id <= latest_snapshot.Id(); + ++snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id)); + if (snapshot.GetCommitKind() == Snapshot::CommitKind::Compact()) { + continue; + } + + PAIMON_ASSIGN_OR_RAISE( + std::vector history_entries, + commit_scanner_->ReadIncrementalEntries(snapshot, changed_partitions)); + for (const ManifestEntry& history_entry : history_entries) { + if (!(history_entry.Kind() == FileKind::Add()) || !history_entry.File()->first_row_id || + changed_partition_set.find(history_entry.Partition()) == + changed_partition_set.end()) { + continue; + } + int64_t history_first_row_id = history_entry.File()->first_row_id.value(); + if (history_first_row_id >= check_next_row_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE( + bool conflicts, + row_id_column_conflict_checker.value()->ConflictsWith(history_entry.File())); + if (conflicts) { + return Status::Invalid( + "For Data Evolution table, multiple MERGE INTO operations have " + "encountered conflicts while checking row-id history from " + "snapshot."); + } + } + } + + return Status::OK(); +} + +Status ConflictDetection::CheckGlobalIndexRowIdExistence( + const std::vector& base_entries, + const std::vector& delta_index_entries) const { + if (!options_.DataEvolutionEnabled()) { + return Status::OK(); + } + + std::vector indexes_to_check; + for (const IndexManifestEntry& index_entry : delta_index_entries) { + if (!(index_entry.kind == FileKind::Add()) || + !index_entry.index_file->GetGlobalIndexMeta()) { + continue; + } + indexes_to_check.push_back(index_entry); + } + if (indexes_to_check.empty()) { + return Status::OK(); + } + + std::unordered_map, PartitionBucketKeyHash> + data_ranges_by_group; + for (const ManifestEntry& base_entry : base_entries) { + if (!(base_entry.Kind() == FileKind::Add()) || !base_entry.File()->first_row_id) { + continue; + } + + int64_t first_row_id = base_entry.File()->first_row_id.value(); + int64_t last_row_id = first_row_id + base_entry.File()->row_count - 1; + data_ranges_by_group[{base_entry.Partition(), base_entry.Bucket()}].emplace_back( + first_row_id, last_row_id); + } + + std::unordered_map + range_index_by_group; + range_index_by_group.reserve(data_ranges_by_group.size()); + for (const auto& [group, data_ranges] : data_ranges_by_group) { + PAIMON_ASSIGN_OR_RAISE(RowRangeIndex row_range_index, + RowRangeIndex::Create(data_ranges, /*merge_adjacent=*/true)); + range_index_by_group.emplace(group, std::move(row_range_index)); + } + + for (const IndexManifestEntry& index_entry : indexes_to_check) { + PartitionBucketKey group_key{index_entry.partition, index_entry.bucket}; + auto group_iter = range_index_by_group.find(group_key); + if (group_iter == range_index_by_group.end()) { + return Status::Invalid(fmt::format( + "Global index row ID existence conflict: index file '{}' references row range {}, " + "but this range is not fully covered by current data files.", + index_entry.index_file->FileName(), + Range(index_entry.index_file->GetGlobalIndexMeta().value().row_range_start, + index_entry.index_file->GetGlobalIndexMeta().value().row_range_end) + .ToString())); + } + + const GlobalIndexMeta& global_index = index_entry.index_file->GetGlobalIndexMeta().value(); + Range index_range(global_index.row_range_start, global_index.row_range_end); + + std::vector intersected = + group_iter->second.IntersectedRanges(index_range.from, index_range.to); + bool covered = intersected.size() == 1 && intersected[0].from <= index_range.from && + intersected[0].to >= index_range.to; + if (!covered) { + return Status::Invalid(fmt::format( + "Global index row ID existence conflict: index file '{}' references row range {}, " + "but this range is not fully covered by current data files.", + index_entry.index_file->FileName(), index_range.ToString())); + } + } + + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/conflict_detection.h b/src/paimon/core/operation/commit/conflict_detection.h new file mode 100644 index 00000000..803da2cb --- /dev/null +++ b/src/paimon/core/operation/commit/conflict_detection.h @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/common/data/binary_row.h" +#include "paimon/common/utils/linked_hash_map.h" +#include "paimon/common/utils/range_helper.h" +#include "paimon/core/core_options.h" +#include "paimon/core/snapshot.h" +#include "paimon/status.h" + +namespace paimon { + +class ManifestEntry; +struct IndexManifestEntry; +class CommitScanner; +class ManifestFile; +class ManifestList; +class RowIdColumnConflictChecker; +class SnapshotManager; +class TableSchema; + +/// Util class for detecting conflicts between base and delta files. +class ConflictDetection { + public: + ConflictDetection(std::shared_ptr table_schema, const CoreOptions& options, + std::shared_ptr snapshot_manager, + std::shared_ptr manifest_list, + std::shared_ptr manifest_file, + std::shared_ptr commit_scanner); + + Status CheckConflicts(const Snapshot& latest_snapshot, + const std::vector& base_entries, + const std::vector& delta_entries, + const std::vector& delta_index_entries, + const std::optional>& + row_id_column_conflict_checker, + const Snapshot::CommitKind& commit_kind) const; + + void SetRowIdCheckFromSnapshot(const std::optional& row_id_check_from_snapshot); + + bool HasRowIdCheckFromSnapshot() const; + + bool ShouldBeOverwriteCommit(const std::vector& append_table_files, + const std::vector& append_index_files) const; + + Status CollectUncheckedBucketPartitions( + const std::vector& delta_entries, + std::unordered_map* total_buckets) const; + + Status CheckSameBucketByTotalBuckets( + const std::unordered_map& expected_total_buckets, + const std::unordered_map& previous_total_buckets) const; + + private: + Status CheckBucketKeepSame(const std::vector& all_entries, + const Snapshot::CommitKind& commit_kind) const; + + Status BucketNumMismatch(const BinaryRow& partition, int32_t num_buckets, + int32_t previous_num_buckets) const; + + void MarkBucketCheckedPartitions( + const std::unordered_map& total_buckets) const; + + Status CheckDeleteInEntries(const std::vector& merged_entries) const; + + Status CheckKeyRange(const std::vector& merged_entries) const; + + Status CheckRowIdExistence(const std::vector& base_entries, + const std::vector& delta_entries, + const std::optional& next_row_id) const; + + Status CheckRowIdRangeConflicts(const Snapshot::CommitKind& commit_kind, + const std::vector& merged_entries) const; + + Status CheckDataFileRowIdRangeConflicts(RangeHelper& range_helper, + const std::vector& data_files) const; + + Status CheckDedicatedFileRowIdRangeConflicts( + const std::vector& data_files, + const std::vector& dedicated_files) const; + + Status CheckForRowIdFromSnapshot( + const Snapshot& latest_snapshot, const std::vector& delta_entries, + const std::vector& delta_index_entries, + const std::optional>& + row_id_column_conflict_checker) const; + + Status CheckGlobalIndexRowIdExistence( + const std::vector& base_entries, + const std::vector& delta_index_entries) const; + + private: + static constexpr size_t kSameBucketCheckCacheMaxSize = 1000; + + std::shared_ptr table_schema_; + CoreOptions options_; + std::optional row_id_check_from_snapshot_; + std::shared_ptr snapshot_manager_; + std::shared_ptr manifest_list_; + std::shared_ptr manifest_file_; + std::shared_ptr commit_scanner_; + mutable LinkedHashMap same_bucket_checked_partitions_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/conflict_detection_test.cpp b/src/paimon/core/operation/commit/conflict_detection_test.cpp new file mode 100644 index 00000000..4da37304 --- /dev/null +++ b/src/paimon/core/operation/commit/conflict_detection_test.cpp @@ -0,0 +1,590 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/conflict_detection.h" + +#include "arrow/c/abi.h" +#include "arrow/c/bridge.h" +#include "arrow/type.h" +#include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/memory/bytes.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +Snapshot MakeSnapshot(const Snapshot::CommitKind& commit_kind) { + return Snapshot( + /*id=*/1, + /*schema_id=*/1, + /*base_manifest_list=*/"base-manifest-list", + /*base_manifest_list_size=*/std::nullopt, + /*delta_manifest_list=*/"delta-manifest-list", + /*delta_manifest_list_size=*/std::nullopt, + /*changelog_manifest_list=*/std::nullopt, + /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, + /*commit_user=*/"test-user", + /*commit_identifier=*/1, commit_kind, + /*time_millis=*/0, + /*total_record_count=*/0, + /*delta_record_count=*/0, + /*changelog_record_count=*/std::nullopt, + /*watermark=*/std::nullopt, + /*statistics=*/std::nullopt, + /*properties=*/std::nullopt, + /*next_row_id=*/std::nullopt); +} + +Status CheckConflicts(const ConflictDetection& detection, + const std::vector& base_entries, + const std::vector& delta_entries, + const Snapshot::CommitKind& commit_kind) { + return detection.CheckConflicts(MakeSnapshot(commit_kind), base_entries, delta_entries, + /*delta_index_entries=*/{}, + /*row_id_column_conflict_checker=*/std::nullopt, commit_kind); +} + +Status CheckConflicts(const ConflictDetection& detection, + const std::vector& base_entries, + const std::vector& delta_entries, + const std::vector& delta_index_entries, + const Snapshot::CommitKind& commit_kind) { + return detection.CheckConflicts(MakeSnapshot(commit_kind), base_entries, delta_entries, + delta_index_entries, + /*row_id_column_conflict_checker=*/std::nullopt, commit_kind); +} + +} // namespace + +class ConflictDetectionTest : public testing::Test { + public: + void SetUp() override { + fields_ = {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::float64())}; + } + + protected: + ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& kind) const { + int32_t arity = 1; + BinaryRow row(arity); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, 10); + writer.Complete(); + return CreateManifestEntry(file_name, row, kind); + } + + ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition, + const FileKind& kind) const { + return CreateManifestEntry(file_name, partition, kind, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/2, /*bucket=*/0); + } + + ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition, + const FileKind& kind, const BinaryRow& min_key, + const BinaryRow& max_key, int32_t level, int32_t bucket = 0, + int32_t total_buckets = 2) const { + auto data_file_meta = std::make_shared( + file_name, 1024, 8, min_key, max_key, SimpleStats::EmptyStats(), + SimpleStats::EmptyStats(), /*min_seq_no=*/16, /*max_seq_no=*/32, + /*schema_id=*/1, level, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/3, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta); + } + + ManifestEntry CreateManifestEntryWithFirstRowId(const std::string& file_name, + const BinaryRow& partition, + const FileKind& kind, int32_t bucket, + int64_t first_row_id, int64_t row_count) const { + auto data_file_meta = std::make_shared( + file_name, 1024, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, + /*max_seq_no=*/32, + /*schema_id=*/1, /*level=*/2, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, first_row_id, + /*write_cols=*/std::nullopt); + return ManifestEntry(kind, partition, bucket, /*total_buckets=*/2, data_file_meta); + } + + IndexManifestEntry CreateGlobalIndexEntry(const std::string& file_name, + const BinaryRow& partition, int32_t bucket, + int64_t row_range_start, + int64_t row_range_end) const { + GlobalIndexMeta global_index_meta(row_range_start, row_range_end, /*index_field_id=*/1, + /*extra_field_ids=*/std::nullopt, + std::make_shared("meta", GetDefaultPool().get())); + auto index_file_meta = std::make_shared( + "HASH", file_name, /*file_size=*/100, /*row_count=*/5, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index_meta); + return IndexManifestEntry(FileKind::Add(), partition, bucket, index_file_meta); + } + + BinaryRow CreateIntRow(int32_t value) const { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; + } + + arrow::FieldVector fields_; +}; + +TEST_F(ConflictDetectionTest, TestFileDeletionConflicts) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("f1", FileKind::Add())); + + std::vector changes; + changes.push_back(CreateManifestEntry("f1", FileKind::Delete())); + + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("f2", FileKind::Add())); + + std::vector changes; + changes.push_back(CreateManifestEntry("f1", FileKind::Delete())); + changes.push_back(CreateManifestEntry("f3", FileKind::Add())); + + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "Trying to delete file f1"); + } + { + std::vector base_entries; + std::vector changes; + changes.push_back(CreateManifestEntry("f1", FileKind::Delete())); + + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "Trying to delete file f1"); + } +} + +TEST_F(ConflictDetectionTest, TestGlobalIndexRowIdExistenceConflicts) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + const BinaryRow partition = CreateIntRow(10); + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId("base-1", partition, FileKind::Add(), + /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/10)); + base_entries.push_back(CreateManifestEntryWithFirstRowId("base-2", partition, FileKind::Add(), + /*bucket=*/0, /*first_row_id=*/10, + /*row_count=*/10)); + std::vector changes; + + ASSERT_OK( + CheckConflicts(detection, base_entries, changes, + {CreateGlobalIndexEntry("global-index-covered", partition, /*bucket=*/0, + /*row_range_start=*/0, /*row_range_end=*/19)}, + Snapshot::CommitKind::Append())); + + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, + {CreateGlobalIndexEntry("global-index-missing", partition, /*bucket=*/0, + /*row_range_start=*/0, /*row_range_end=*/20)}, + Snapshot::CommitKind::Append()), + "Global index row ID existence conflict"); +} + +TEST_F(ConflictDetectionTest, TestDedicatedStorageRowIdRangeConflicts) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + const BinaryRow partition = CreateIntRow(10); + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId("data-0.orc", partition, + FileKind::Add(), /*bucket=*/0, + /*first_row_id=*/0, + /*row_count=*/10)); + + std::vector out_of_range_dedicated_entries; + out_of_range_dedicated_entries.push_back(CreateManifestEntryWithFirstRowId( + "blob-0.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/5, + /*row_count=*/10)); + ASSERT_NOK_WITH_MSG(CheckConflicts(detection, base_entries, out_of_range_dedicated_entries, + Snapshot::CommitKind::Compact()), + "row-id range conflicts"); + + std::vector contained_dedicated_entries; + contained_dedicated_entries.push_back(CreateManifestEntryWithFirstRowId( + "blob-1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/5, + /*row_count=*/4)); + ASSERT_OK(CheckConflicts(detection, base_entries, contained_dedicated_entries, + Snapshot::CommitKind::Compact())); + + std::vector disjoint_data_entries; + disjoint_data_entries.push_back(CreateManifestEntryWithFirstRowId( + "data-a.orc", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/5)); + disjoint_data_entries.push_back(CreateManifestEntryWithFirstRowId( + "data-b.orc", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/10, + /*row_count=*/5)); + std::vector spanning_dedicated_entries; + spanning_dedicated_entries.push_back(CreateManifestEntryWithFirstRowId( + "vector-0.vector.data", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/3, + /*row_count=*/10)); + ASSERT_NOK_WITH_MSG(CheckConflicts(detection, disjoint_data_entries, spanning_dedicated_entries, + Snapshot::CommitKind::Compact()), + "spans multiple data file ranges"); + + std::vector no_data_entries; + std::vector dedicated_only_entries; + dedicated_only_entries.push_back(CreateManifestEntryWithFirstRowId( + "blob-only.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/1)); + ASSERT_NOK_WITH_MSG(CheckConflicts(detection, no_data_entries, dedicated_only_entries, + Snapshot::CommitKind::Compact()), + "is not covered by one data file range"); +} + +TEST_F(ConflictDetectionTest, TestBucketKeepSame) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + + const BinaryRow partition = CreateIntRow(10); + { + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/2)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "Total buckets of partition"); + } + { + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/2)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", CreateIntRow(20), FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/2)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + + ASSERT_OK( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Overwrite())); + } +} + +TEST_F(ConflictDetectionTest, TestBucketKeepSameHelpers) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + const BinaryRow partition = CreateIntRow(10); + std::vector changes; + changes.push_back(CreateManifestEntry("delta-1", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + /*level=*/1, + /*bucket=*/0, /*total_buckets=*/4)); + changes.push_back(CreateManifestEntry("delta-2", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + /*level=*/1, + /*bucket=*/1, /*total_buckets=*/4)); + + std::unordered_map expected_total_buckets; + ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &expected_total_buckets)); + ASSERT_EQ(1U, expected_total_buckets.size()); + ASSERT_EQ(4, expected_total_buckets.at(partition)); + + std::unordered_map previous_total_buckets; + previous_total_buckets.emplace(partition, 4); + ASSERT_OK( + detection.CheckSameBucketByTotalBuckets(expected_total_buckets, previous_total_buckets)); + + std::unordered_map cached_total_buckets; + ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &cached_total_buckets)); + ASSERT_TRUE(cached_total_buckets.empty()); + + ConflictDetection mismatch_detection(table_schema, core_options, nullptr, nullptr, nullptr, + nullptr); + ASSERT_NOK_WITH_MSG( + mismatch_detection.CheckSameBucketByTotalBuckets(expected_total_buckets, {{partition, 2}}), + "Total buckets of partition"); +} + +TEST_F(ConflictDetectionTest, TestCollectUncheckedBucketPartitionsMismatch) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + const BinaryRow partition = CreateIntRow(10); + std::vector changes; + changes.push_back(CreateManifestEntry("delta-1", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + /*level=*/1, + /*bucket=*/0, /*total_buckets=*/2)); + changes.push_back(CreateManifestEntry("delta-2", partition, FileKind::Add(), + DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + /*level=*/1, + /*bucket=*/1, /*total_buckets=*/4)); + + std::unordered_map total_buckets; + ASSERT_NOK_WITH_MSG(detection.CollectUncheckedBucketPartitions(changes, &total_buckets), + "Total buckets of partition"); +} + +TEST_F(ConflictDetectionTest, TestBucketKeepSameCacheEviction) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + const int32_t total_buckets = 4; + for (int32_t value = 0; value <= 1000; ++value) { + std::vector changes; + changes.push_back(CreateManifestEntry("delta", CreateIntRow(value), FileKind::Add(), + DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, + /*bucket=*/0, total_buckets)); + + std::unordered_map expected_total_buckets; + ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &expected_total_buckets)); + ASSERT_EQ(1U, expected_total_buckets.size()); + ASSERT_OK(detection.CheckSameBucketByTotalBuckets(expected_total_buckets, + expected_total_buckets)); + } + + std::vector evicted_partition_changes; + evicted_partition_changes.push_back( + CreateManifestEntry("delta", CreateIntRow(0), FileKind::Add(), DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/1, /*bucket=*/0, total_buckets)); + std::unordered_map evicted_partition_buckets; + ASSERT_OK(detection.CollectUncheckedBucketPartitions(evicted_partition_changes, + &evicted_partition_buckets)); + ASSERT_EQ(1U, evicted_partition_buckets.size()); +} + +TEST_F(ConflictDetectionTest, TestDeletionVectorsNotSupportedWithBucketUnawareMode) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::BUCKET, "0"}, + {Options::DELETION_VECTORS_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + ASSERT_NOK_WITH_MSG(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{}, + Snapshot::CommitKind::Append()), + "not yet support dv with BUCKET_UNAWARE mode"); +} + +TEST_F(ConflictDetectionTest, + TestDeletionVectorsNotSupportedWithResolvedBucketUnawareModeFromMinusOne) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::BUCKET, "-1"}, + {Options::DELETION_VECTORS_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + ASSERT_NOK_WITH_MSG(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{}, + Snapshot::CommitKind::Append()), + "not yet support dv with BUCKET_UNAWARE mode"); +} + +TEST_F(ConflictDetectionTest, TestDeletionVectorsAllowedWithResolvedDynamicBucketMode) { + auto fields = {arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::int32(), /*nullable=*/false), + arrow::field("f2", arrow::int32())}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{"f1", "f0"}, {{Options::BUCKET, "-1"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::BUCKET, "-1"}, + {Options::DELETION_VECTORS_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + ASSERT_OK(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{}, + Snapshot::CommitKind::Append())); +} + +TEST_F(ConflictDetectionTest, TestCheckLsmKeyRangeConflict) { + auto fields = {arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::int32(), /*nullable=*/false), + arrow::field("f2", arrow::int32())}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{"f1", "f0"}, {{Options::BUCKET, "4"}})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({{Options::BUCKET, "4"}})); + ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + + const BinaryRow partition = CreateIntRow(10); + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/1)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(3), + CreateIntRow(5), /*level=*/1)); + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "LSM conflicts detected"); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/1)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(4), + CreateIntRow(5), /*level=*/1)); + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/0)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(2), + CreateIntRow(5), /*level=*/0)); + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/1, /*bucket=*/0)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", partition, FileKind::Add(), CreateIntRow(2), + CreateIntRow(5), /*level=*/1, + /*bucket=*/1)); + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), + CreateIntRow(1), CreateIntRow(3), + /*level=*/1)); + std::vector changes; + changes.push_back(CreateManifestEntry("delta", CreateIntRow(20), FileKind::Add(), + CreateIntRow(2), CreateIntRow(5), /*level=*/1)); + ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); + } +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/manifest_entry_changes.cpp b/src/paimon/core/operation/commit/manifest_entry_changes.cpp new file mode 100644 index 00000000..1ca54a4b --- /dev/null +++ b/src/paimon/core/operation/commit/manifest_entry_changes.cpp @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/manifest_entry_changes.h" + +#include + +#include "fmt/format.h" +#include "fmt/ranges.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" +#include "paimon/core/io/compact_increment.h" +#include "paimon/core/io/data_increment.h" + +namespace paimon { + +ManifestEntryChanges::ManifestEntryChanges(int32_t default_num_bucket) + : default_num_bucket_(default_num_bucket) {} + +Status ManifestEntryChanges::Collect(const std::shared_ptr& message) { + auto commit_message = std::dynamic_pointer_cast(message); + if (!commit_message) { + return Status::Invalid("fail to cast commit message to commit message impl"); + } + + DataIncrement new_files_increment = commit_message->GetNewFilesIncrement(); + for (const std::shared_ptr& file : new_files_increment.NewFiles()) { + append_table_files.push_back(MakeEntry(FileKind::Add(), commit_message, file)); + } + for (const std::shared_ptr& file : new_files_increment.DeletedFiles()) { + append_table_files.push_back(MakeEntry(FileKind::Delete(), commit_message, file)); + } + for (const std::shared_ptr& file : new_files_increment.ChangelogFiles()) { + append_changelog.push_back(MakeEntry(FileKind::Add(), commit_message, file)); + } + for (const std::shared_ptr& file : new_files_increment.DeletedIndexFiles()) { + append_index_files.emplace_back(FileKind::Delete(), commit_message->Partition(), + commit_message->Bucket(), file); + } + for (const std::shared_ptr& file : new_files_increment.NewIndexFiles()) { + append_index_files.emplace_back(FileKind::Add(), commit_message->Partition(), + commit_message->Bucket(), file); + } + + CompactIncrement compact_increment = commit_message->GetCompactIncrement(); + for (const std::shared_ptr& file : compact_increment.CompactBefore()) { + compact_table_files.push_back(MakeEntry(FileKind::Delete(), commit_message, file)); + } + for (const std::shared_ptr& file : compact_increment.CompactAfter()) { + compact_table_files.push_back(MakeEntry(FileKind::Add(), commit_message, file)); + } + for (const std::shared_ptr& file : compact_increment.ChangelogFiles()) { + compact_changelog.push_back(MakeEntry(FileKind::Add(), commit_message, file)); + } + for (const std::shared_ptr& file : compact_increment.DeletedIndexFiles()) { + compact_index_files.emplace_back(FileKind::Delete(), commit_message->Partition(), + commit_message->Bucket(), file); + } + for (const std::shared_ptr& file : compact_increment.NewIndexFiles()) { + compact_index_files.emplace_back(FileKind::Add(), commit_message->Partition(), + commit_message->Bucket(), file); + } + + return Status::OK(); +} + +bool ManifestEntryChanges::HasAppendChanges() const { + return !append_table_files.empty() || !append_changelog.empty() || !append_index_files.empty(); +} + +bool ManifestEntryChanges::HasGlobalIndexFileAdditions() const { + for (const IndexManifestEntry& index_entry : append_index_files) { + if (index_entry.kind == FileKind::Add() && index_entry.index_file->GetGlobalIndexMeta()) { + return true; + } + } + return false; +} + +bool ManifestEntryChanges::HasCompactChanges() const { + return !compact_table_files.empty() || !compact_changelog.empty() || + !compact_index_files.empty(); +} + +std::string ManifestEntryChanges::ToString() const { + std::vector parts; + if (!append_table_files.empty()) { + parts.push_back(fmt::format("{} append table files", append_table_files.size())); + } + if (!append_changelog.empty()) { + parts.push_back(fmt::format("{} append Changelogs", append_changelog.size())); + } + if (!append_index_files.empty()) { + parts.push_back(fmt::format("{} append index files", append_index_files.size())); + } + if (!compact_table_files.empty()) { + parts.push_back(fmt::format("{} compact table files", compact_table_files.size())); + } + if (!compact_changelog.empty()) { + parts.push_back(fmt::format("{} compact Changelogs", compact_changelog.size())); + } + if (!compact_index_files.empty()) { + parts.push_back(fmt::format("{} compact index files", compact_index_files.size())); + } + return fmt::format("{}", fmt::join(parts, ", ")); +} + +std::vector ManifestEntryChanges::ChangedPartitions( + const std::vector& data_file_changes, + const std::vector& index_file_changes) { + std::unordered_set changed_partitions; + for (const ManifestEntry& file : data_file_changes) { + changed_partitions.insert(file.Partition()); + } + for (const IndexManifestEntry& file : index_file_changes) { + if (file.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX || + file.index_file->GetGlobalIndexMeta()) { + changed_partitions.insert(file.partition); + } + } + return std::vector(changed_partitions.begin(), changed_partitions.end()); +} + +ManifestEntry ManifestEntryChanges::MakeEntry( + const FileKind& kind, const std::shared_ptr& commit_message, + const std::shared_ptr& file) const { + int32_t total_buckets = commit_message->TotalBuckets() == std::nullopt + ? default_num_bucket_ + : commit_message->TotalBuckets().value(); + return ManifestEntry(kind, commit_message->Partition(), commit_message->Bucket(), total_buckets, + file); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/manifest_entry_changes.h b/src/paimon/core/operation/commit/manifest_entry_changes.h new file mode 100644 index 00000000..bafb8bf0 --- /dev/null +++ b/src/paimon/core/operation/commit/manifest_entry_changes.h @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/commit_message.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/status.h" + +namespace paimon { + +/// Detailed changes from `CommitMessage`s. +class ManifestEntryChanges { + public: + explicit ManifestEntryChanges(int32_t default_num_bucket); + + Status Collect(const std::shared_ptr& message); + + bool HasAppendChanges() const; + + bool HasGlobalIndexFileAdditions() const; + + bool HasCompactChanges() const; + + std::string ToString() const; + + static std::vector ChangedPartitions( + const std::vector& data_file_changes, + const std::vector& index_file_changes); + + public: + std::vector append_table_files; + std::vector append_changelog; + std::vector append_index_files; + std::vector compact_table_files; + std::vector compact_changelog; + std::vector compact_index_files; + + private: + ManifestEntry MakeEntry(const FileKind& kind, + const std::shared_ptr& commit_message, + const std::shared_ptr& file) const; + + private: + int32_t default_num_bucket_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp new file mode 100644 index 00000000..e0caf2a4 --- /dev/null +++ b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/manifest_entry_changes.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/io/compact_increment.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_increment.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/data/timestamp.h" +#include "paimon/defs.h" +#include "paimon/memory/bytes.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class ManifestEntryChangesTest : public testing::Test { + protected: + BinaryRow CreateIntRow(int32_t value) const { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; + } + + std::shared_ptr CreateDataFileMeta(const std::string& file_name) const { + return std::make_shared( + file_name, 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, + /*max_seq_no=*/32, + /*schema_id=*/1, /*level=*/2, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + } + + std::shared_ptr CreateIndexFileMeta( + const std::string& file_name, const std::string& index_type = "bitmap") const { + return std::make_shared( + index_type, file_name, /*file_size=*/100, /*row_count=*/5, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, + /*global_index_meta=*/std::nullopt); + } + + std::shared_ptr CreateGlobalIndexFileMeta(const std::string& file_name) const { + GlobalIndexMeta global_index(/*row_range_start=*/0, /*row_range_end=*/3, + /*index_field_id=*/1, /*extra_field_ids=*/std::nullopt, + std::make_shared("meta", GetDefaultPool().get())); + return std::make_shared( + "HASH", file_name, /*file_size=*/100, /*row_count=*/5, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index); + } +}; + +TEST_F(ManifestEntryChangesTest, TestCollectAndSummary) { + const BinaryRow partition = CreateIntRow(10); + + DataIncrement data_increment( + {CreateDataFileMeta("append-add")}, {CreateDataFileMeta("append-del")}, + {CreateDataFileMeta("append-changelog")}, {CreateIndexFileMeta("append-index-add")}, + {CreateIndexFileMeta("append-index-del")}); + CompactIncrement compact_increment( + {CreateDataFileMeta("compact-before")}, {CreateDataFileMeta("compact-after")}, + {CreateDataFileMeta("compact-changelog")}, {CreateIndexFileMeta("compact-index-add")}, + {CreateIndexFileMeta("compact-index-del")}); + + std::shared_ptr message = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/4, data_increment, compact_increment); + + ManifestEntryChanges changes(/*default_num_bucket=*/8); + ASSERT_OK(changes.Collect(message)); + + ASSERT_EQ(2u, changes.append_table_files.size()); + ASSERT_EQ(1u, changes.append_changelog.size()); + ASSERT_EQ(2u, changes.append_index_files.size()); + ASSERT_EQ(2u, changes.compact_table_files.size()); + ASSERT_EQ(1u, changes.compact_changelog.size()); + ASSERT_EQ(2u, changes.compact_index_files.size()); + + EXPECT_TRUE(changes.HasAppendChanges()); + EXPECT_FALSE(changes.HasGlobalIndexFileAdditions()); + EXPECT_TRUE(changes.HasCompactChanges()); + + EXPECT_EQ(FileKind::Add(), changes.append_table_files[0].Kind()); + EXPECT_EQ(FileKind::Delete(), changes.append_table_files[1].Kind()); + EXPECT_EQ(4, changes.append_table_files[0].TotalBuckets()); + + std::string summary = changes.ToString(); + EXPECT_NE(std::string::npos, summary.find("2 append table files")); + EXPECT_NE(std::string::npos, summary.find("1 append Changelogs")); + EXPECT_NE(std::string::npos, summary.find("2 compact index files")); +} + +TEST_F(ManifestEntryChangesTest, TestHasGlobalIndexFileAdditions) { + const BinaryRow partition = CreateIntRow(10); + + DataIncrement data_increment( + /*new_files=*/{}, /*deleted_files=*/{}, /*changelog_files=*/{}, + /*new_index_files=*/{CreateGlobalIndexFileMeta("append-global-index")}, + /*deleted_index_files=*/{}); + CompactIncrement compact_increment(/*compact_before=*/{}, /*compact_after=*/{}, + /*changelog_files=*/{}, + /*new_index_files=*/{}, + /*deleted_index_files=*/{}); + + std::shared_ptr message = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/4, data_increment, compact_increment); + + ManifestEntryChanges changes(/*default_num_bucket=*/8); + ASSERT_OK(changes.Collect(message)); + + EXPECT_TRUE(changes.HasGlobalIndexFileAdditions()); +} + +TEST_F(ManifestEntryChangesTest, TestCollectInvalidCommitMessageType) { + ManifestEntryChanges changes(/*default_num_bucket=*/8); + std::shared_ptr invalid_message = std::make_shared(); + ASSERT_NOK_WITH_MSG(changes.Collect(invalid_message), + "fail to cast commit message to commit message impl"); +} + +TEST_F(ManifestEntryChangesTest, TestChangedPartitionsIncludesDvAndGlobalIndex) { + const BinaryRow partition_data = CreateIntRow(10); + const BinaryRow partition_dv = CreateIntRow(20); + const BinaryRow partition_global = CreateIntRow(30); + const BinaryRow partition_plain_index = CreateIntRow(40); + + std::vector data_changes; + data_changes.emplace_back(FileKind::Add(), partition_data, /*bucket=*/0, /*total_buckets=*/2, + CreateDataFileMeta("data-file")); + + std::vector index_changes; + index_changes.emplace_back(FileKind::Add(), partition_dv, /*bucket=*/0, + CreateIndexFileMeta("dv-file", "DELETION_VECTORS")); + index_changes.emplace_back(FileKind::Add(), partition_global, /*bucket=*/0, + CreateGlobalIndexFileMeta("global-index")); + index_changes.emplace_back(FileKind::Add(), partition_plain_index, /*bucket=*/0, + CreateIndexFileMeta("plain-index", "bitmap")); + + std::vector changed = + ManifestEntryChanges::ChangedPartitions(data_changes, index_changes); + + auto contains = [&changed](const BinaryRow& target) { + return std::find(changed.begin(), changed.end(), target) != changed.end(); + }; + + EXPECT_TRUE(contains(partition_data)); + EXPECT_TRUE(contains(partition_dv)); + EXPECT_TRUE(contains(partition_global)); + EXPECT_FALSE(contains(partition_plain_index)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider.cpp b/src/paimon/core/operation/commit/overwrite_changes_provider.cpp new file mode 100644 index 00000000..e8c02c9f --- /dev/null +++ b/src/paimon/core/operation/commit/overwrite_changes_provider.cpp @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/overwrite_changes_provider.h" + +#include + +#include "paimon/core/manifest/file_kind.h" + +namespace paimon { + +OverwriteChangesProvider::OverwriteChangesProvider(std::vector changes, + std::vector index_entries, + ManifestScan manifest_scan, IndexScan index_scan) + : changes_(std::move(changes)), + index_entries_(std::move(index_entries)), + manifest_scan_(std::move(manifest_scan)), + index_scan_(std::move(index_scan)) {} + +Result> OverwriteChangesProvider::Provide( + const std::optional& latest_snapshot) const { + std::vector delta_files; + std::vector changelog_files; + std::vector index_entries = index_entries_; + + if (!latest_snapshot) { + delta_files.insert(delta_files.end(), changes_.begin(), changes_.end()); + return std::make_shared(std::move(delta_files), std::move(changelog_files), + std::move(index_entries)); + } + + PAIMON_ASSIGN_OR_RAISE(std::vector entries, + manifest_scan_(latest_snapshot.value())); + for (const auto& entry : entries) { + delta_files.emplace_back(FileKind::Delete(), entry.Partition(), entry.Bucket(), + entry.TotalBuckets(), entry.File()); + } + + delta_files.insert(delta_files.end(), changes_.begin(), changes_.end()); + + PAIMON_ASSIGN_OR_RAISE(std::vector previous_index_entries, + index_scan_(latest_snapshot.value())); + for (const auto& entry : previous_index_entries) { + index_entries.emplace_back(FileKind::Delete(), entry.partition, entry.bucket, + entry.index_file); + } + + return std::make_shared(std::move(delta_files), std::move(changelog_files), + std::move(index_entries)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider.h b/src/paimon/core/operation/commit/overwrite_changes_provider.h new file mode 100644 index 00000000..b63170a7 --- /dev/null +++ b/src/paimon/core/operation/commit/overwrite_changes_provider.h @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/operation/commit/commit_changes_provider.h" + +namespace paimon { + +class OverwriteChangesProvider final : public CommitChangesProvider { + public: + using ManifestScan = + std::function>(const Snapshot& snapshot)>; + using IndexScan = + std::function>(const Snapshot& snapshot)>; + + OverwriteChangesProvider(std::vector changes, + std::vector index_entries, + ManifestScan manifest_scan, IndexScan index_scan); + + Result> Provide( + const std::optional& latest_snapshot) const override; + + private: + std::vector changes_; + std::vector index_entries_; + ManifestScan manifest_scan_; + IndexScan index_scan_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp b/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp new file mode 100644 index 00000000..8130fe65 --- /dev/null +++ b/src/paimon/core/operation/commit/overwrite_changes_provider_test.cpp @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/overwrite_changes_provider.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +BinaryRow CreateIntRow(int32_t value) { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; +} + +ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& kind, + int32_t partition_value, int32_t level = 0) { + auto file_meta = std::make_shared( + file_name, /*file_size=*/1024, /*row_count=*/8, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_seq_no=*/0, + /*max_seq_no=*/0, + /*schema_id=*/0, level, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + + return ManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, /*total_buckets=*/1, + file_meta); +} + +IndexManifestEntry CreateIndexEntry(const std::string& file_name, int32_t partition_value, + const FileKind& kind = FileKind::Add()) { + auto index_file = std::make_shared( + /*index_type=*/"HASH", file_name, /*file_size=*/10, /*row_count=*/1, + /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, + /*global_index_meta=*/std::nullopt); + return IndexManifestEntry(kind, CreateIntRow(partition_value), /*bucket=*/0, index_file); +} + +Snapshot MakeSnapshot() { + return Snapshot( + /*id=*/1, + /*schema_id=*/0, + /*base_manifest_list=*/"base-manifest-list", + /*base_manifest_list_size=*/std::nullopt, + /*delta_manifest_list=*/"delta-manifest-list", + /*delta_manifest_list_size=*/std::nullopt, + /*changelog_manifest_list=*/std::nullopt, + /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, + /*commit_user=*/"test-user", + /*commit_identifier=*/1, Snapshot::CommitKind::Overwrite(), + /*time_millis=*/0, + /*total_record_count=*/0, + /*delta_record_count=*/0, + /*changelog_record_count=*/std::nullopt, + /*watermark=*/std::nullopt, + /*statistics=*/std::nullopt, + /*properties=*/std::nullopt, + /*next_row_id=*/std::nullopt); +} + +} // namespace + +TEST(OverwriteChangesProviderTest, TestProvideWithoutLatestSnapshotUsesChangesOnly) { + std::vector changes = { + CreateManifestEntry("new-1", FileKind::Add(), /*partition_value=*/1), + CreateManifestEntry("new-2", FileKind::Delete(), /*partition_value=*/1)}; + std::vector index_entries = { + CreateIndexEntry("index-new", /*partition_value=*/1)}; + + int manifest_scan_calls = 0; + int index_scan_calls = 0; + OverwriteChangesProvider provider( + changes, index_entries, + [&manifest_scan_calls](const Snapshot&) -> Result> { + ++manifest_scan_calls; + return Status::Invalid("should not call manifest_scan"); + }, + [&index_scan_calls](const Snapshot&) -> Result> { + ++index_scan_calls; + return Status::Invalid("should not call index_scan"); + }); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr provided, provider.Provide(std::nullopt)); + + const std::vector& delta_files = provided->delta_files; + const std::vector& changelog_files = provided->changelog_files; + const std::vector& provided_index_entries = provided->index_entries; + + ASSERT_EQ(0, manifest_scan_calls); + ASSERT_EQ(0, index_scan_calls); + ASSERT_TRUE(changelog_files.empty()); + ASSERT_EQ(changes.size(), delta_files.size()); + ASSERT_EQ(index_entries.size(), provided_index_entries.size()); + EXPECT_EQ("new-1", delta_files[0].FileName()); + EXPECT_EQ("new-2", delta_files[1].FileName()); + EXPECT_EQ("index-new", provided_index_entries[0].index_file->FileName()); +} + +TEST(OverwriteChangesProviderTest, TestProvideWithLatestSnapshotAddsDeletesAndAppendsAllChanges) { + std::vector changes = { + CreateManifestEntry("existing-a", FileKind::Add(), /*partition_value=*/1), + CreateManifestEntry("new-b", FileKind::Add(), /*partition_value=*/1), + CreateManifestEntry("force-delete-c", FileKind::Delete(), /*partition_value=*/1)}; + std::vector index_entries = { + CreateIndexEntry("index-new", /*partition_value=*/1)}; + + OverwriteChangesProvider provider( + changes, index_entries, + [](const Snapshot&) -> Result> { + return std::vector{ + CreateManifestEntry("existing-a", FileKind::Add(), /*partition_value=*/1), + CreateManifestEntry("existing-x", FileKind::Add(), /*partition_value=*/1)}; + }, + [](const Snapshot&) -> Result> { + return std::vector{ + CreateIndexEntry("index-old", /*partition_value=*/1)}; + }); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr provided, + provider.Provide(std::optional(MakeSnapshot()))); + + const std::vector& delta_files = provided->delta_files; + const std::vector& changelog_files = provided->changelog_files; + const std::vector& provided_index_entries = provided->index_entries; + + ASSERT_TRUE(changelog_files.empty()); + + // delta = delete(existing-a), delete(existing-x), + // add(existing-a), add(new-b), delete(force-delete-c) + ASSERT_EQ(5u, delta_files.size()); + EXPECT_TRUE(delta_files[0].Kind() == FileKind::Delete()); + EXPECT_EQ("existing-a", delta_files[0].FileName()); + EXPECT_TRUE(delta_files[1].Kind() == FileKind::Delete()); + EXPECT_EQ("existing-x", delta_files[1].FileName()); + EXPECT_TRUE(delta_files[2].Kind() == FileKind::Add()); + EXPECT_EQ("existing-a", delta_files[2].FileName()); + EXPECT_TRUE(delta_files[3].Kind() == FileKind::Add()); + EXPECT_EQ("new-b", delta_files[3].FileName()); + EXPECT_TRUE(delta_files[4].Kind() == FileKind::Delete()); + EXPECT_EQ("force-delete-c", delta_files[4].FileName()); + + // index = provided new + delete old + ASSERT_EQ(2u, provided_index_entries.size()); + EXPECT_TRUE(provided_index_entries[0].kind == FileKind::Add()); + EXPECT_EQ("index-new", provided_index_entries[0].index_file->FileName()); + EXPECT_TRUE(provided_index_entries[1].kind == FileKind::Delete()); + EXPECT_EQ("index-old", provided_index_entries[1].index_file->FileName()); +} + +TEST(OverwriteChangesProviderTest, TestProvidePropagatesScanErrors) { + std::vector changes = { + CreateManifestEntry("new-1", FileKind::Add(), /*partition_value=*/1)}; + std::vector index_entries; + + OverwriteChangesProvider provider_manifest_fail( + changes, index_entries, + [](const Snapshot&) -> Result> { + return Status::Invalid("manifest scan failed"); + }, + [](const Snapshot&) -> Result> { + return std::vector{}; + }); + + ASSERT_NOK_WITH_MSG(provider_manifest_fail.Provide(std::optional(MakeSnapshot())), + "manifest scan failed"); + + OverwriteChangesProvider provider_index_fail( + changes, index_entries, + [](const Snapshot&) -> Result> { + return std::vector{}; + }, + [](const Snapshot&) -> Result> { + return Status::Invalid("index scan failed"); + }); + + ASSERT_NOK_WITH_MSG(provider_index_fail.Provide(std::optional(MakeSnapshot())), + "index scan failed"); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/retry_waiter.cpp b/src/paimon/core/operation/commit/retry_waiter.cpp new file mode 100644 index 00000000..94422225 --- /dev/null +++ b/src/paimon/core/operation/commit/retry_waiter.cpp @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/retry_waiter.h" + +#include +#include +#include +#include +#include +#include + +namespace paimon { + +RetryWaiter::RetryWaiter(int64_t min_retry_wait_ms, int64_t max_retry_wait_ms) + : min_retry_wait_ms_(std::max(0, min_retry_wait_ms)), + max_retry_wait_ms_(std::max(0, max_retry_wait_ms)) {} + +void RetryWaiter::RetryWait(int32_t retry_count) const { + int32_t non_negative_retry_count = std::max(0, retry_count); + double exponential = std::pow(2.0, static_cast(non_negative_retry_count)); + int64_t retry_wait = 0; + if (min_retry_wait_ms_ > 0 && max_retry_wait_ms_ > 0) { + double max_safe_exponential = + static_cast(max_retry_wait_ms_) / static_cast(min_retry_wait_ms_); + if (!std::isfinite(exponential) || exponential >= max_safe_exponential) { + retry_wait = max_retry_wait_ms_; + } else { + retry_wait = static_cast(min_retry_wait_ms_ * exponential); + } + } + + int64_t jitter_upper = std::max(1, static_cast(retry_wait * 0.2)); + std::mt19937 rng(std::random_device{}()); // NOLINT(whitespace/braces) + std::uniform_int_distribution dist(0, jitter_upper - 1); + retry_wait += dist(rng); + + if (retry_wait <= 0) { + return; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(retry_wait)); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/retry_waiter.h b/src/paimon/core/operation/commit/retry_waiter.h new file mode 100644 index 00000000..c1e08305 --- /dev/null +++ b/src/paimon/core/operation/commit/retry_waiter.h @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +namespace paimon { + +class RetryWaiter { + public: + RetryWaiter(int64_t min_retry_wait_ms, int64_t max_retry_wait_ms); + + void RetryWait(int32_t retry_count) const; + + private: + int64_t min_retry_wait_ms_; + int64_t max_retry_wait_ms_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/retry_waiter_test.cpp b/src/paimon/core/operation/commit/retry_waiter_test.cpp new file mode 100644 index 00000000..2d8ea105 --- /dev/null +++ b/src/paimon/core/operation/commit/retry_waiter_test.cpp @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/retry_waiter.h" + +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(RetryWaiterTest, TestRetryWaitWithZeroBoundsReturnsQuickly) { + RetryWaiter waiter(/*min_retry_wait_ms=*/0, /*max_retry_wait_ms=*/0); + + auto begin = std::chrono::steady_clock::now(); + waiter.RetryWait(/*retry_count=*/3); + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - begin); + + ASSERT_LT(elapsed.count(), 20); +} + +TEST(RetryWaiterTest, TestRetryWaitWithLargeRetryCountIsClamped) { + RetryWaiter waiter(/*min_retry_wait_ms=*/10, /*max_retry_wait_ms=*/10); + + auto begin = std::chrono::steady_clock::now(); + waiter.RetryWait(/*retry_count=*/1024); + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - begin); + + ASSERT_GE(elapsed.count(), 8); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp b/src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp new file mode 100644 index 00000000..d805233f --- /dev/null +++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker.cpp @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/row_id_column_conflict_checker.h" + +#include +#include +#include +#include + +#include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/range_helper.h" +#include "paimon/status.h" + +namespace paimon { + +Result> RowIdColumnConflictChecker::FromDataFiles( + const std::shared_ptr& schema_manager, + const std::vector>& delta_files) { + auto checker = + std::shared_ptr(new RowIdColumnConflictChecker(schema_manager)); + + PAIMON_RETURN_NOT_OK(checker->BuildWriteRanges(delta_files)); + return checker; +} + +Status RowIdColumnConflictChecker::BuildWriteRanges( + const std::vector>& delta_files) { + std::vector> row_id_files; + row_id_files.reserve(delta_files.size()); + for (const auto& file : delta_files) { + if (file && file->first_row_id.has_value()) { + row_id_files.push_back(file); + } + } + + if (row_id_files.empty()) { + write_ranges_.clear(); + return Status::OK(); + } + + // 1. merge overlapping ranges and calculate [Range, unordered_set] struct. + RangeHelper> range_helper( + [](const std::shared_ptr& file) -> Result { + return file->first_row_id.value(); + }, + [](const std::shared_ptr& file) -> Result { + return file->first_row_id.value() + file->row_count - 1; + }); + PAIMON_ASSIGN_OR_RAISE( + std::vector>> merged_range_groups, + range_helper.MergeOverlappingRanges(std::move(row_id_files))); + + write_ranges_.clear(); + + for (const auto& group : merged_range_groups) { + Range merged_range = MergeRange(group); + std::unordered_set field_ids; + + for (const auto& file : group) { + PAIMON_RETURN_NOT_OK(AddWriteFieldIds(file, &field_ids)); + } + + write_ranges_.push_back(WriteRange{merged_range, std::move(field_ids)}); + } + + // 2. sort by range for binary search + std::sort(write_ranges_.begin(), write_ranges_.end(), + [](const WriteRange& a, const WriteRange& b) { + if (a.range.from != b.range.from) { + return a.range.from < b.range.from; + } + return a.range.to < b.range.to; + }); + + return Status::OK(); +} + +Range RowIdColumnConflictChecker::MergeRange( + const std::vector>& files) const { + int64_t from = std::numeric_limits::max(); + int64_t to = std::numeric_limits::min(); + for (const auto& file : files) { + const int64_t file_from = file->first_row_id.value(); + const int64_t file_to = file_from + file->row_count - 1; + from = std::min(from, file_from); + to = std::max(to, file_to); + } + return Range(from, to); +} + +Status RowIdColumnConflictChecker::AddWriteFieldIds(const std::shared_ptr& file, + std::unordered_set* field_ids) { + if (!file->write_cols.has_value()) { + std::map field_id_by_name; + PAIMON_ASSIGN_OR_RAISE(field_id_by_name, FieldIdByName(file->schema_id)); + for (const auto& entry : field_id_by_name) { + field_ids->insert(entry.second); + } + return Status::OK(); + } + + for (const auto& write_col : file->write_cols.value()) { + PAIMON_ASSIGN_OR_RAISE(std::optional field_id, FieldId(file, write_col)); + if (field_id.has_value()) { + field_ids->insert(field_id.value()); + } + } + + return Status::OK(); +} + +Result RowIdColumnConflictChecker::ConflictsWith( + const std::shared_ptr& file) const { + if (!file->first_row_id.has_value()) { + return false; + } + + Range range(file->first_row_id.value(), file->first_row_id.value() + file->row_count - 1); + int32_t index = FirstPossibleRange(range); + while (index < static_cast(write_ranges_.size())) { + const auto& write_range = write_ranges_[index]; + if (write_range.range.from > range.to) { + return false; + } + // overlapping row range and overlapping write fields + if (Range::HasIntersection(write_range.range, range)) { + PAIMON_ASSIGN_OR_RAISE(bool has_common_write_field, + ContainsAnyWriteField(write_range.field_ids, file)); + if (has_common_write_field) { + return true; + } + } + ++index; + } + + return false; +} + +int32_t RowIdColumnConflictChecker::FirstPossibleRange(const Range& range) const { + int32_t low = 0; + auto high = static_cast(write_ranges_.size()); + while (low < high) { + const int32_t mid = low + (high - low) / 2; + if (write_ranges_[mid].range.to < range.from) { + low = mid + 1; + } else { + high = mid; + } + } + return low; +} + +Result RowIdColumnConflictChecker::ContainsAnyWriteField( + const std::unordered_set& field_ids, const std::shared_ptr& file) const { + // If write cols == null, it's a full-schema write + if (!file->write_cols.has_value()) { + return true; + } + + for (const auto& write_col : file->write_cols.value()) { + PAIMON_ASSIGN_OR_RAISE(std::optional field_id, FieldId(file, write_col)); + if (field_id.has_value() && field_ids.count(field_id.value()) > 0) { + return true; + } + } + return false; +} + +Result> RowIdColumnConflictChecker::FieldId( + const std::shared_ptr& file, const std::string& write_col) const { + std::map field_id_by_name; + PAIMON_ASSIGN_OR_RAISE(field_id_by_name, FieldIdByName(file->schema_id)); + auto it = field_id_by_name.find(write_col); + if (it != field_id_by_name.end()) { + return std::optional(it->second); + } + + if (SpecialFields::IsSystemField(write_col)) { + return std::optional(); + } + + return Status::Invalid("Cannot find write column '" + write_col + "' in schema " + + std::to_string(file->schema_id) + "."); +} + +Result> RowIdColumnConflictChecker::FieldIdByName( + int64_t schema_id) const { + auto cache_it = field_id_by_name_cache_.find(schema_id); + if (cache_it != field_id_by_name_cache_.end()) { + return cache_it->second; + } + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + schema_manager_->ReadSchema(schema_id)); + std::map mapping; + for (const auto& field : schema->Fields()) { + mapping[field.Name()] = field.Id(); + } + + auto [it, inserted] = field_id_by_name_cache_.emplace(schema_id, std::move(mapping)); + (void)inserted; + return it->second; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker.h b/src/paimon/core/operation/commit/row_id_column_conflict_checker.h new file mode 100644 index 00000000..0576ed74 --- /dev/null +++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker.h @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/utils/range.h" + +namespace paimon { + +/// Detects row-id range conflicts only when written field ids overlap. The detection process is as +/// below: +/// +/// Merge delta files by row range and calculate updated columns. +/// Sort those items by range. +/// For each checking files, do binary search to find overlapping ranges. If their updated +/// columns also overlap, return conflicting result. +/// +class RowIdColumnConflictChecker { + public: + static Result> FromDataFiles( + const std::shared_ptr& schema_manager, + const std::vector>& delta_files); + + bool IsEmpty() const { + return write_ranges_.empty(); + } + + /// Check whether a committed incremental file entry conflicts with current committing delta + /// files. If an existing file has both overlapping row range and overlapping write fields, then + /// it conflicts. + /// + /// @param file committed incremental data file + /// @return true if conflict + Result ConflictsWith(const std::shared_ptr& file) const; + + private: + /// Range and field id Set. + struct WriteRange { + Range range; + std::unordered_set field_ids; + }; + + explicit RowIdColumnConflictChecker(const std::shared_ptr& schema_manager) + : schema_manager_(schema_manager) {} + + Status BuildWriteRanges(const std::vector>& delta_files); + Status AddWriteFieldIds(const std::shared_ptr& file, + std::unordered_set* field_ids); + Range MergeRange(const std::vector>& files) const; + int32_t FirstPossibleRange(const Range& range) const; + Result ContainsAnyWriteField(const std::unordered_set& field_ids, + const std::shared_ptr& file) const; + Result> FieldId(const std::shared_ptr& file, + const std::string& write_col) const; + Result> FieldIdByName(int64_t schema_id) const; + + private: + std::shared_ptr schema_manager_; + std::vector write_ranges_; + mutable std::map> field_id_by_name_cache_; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp new file mode 100644 index 00000000..b8e0cee1 --- /dev/null +++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/row_id_column_conflict_checker.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class RowIdColumnConflictCheckerTest : public testing::Test { + protected: + void SetUp() override { + auto fs = std::make_shared(); + const std::string table_root = + GetDataDir() + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table/"; + schema_manager_ = std::make_shared(fs, table_root); + } + + std::shared_ptr CreateFile( + const std::string& file_name, std::optional first_row_id, int64_t row_count, + int64_t schema_id, std::optional> write_cols) const { + return std::make_shared( + file_name, /*file_size=*/0, row_count, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_seq_no=*/0, + /*max_seq_no=*/0, schema_id, /*level=*/0, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, + /*first_row_id=*/first_row_id, write_cols); + } + + Result> CreateChecker( + const std::vector>& files) const { + return RowIdColumnConflictChecker::FromDataFiles(schema_manager_, files); + } + + private: + std::shared_ptr schema_manager_; +}; + +TEST_F(RowIdColumnConflictCheckerTest, TestAllowsDisjointWriteColumns) { + ASSERT_OK_AND_ASSIGN( + auto checker, CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/0, std::vector{"b"})})); + + auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/0, std::vector{"c"}); + ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); + EXPECT_FALSE(conflicts); +} + +TEST_F(RowIdColumnConflictCheckerTest, TestDetectsSameWriteColumns) { + ASSERT_OK_AND_ASSIGN( + auto checker, CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/0, std::vector{"b"})})); + + auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/0, std::vector{"b"}); + ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); + EXPECT_TRUE(conflicts); +} + +TEST_F(RowIdColumnConflictCheckerTest, TestUsesFieldIdAcrossRename) { + ASSERT_OK_AND_ASSIGN( + auto checker, CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/1, std::vector{"c"})})); + + auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/0, std::vector{"b"}); + ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); + EXPECT_TRUE(conflicts); +} + +TEST_F(RowIdColumnConflictCheckerTest, TestTreatsNullWriteColumnsAsFullSchemaWrite) { + ASSERT_OK_AND_ASSIGN(auto checker, + CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/0, /*write_cols=*/std::nullopt)})); + + auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/0, std::vector{"b"}); + ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); + EXPECT_TRUE(conflicts); +} + +TEST_F(RowIdColumnConflictCheckerTest, TestMergesOverlappedDeltaRangesAndWriteColumns) { + ASSERT_OK_AND_ASSIGN( + auto checker, CreateChecker({CreateFile("current-b", /*first_row_id=*/0, /*row_count=*/11, + /*schema_id=*/0, std::vector{"b"}), + CreateFile("current-c", /*first_row_id=*/5, /*row_count=*/11, + /*schema_id=*/0, std::vector{"c"})})); + + auto historical_b = CreateFile("historical-b", /*first_row_id=*/12, /*row_count=*/1, + /*schema_id=*/0, std::vector{"b"}); + auto historical_c = CreateFile("historical-c", /*first_row_id=*/12, /*row_count=*/1, + /*schema_id=*/0, std::vector{"c"}); + ASSERT_OK_AND_ASSIGN(bool conflicts_b, checker->ConflictsWith(historical_b)); + ASSERT_OK_AND_ASSIGN(bool conflicts_c, checker->ConflictsWith(historical_c)); + EXPECT_TRUE(conflicts_b); + EXPECT_TRUE(conflicts_c); +} + +TEST_F(RowIdColumnConflictCheckerTest, TestScansAllOverlappedRangesAfterBinarySearch) { + ASSERT_OK_AND_ASSIGN( + auto checker, CreateChecker({CreateFile("current-b", /*first_row_id=*/0, /*row_count=*/5, + /*schema_id=*/0, std::vector{"b"}), + CreateFile("current-c", /*first_row_id=*/10, /*row_count=*/5, + /*schema_id=*/0, std::vector{"c"})})); + + auto historical = CreateFile("historical", /*first_row_id=*/3, /*row_count=*/10, + /*schema_id=*/0, std::vector{"c"}); + ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); + EXPECT_TRUE(conflicts); +} + +TEST_F(RowIdColumnConflictCheckerTest, TestIgnoreUnknownNonSystemWriteColumn) { + ASSERT_OK_AND_ASSIGN( + auto checker, CreateChecker({CreateFile("current", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/0, std::vector{"b"})})); + + auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, + /*schema_id=*/0, std::vector{"missing"}); + auto conflicts = checker->ConflictsWith(historical); + EXPECT_FALSE(conflicts.ok()); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp new file mode 100644 index 00000000..8fc61394 --- /dev/null +++ b/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/row_tracking_commit_utils.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/status.h" + +namespace paimon { + +namespace { + +bool IsVectorStoreFile(const std::string& file_name) { + return file_name.find(".vector.") != std::string::npos; +} + +ManifestEntry CloneEntryWithClonedFileMeta(const ManifestEntry& entry) { + auto cloned_file = std::make_shared(*entry.File()); + return ManifestEntry(entry.Kind(), entry.Partition(), entry.Bucket(), entry.TotalBuckets(), + cloned_file); +} + +} // namespace + +Result RowTrackingCommitUtils::AssignRowTracking( + int64_t new_snapshot_id, int64_t first_row_id_start, + const std::vector& delta_files) { + // assigned snapshot id to delta files + std::vector snapshot_assigned; + AssignSnapshotId(new_snapshot_id, delta_files, &snapshot_assigned); + // assign row id for new files + std::vector row_id_assigned; + PAIMON_ASSIGN_OR_RAISE( + int64_t next_row_id_start, + AssignRowTrackingMeta(first_row_id_start, snapshot_assigned, &row_id_assigned)); + return RowTrackingAssigned{next_row_id_start, std::move(row_id_assigned)}; +} + +void RowTrackingCommitUtils::AssignSnapshotId(int64_t snapshot_id, + const std::vector& delta_files, + std::vector* snapshot_assigned) { + for (const auto& entry : delta_files) { + ManifestEntry assigned_entry = CloneEntryWithClonedFileMeta(entry); + int64_t min_seq_number = assigned_entry.File()->min_sequence_number; + int64_t max_seq_number = assigned_entry.File()->max_sequence_number; + if (min_seq_number == 0L) { + // Case 1: New file (e.g., from INSERT) + // All records in this file get the current snapshot ID as sequence number + assigned_entry.AssignSequenceNumber(snapshot_id, snapshot_id); + } else if (max_seq_number == 0L) { + // Case 2: File with some modified records + // - min: preserve original sequence number (from unmodified records) + // - max: assign current snapshot ID + assigned_entry.AssignSequenceNumber(min_seq_number, snapshot_id); + } else { + // Case 3: Pure compact file (no modified records) + // Preserve original min/max sequence numbers from source files. + } + snapshot_assigned->emplace_back(std::move(assigned_entry)); + } +} + +Result RowTrackingCommitUtils::AssignRowTrackingMeta( + int64_t first_row_id_start, const std::vector& delta_files, + std::vector* row_id_assigned) { + if (delta_files.empty()) { + return first_row_id_start; + } + // assign row id for new files + int64_t start = first_row_id_start; + int64_t blob_start_default = first_row_id_start; + std::map blob_starts; + int64_t vector_store_start = first_row_id_start; + + for (const auto& entry : delta_files) { + ManifestEntry assigned_entry = CloneEntryWithClonedFileMeta(entry); + if (!entry.File()->file_source) { + return Status::Invalid( + "This is a bug, file source field for row-tracking table must present."); + } + + bool contains_row_id = + entry.File()->write_cols.has_value() && + std::find(entry.File()->write_cols->begin(), entry.File()->write_cols->end(), + SpecialFields::RowId().Name()) != entry.File()->write_cols->end(); + + if (entry.File()->file_source.value() == FileSource::Append() && + entry.File()->first_row_id == std::nullopt && !contains_row_id) { + int64_t row_count = entry.File()->row_count; + if (BlobUtils::IsBlobFile(entry.File()->file_name)) { + if (!entry.File()->write_cols || entry.File()->write_cols->empty()) { + return Status::Invalid(fmt::format( + "invalid blob file {}: does not have write_cols", entry.File()->file_name)); + } + std::string blob_field_name = entry.File()->write_cols->at(0); + int64_t blob_start = blob_starts.count(blob_field_name) + ? blob_starts[blob_field_name] + : blob_start_default; + if (blob_start >= start) { + return Status::Invalid( + fmt::format("This is a bug, blobStart {} should be less than start {} when " + "assigning a blob entry file.", + blob_start, start)); + } + assigned_entry.AssignFirstRowId(blob_start); + blob_starts[blob_field_name] = blob_start + row_count; + } else if (IsVectorStoreFile(entry.File()->file_name)) { + if (vector_store_start >= start) { + return Status::Invalid(fmt::format( + "This is a bug, vectorStoreStart {} should be less than start {} " + "when assigning a vector-store entry file.", + vector_store_start, start)); + } + assigned_entry.AssignFirstRowId(vector_store_start); + vector_store_start += row_count; + } else { + assigned_entry.AssignFirstRowId(start); + blob_start_default = start; + blob_starts.clear(); + start += row_count; + } + } else { + // for compact file, do not assign first row id. + } + row_id_assigned->emplace_back(std::move(assigned_entry)); + } + return start; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils.h b/src/paimon/core/operation/commit/row_tracking_commit_utils.h new file mode 100644 index 00000000..766d36a7 --- /dev/null +++ b/src/paimon/core/operation/commit/row_tracking_commit_utils.h @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/result.h" + +namespace paimon { + +/// Utils for row tracking commit. +class RowTrackingCommitUtils { + public: + struct RowTrackingAssigned { + int64_t next_row_id_start; + std::vector assigned_entries; + }; + + // Assign sequence numbers and row ids for row-tracking commit. + static Result AssignRowTracking( + int64_t new_snapshot_id, int64_t first_row_id_start, + const std::vector& delta_files); + + private: + static void AssignSnapshotId(int64_t snapshot_id, const std::vector& delta_files, + std::vector* snapshot_assigned); + + static Result AssignRowTrackingMeta(int64_t first_row_id_start, + const std::vector& delta_files, + std::vector* row_id_assigned); +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp new file mode 100644 index 00000000..55be4c67 --- /dev/null +++ b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/row_tracking_commit_utils.h" + +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class RowTrackingCommitUtilsTest : public testing::Test { + protected: + BinaryRow CreateIntRow(int32_t value) const { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; + } + + ManifestEntry CreateEntry(const std::string& file_name, int64_t row_count, + int64_t min_seq_number, int64_t max_seq_number, + const std::optional& file_source, + const std::optional>& write_cols) const { + auto file_meta = std::make_shared( + file_name, /*file_size=*/row_count, row_count, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + min_seq_number, max_seq_number, + /*schema_id=*/1, /*level=*/0, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, file_source, + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, write_cols); + return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1, + file_meta); + } +}; + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingStampsSequence) { + std::vector input; + input.push_back(CreateEntry("new-file", /*row_count=*/10, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"f0"})); + input.push_back(CreateEntry("partial-modified", /*row_count=*/8, /*min_seq_number=*/7, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"f0"})); + input.push_back(CreateEntry("compact-file", /*row_count=*/6, /*min_seq_number=*/3, + /*max_seq_number=*/5, FileSource::Compact(), + std::vector{"f0"})); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input)); + + ASSERT_EQ(3u, assigned.assigned_entries.size()); + EXPECT_EQ(100, assigned.assigned_entries[0].File()->min_sequence_number); + EXPECT_EQ(100, assigned.assigned_entries[0].File()->max_sequence_number); + EXPECT_EQ(7, assigned.assigned_entries[1].File()->min_sequence_number); + EXPECT_EQ(100, assigned.assigned_entries[1].File()->max_sequence_number); + EXPECT_EQ(3, assigned.assigned_entries[2].File()->min_sequence_number); + EXPECT_EQ(5, assigned.assigned_entries[2].File()->max_sequence_number); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingStampsSequenceRangeStartingAtZero) { + std::vector input; + input.push_back(CreateEntry("range-starts-at-zero", /*row_count=*/10, + /*min_seq_number=*/0, /*max_seq_number=*/9, FileSource::Append(), + std::vector{"f0"})); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input)); + + ASSERT_EQ(1u, assigned.assigned_entries.size()); + EXPECT_EQ(100, assigned.assigned_entries[0].File()->min_sequence_number); + EXPECT_EQ(100, assigned.assigned_entries[0].File()->max_sequence_number); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTracking) { + std::vector input; + input.push_back(CreateEntry("normal-file", /*row_count=*/10, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"f0"})); + input.push_back(CreateEntry("blob-a.blob", /*row_count=*/3, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"blob_a"})); + input.push_back(CreateEntry("blob-a-2.blob", /*row_count=*/2, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"blob_a"})); + input.push_back(CreateEntry("vector-1.vector.data", /*row_count=*/4, + /*min_seq_number=*/0, /*max_seq_number=*/0, FileSource::Append(), + std::vector{"vec"})); + input.push_back(CreateEntry("normal-file-2", /*row_count=*/5, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"f0"})); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/200, /*first_row_id_start=*/0, input)); + + ASSERT_EQ(5u, assigned.assigned_entries.size()); + EXPECT_EQ(0, assigned.assigned_entries[0].File()->first_row_id.value()); + EXPECT_EQ(0, assigned.assigned_entries[1].File()->first_row_id.value()); + EXPECT_EQ(3, assigned.assigned_entries[2].File()->first_row_id.value()); + EXPECT_EQ(0, assigned.assigned_entries[3].File()->first_row_id.value()); + EXPECT_EQ(10, assigned.assigned_entries[4].File()->first_row_id.value()); + EXPECT_EQ(15, assigned.next_row_id_start); + + for (const auto& entry : assigned.assigned_entries) { + EXPECT_EQ(200, entry.File()->min_sequence_number); + EXPECT_EQ(200, entry.File()->max_sequence_number); + } +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingWithoutFileSource) { + std::vector input; + input.push_back(CreateEntry("invalid-no-source", /*row_count=*/1, /*min_seq_number=*/0, + /*max_seq_number=*/0, std::nullopt, + std::vector{"f0"})); + + ASSERT_NOK_WITH_MSG(RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/1, /*first_row_id_start=*/0, input), + "file source field for row-tracking table must present"); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingDoesNotMutateInputEntries) { + std::vector input; + input.push_back(CreateEntry("normal-file", /*row_count=*/10, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"f0"})); + input.push_back(CreateEntry("normal-file-2", /*row_count=*/5, /*min_seq_number=*/7, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"f0"})); + + std::shared_ptr input_file_0 = input[0].File(); + std::shared_ptr input_file_1 = input[1].File(); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/200, /*first_row_id_start=*/1000, input)); + + ASSERT_EQ(2u, assigned.assigned_entries.size()); + + EXPECT_EQ(0, input[0].File()->min_sequence_number); + EXPECT_EQ(0, input[0].File()->max_sequence_number); + EXPECT_EQ(std::nullopt, input[0].File()->first_row_id); + + EXPECT_EQ(7, input[1].File()->min_sequence_number); + EXPECT_EQ(0, input[1].File()->max_sequence_number); + EXPECT_EQ(std::nullopt, input[1].File()->first_row_id); + + EXPECT_NE(input_file_0.get(), assigned.assigned_entries[0].File().get()); + EXPECT_NE(input_file_1.get(), assigned.assigned_entries[1].File().get()); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingReassignsOnRetryWithAdvancedRowId) { + std::vector input; + input.push_back(CreateEntry("retry-file", /*row_count=*/10, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"f0"})); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned first_attempt, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/1000, input)); + ASSERT_EQ(1u, first_attempt.assigned_entries.size()); + EXPECT_EQ(100, first_attempt.assigned_entries[0].File()->min_sequence_number); + EXPECT_EQ(100, first_attempt.assigned_entries[0].File()->max_sequence_number); + ASSERT_TRUE(first_attempt.assigned_entries[0].File()->first_row_id.has_value()); + EXPECT_EQ(1000, first_attempt.assigned_entries[0].File()->first_row_id.value()); + EXPECT_EQ(1010, first_attempt.next_row_id_start); + + // Simulate CAS retry with a newer latest snapshot and advanced next_row_id. + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned second_attempt, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/101, /*first_row_id_start=*/2000, input)); + ASSERT_EQ(1u, second_attempt.assigned_entries.size()); + EXPECT_EQ(101, second_attempt.assigned_entries[0].File()->min_sequence_number); + EXPECT_EQ(101, second_attempt.assigned_entries[0].File()->max_sequence_number); + ASSERT_TRUE(second_attempt.assigned_entries[0].File()->first_row_id.has_value()); + EXPECT_EQ(2000, second_attempt.assigned_entries[0].File()->first_row_id.value()); + EXPECT_EQ(2010, second_attempt.next_row_id_start); + + // Input remains immutable across attempts; retry assignment always starts from fresh metadata. + EXPECT_EQ(0, input[0].File()->min_sequence_number); + EXPECT_EQ(0, input[0].File()->max_sequence_number); + EXPECT_EQ(std::nullopt, input[0].File()->first_row_id); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp b/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp new file mode 100644 index 00000000..51729b99 --- /dev/null +++ b/src/paimon/core/operation/commit/sequence_snapshot_properties.cpp @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/sequence_snapshot_properties.h" + +#include + +#include +#include + +#include "paimon/core/manifest/file_kind.h" + +namespace paimon { + +Result> SequenceSnapshotProperties::MaxSequenceNumber( + const std::optional& snapshot) { + if (!snapshot || !snapshot.value().Properties()) { + return std::optional(); + } + + const auto& properties = snapshot.value().Properties().value(); + auto iter = properties.find(kMaxSequenceNumberKey); + if (iter == properties.end()) { + return std::optional(); + } + + try { + size_t parsed = 0; + int64_t value = std::stoll(iter->second, &parsed); + if (parsed != iter->second.size()) { + return Status::Invalid( + fmt::format("Invalid {} value '{}': trailing characters are not allowed", + kMaxSequenceNumberKey, iter->second)); + } + return std::optional(value); + } catch (const std::exception& e) { + return Status::Invalid(fmt::format("Invalid {} value '{}': {}", kMaxSequenceNumberKey, + iter->second, e.what())); + } +} + +std::optional SequenceSnapshotProperties::MaxSequenceNumberFromFiles( + const std::vector& files) { + int64_t max_sequence_number = std::numeric_limits::min(); + bool found = false; + for (const auto& file : files) { + if (!(file.Kind() == FileKind::Add())) { + continue; + } + max_sequence_number = std::max(max_sequence_number, file.File()->max_sequence_number); + found = true; + } + + if (!found) { + return std::nullopt; + } + return max_sequence_number; +} + +std::map SequenceSnapshotProperties::MergeMaxSequenceNumber( + const std::map& properties, + const std::optional& latest_max_sequence_number, + const std::vector& delta_files) { + std::map snapshot_properties = properties; + + std::optional delta_max_sequence_number = MaxSequenceNumberFromFiles(delta_files); + if (delta_max_sequence_number || latest_max_sequence_number) { + int64_t merged_max_sequence_number = latest_max_sequence_number + ? latest_max_sequence_number.value() + : delta_max_sequence_number.value(); + if (delta_max_sequence_number) { + merged_max_sequence_number = + std::max(merged_max_sequence_number, delta_max_sequence_number.value()); + } + snapshot_properties[kMaxSequenceNumberKey] = std::to_string(merged_max_sequence_number); + } + + return snapshot_properties; +} + +} // namespace paimon diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties.h b/src/paimon/core/operation/commit/sequence_snapshot_properties.h new file mode 100644 index 00000000..06beac30 --- /dev/null +++ b/src/paimon/core/operation/commit/sequence_snapshot_properties.h @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/snapshot.h" +#include "paimon/result.h" + +namespace paimon { + +class ManifestFile; +class ManifestFileMeta; + +class SequenceSnapshotProperties { + public: + SequenceSnapshotProperties() = delete; + + static constexpr const char* kMaxSequenceNumberKey = "sequence.generation.max-sequence-number"; + + static Result> MaxSequenceNumber( + const std::optional& snapshot); + + static std::optional MaxSequenceNumberFromFiles( + const std::vector& files); + + static std::map MergeMaxSequenceNumber( + const std::map& properties, + const std::optional& latest_max_sequence_number, + const std::vector& delta_files); +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/commit_context.cpp b/src/paimon/core/operation/commit_context.cpp index 490a0756..fe98095d 100644 --- a/src/paimon/core/operation/commit_context.cpp +++ b/src/paimon/core/operation/commit_context.cpp @@ -31,6 +31,7 @@ namespace paimon { CommitContext::CommitContext(const std::string& root_path, const std::string& commit_user, bool ignore_empty_commit, bool use_rest_catalog_commit, + bool append_commit_check_conflict, const std::shared_ptr& memory_pool, const std::shared_ptr& executor, const std::shared_ptr& specific_file_system, @@ -39,6 +40,7 @@ CommitContext::CommitContext(const std::string& root_path, const std::string& co commit_user_(commit_user), ignore_empty_commit_(ignore_empty_commit), use_rest_catalog_commit_(use_rest_catalog_commit), + append_commit_check_conflict_(append_commit_check_conflict), memory_pool_(memory_pool), executor_(executor), specific_file_system_(specific_file_system), @@ -53,6 +55,7 @@ class CommitContextBuilder::Impl { void Reset() { ignore_empty_commit_ = true; use_rest_catalog_commit_ = false; + append_commit_check_conflict_ = false; memory_pool_ = GetDefaultPool(); executor_ = CreateDefaultExecutor(); specific_file_system_.reset(); @@ -64,6 +67,7 @@ class CommitContextBuilder::Impl { std::string commit_user_; bool ignore_empty_commit_ = true; bool use_rest_catalog_commit_ = false; + bool append_commit_check_conflict_ = false; std::shared_ptr memory_pool_ = GetDefaultPool(); std::shared_ptr executor_ = CreateDefaultExecutor(); std::shared_ptr specific_file_system_; @@ -101,6 +105,12 @@ CommitContextBuilder& CommitContextBuilder::UseRESTCatalogCommit(bool use_rest_c return *this; } +CommitContextBuilder& CommitContextBuilder::AppendCommitCheckConflict( + bool append_commit_check_conflict) { + impl_->append_commit_check_conflict_ = append_commit_check_conflict; + return *this; +} + CommitContextBuilder& CommitContextBuilder::WithMemoryPool( const std::shared_ptr& memory_pool) { impl_->memory_pool_ = memory_pool; @@ -126,8 +136,8 @@ Result> CommitContextBuilder::Finish() { } auto ctx = std::make_unique( impl_->root_path_, impl_->commit_user_, impl_->ignore_empty_commit_, - impl_->use_rest_catalog_commit_, impl_->memory_pool_, impl_->executor_, - impl_->specific_file_system_, impl_->options_); + impl_->use_rest_catalog_commit_, impl_->append_commit_check_conflict_, impl_->memory_pool_, + impl_->executor_, impl_->specific_file_system_, impl_->options_); impl_->Reset(); return ctx; } diff --git a/src/paimon/core/operation/commit_context_test.cpp b/src/paimon/core/operation/commit_context_test.cpp new file mode 100644 index 00000000..e790958f --- /dev/null +++ b/src/paimon/core/operation/commit_context_test.cpp @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/commit_context.h" + +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/executor.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/mock/mock_file_system.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(CommitContextTest, TestDefaultValue) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); + + ASSERT_EQ(ctx->GetRootPath(), expected_root_path); + ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); + ASSERT_TRUE(ctx->IgnoreEmptyCommit()); + ASSERT_FALSE(ctx->UseRESTCatalogCommit()); + ASSERT_FALSE(ctx->AppendCommitCheckConflict()); + ASSERT_TRUE(ctx->GetMemoryPool()); + ASSERT_TRUE(ctx->GetExecutor()); + ASSERT_FALSE(ctx->GetSpecificFileSystem()); + ASSERT_TRUE(ctx->GetOptions().empty()); +} + +TEST(CommitContextTest, TestSetContent) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + + auto memory_pool = GetDefaultPool(); + std::shared_ptr executor = CreateDefaultExecutor(); + auto fs = std::make_shared(); + + ASSERT_OK_AND_ASSIGN(auto ctx, builder.IgnoreEmptyCommit(false) + .UseRESTCatalogCommit(true) + .AppendCommitCheckConflict(true) + .WithMemoryPool(memory_pool) + .WithExecutor(executor) + .WithFileSystem(fs) + .AddOption("key", "value") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto expected_root_path, PathUtil::NormalizePath("table_root_path")); + ASSERT_EQ(ctx->GetRootPath(), expected_root_path); + ASSERT_EQ(ctx->GetCommitUser(), "commit_user_1"); + ASSERT_FALSE(ctx->IgnoreEmptyCommit()); + ASSERT_TRUE(ctx->UseRESTCatalogCommit()); + ASSERT_TRUE(ctx->AppendCommitCheckConflict()); + ASSERT_EQ(ctx->GetMemoryPool(), memory_pool); + ASSERT_EQ(ctx->GetExecutor(), executor); + ASSERT_EQ(ctx->GetSpecificFileSystem(), fs); + + std::map expected_options = {{"key", "value"}}; + ASSERT_EQ(ctx->GetOptions(), expected_options); +} + +TEST(CommitContextTest, TestSetOptionsOverridesAddedOptions) { + CommitContextBuilder builder("table_root_path", "commit_user_1"); + builder.AddOption("old", "value"); + builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}}); + + ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish()); + + std::map expected_options = {{"key1", "value1"}, {"key2", "value2"}}; + ASSERT_EQ(ctx->GetOptions(), expected_options); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/commit_metrics_test.cpp b/src/paimon/core/operation/commit_metrics_test.cpp index c478f599..240d9134 100644 --- a/src/paimon/core/operation/commit_metrics_test.cpp +++ b/src/paimon/core/operation/commit_metrics_test.cpp @@ -18,15 +18,56 @@ #include "paimon/core/operation/metrics/commit_metrics.h" +#include #include +#include #include +#include #include "gtest/gtest.h" +#include "paimon/common/data/binary_row_writer.h" #include "paimon/common/metrics/metrics_impl.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/operation/metrics/commit_stats.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { +namespace { + +BinaryRow CreateIntRow(int32_t value) { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; +} + +ManifestEntry CreateEntry(const FileKind& kind, int32_t partition, int32_t bucket, + int64_t row_count, int64_t file_size, const std::string& file_name) { + BinaryRow part = CreateIntRow(partition); + auto file_meta = std::make_shared( + file_name, file_size, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/0, + /*schema_id=*/1, /*level=*/0, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + return ManifestEntry(kind, part, bucket, /*total_buckets=*/10, file_meta); +} + +} // namespace + TEST(CommitMetricsTest, TestSimple) { auto commit_metrics = std::make_shared(); commit_metrics->SetCounter("some_metric", 100); @@ -48,4 +89,86 @@ TEST(CommitMetricsTest, TestSimple) { ASSERT_EQ(200, counter); } +TEST(CommitMetricsTest, TestReportCommitFromStats) { + auto metrics = std::make_shared(); + + std::vector append_table_files; + append_table_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 201, 1001, "a1")); + append_table_files.push_back(CreateEntry(FileKind::Delete(), 2, 3, 302, 1002, "a2")); + + std::vector append_changelog_files; + append_changelog_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 202, 2001, "c1")); + append_changelog_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 301, 2002, "c2")); + + std::vector compact_table_files; + compact_table_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 203, 3001, "k1")); + compact_table_files.push_back(CreateEntry(FileKind::Delete(), 3, 5, 106, 3002, "k2")); + + std::vector compact_changelog_files; + compact_changelog_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 205, 4001, "ck1")); + compact_changelog_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 307, 4002, "ck2")); + + CommitStats stats(append_table_files, append_changelog_files, compact_table_files, + compact_changelog_files, + /*commit_duration=*/3000, /*generated_snapshots=*/2, /*attempts=*/4, + /*last_committed_snapshot_id=*/10); + CommitMetrics::ReportCommit(metrics, stats); + + ASSERT_OK_AND_ASSIGN(auto last_commit_duration, + metrics->GetCounter(CommitMetrics::LAST_COMMIT_DURATION)); + EXPECT_EQ(3000, last_commit_duration); + ASSERT_OK_AND_ASSIGN(auto last_commit_attempts, + metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS)); + EXPECT_EQ(4, last_commit_attempts); + ASSERT_OK_AND_ASSIGN(auto last_table_files_added, + metrics->GetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED)); + EXPECT_EQ(2, last_table_files_added); + ASSERT_OK_AND_ASSIGN(auto last_table_files_deleted, + metrics->GetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED)); + EXPECT_EQ(2, last_table_files_deleted); + ASSERT_OK_AND_ASSIGN(auto last_table_files_appended, + metrics->GetCounter(CommitMetrics::LAST_TABLE_FILES_APPENDED)); + EXPECT_EQ(2, last_table_files_appended); + ASSERT_OK_AND_ASSIGN(auto last_table_files_compacted, + metrics->GetCounter(CommitMetrics::LAST_TABLE_FILES_COMMIT_COMPACTED)); + EXPECT_EQ(2, last_table_files_compacted); + ASSERT_OK_AND_ASSIGN(auto last_changelog_files_appended, + metrics->GetCounter(CommitMetrics::LAST_CHANGELOG_FILES_APPENDED)); + EXPECT_EQ(2, last_changelog_files_appended); + ASSERT_OK_AND_ASSIGN(auto last_changelog_files_compacted, + metrics->GetCounter(CommitMetrics::LAST_CHANGELOG_FILES_COMMIT_COMPACTED)); + EXPECT_EQ(2, last_changelog_files_compacted); + ASSERT_OK_AND_ASSIGN(auto last_generated_snapshots, + metrics->GetCounter(CommitMetrics::LAST_GENERATED_SNAPSHOTS)); + EXPECT_EQ(2, last_generated_snapshots); + ASSERT_OK_AND_ASSIGN(auto last_delta_records_appended, + metrics->GetCounter(CommitMetrics::LAST_DELTA_RECORDS_APPENDED)); + EXPECT_EQ(503, last_delta_records_appended); + ASSERT_OK_AND_ASSIGN(auto last_changelog_records_appended, + metrics->GetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_APPENDED)); + EXPECT_EQ(503, last_changelog_records_appended); + ASSERT_OK_AND_ASSIGN(auto last_delta_records_compacted, + metrics->GetCounter(CommitMetrics::LAST_DELTA_RECORDS_COMMIT_COMPACTED)); + EXPECT_EQ(309, last_delta_records_compacted); + ASSERT_OK_AND_ASSIGN( + auto last_changelog_records_compacted, + metrics->GetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_COMMIT_COMPACTED)); + EXPECT_EQ(512, last_changelog_records_compacted); + ASSERT_OK_AND_ASSIGN(auto last_partitions_written, + metrics->GetCounter(CommitMetrics::LAST_PARTITIONS_WRITTEN)); + EXPECT_EQ(3, last_partitions_written); + ASSERT_OK_AND_ASSIGN(auto last_buckets_written, + metrics->GetCounter(CommitMetrics::LAST_BUCKETS_WRITTEN)); + EXPECT_EQ(3, last_buckets_written); + ASSERT_OK_AND_ASSIGN(auto last_compaction_input_file_size, + metrics->GetCounter(CommitMetrics::LAST_COMPACTION_INPUT_FILE_SIZE)); + EXPECT_EQ(3002, last_compaction_input_file_size); + ASSERT_OK_AND_ASSIGN(auto last_compaction_output_file_size, + metrics->GetCounter(CommitMetrics::LAST_COMPACTION_OUTPUT_FILE_SIZE)); + EXPECT_EQ(3001, last_compaction_output_file_size); + ASSERT_OK_AND_ASSIGN(auto last_committed_snapshot_id, + metrics->GetCounter(CommitMetrics::LAST_COMMITTED_SNAPSHOT_ID)); + EXPECT_EQ(10, last_committed_snapshot_id); +} + } // namespace paimon::test diff --git a/src/paimon/core/operation/file_store_commit.cpp b/src/paimon/core/operation/file_store_commit.cpp index a4ce3f42..b2ba17fa 100644 --- a/src/paimon/core/operation/file_store_commit.cpp +++ b/src/paimon/core/operation/file_store_commit.cpp @@ -30,11 +30,13 @@ #include "paimon/core/manifest/index_manifest_file.h" #include "paimon/core/manifest/manifest_file.h" #include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/operation/append_only_file_store_scan.h" #include "paimon/core/operation/expire_snapshots.h" #include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" -#include "paimon/core/table/bucket_mode.h" #include "paimon/core/utils/field_mapping.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" @@ -48,6 +50,50 @@ class Schema; namespace paimon { +namespace { + +CommitScanner::ScanSupplier CreateAppendScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) { + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + AppendOnlyFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +CommitScanner::ScanSupplier CreatePkScanSupplier( + const std::shared_ptr& snapshot_manager, + const std::shared_ptr& schema_manager, + const std::shared_ptr& manifest_list, + const std::shared_ptr& manifest_file, + const std::shared_ptr& table_schema, + const std::shared_ptr& arrow_schema, const CoreOptions& options, + const std::shared_ptr& executor, const std::shared_ptr& pool) { + return [snapshot_manager, schema_manager, manifest_list, manifest_file, table_schema, + arrow_schema, options, executor, pool](const std::shared_ptr& scan_filter) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + KeyValueFileStoreScan::Create(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema, arrow_schema, scan_filter, + options, executor, pool)); + return std::unique_ptr(std::move(scan)); + }; +} + +} // namespace + Result> FileStoreCommit::Create( std::unique_ptr ctx) { if (ctx == nullptr) { @@ -70,15 +116,6 @@ Result> FileStoreCommit::Create( return Status::Invalid("not found latest schema"); } const auto& schema = table_schema.value(); - if (!schema->PrimaryKeys().empty() && - ctx->GetOptions().find("enable-pk-commit-in-inte-test") == ctx->GetOptions().end()) { - // Postpone bucket mode (bucket=-2) writes all data files to the bucket-postpone/ directory. - // A compaction job will later redistribute files into real buckets. The commit logic - // (manifest and snapshot generation) is the same as append tables, so we allow it. - if (schema->NumBuckets() != BucketModeDefine::POSTPONE_BUCKET) { - return Status::NotImplemented("not support pk table commit yet"); - } - } auto opts = schema->Options(); for (const auto& [key, value] : ctx->GetOptions()) { opts[key] = value; @@ -89,6 +126,8 @@ Result> FileStoreCommit::Create( CoreOptions::FromMap(opts, ctx->GetSpecificFileSystem())); assert(options.GetFileSystem()); assert(options.GetFileFormat()); + PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path)); if (is_object_store && !ctx->UseRESTCatalogCommit() && opts.find("enable-object-store-commit-in-inte-test") == opts.end()) { @@ -138,11 +177,23 @@ Result> FileStoreCommit::Create( snapshot_manager, path_factory, manifest_list, manifest_file, options.GetFileSystem(), options.GetExpireConfig(), ctx->GetExecutor()); + CommitScanner::ScanSupplier scan_supplier; + if (table_schema.value()->PrimaryKeys().empty()) { + scan_supplier = CreateAppendScanSupplier(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema.value(), arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); + } else { + scan_supplier = CreatePkScanSupplier(snapshot_manager, schema_manager, manifest_list, + manifest_file, table_schema.value(), arrow_schema, + options, ctx->GetExecutor(), ctx->GetMemoryPool()); + } + return std::make_unique( ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path, ctx->GetCommitUser(), options, path_factory, std::move(partition_computer), snapshot_manager, - ctx->IgnoreEmptyCommit(), ctx->UseRESTCatalogCommit(), table_schema.value(), manifest_file, - manifest_list, index_manifest_file, expire_snapshots, schema_manager); + ctx->IgnoreEmptyCommit(), ctx->UseRESTCatalogCommit(), ctx->AppendCommitCheckConflict(), + table_schema.value(), manifest_file, manifest_list, index_manifest_file, expire_snapshots, + schema_manager, std::move(scan_supplier)); } } // namespace paimon diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index 06eeeae5..76a34d5a 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp @@ -21,10 +21,13 @@ #include #include +#include #include +#include #include #include #include +#include #include #include "fmt/format.h" @@ -35,8 +38,11 @@ #include "paimon/common/executor/future.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/date_time_utils.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/catalog/catalog_snapshot_commit.h" #include "paimon/core/catalog/renaming_snapshot_commit.h" @@ -57,14 +63,19 @@ #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" #include "paimon/core/manifest/partition_entry.h" -#include "paimon/core/operation/append_only_file_store_scan.h" +#include "paimon/core/operation/commit/commit_changes_provider.h" +#include "paimon/core/operation/commit/compacted_changelog_path_resolver.h" +#include "paimon/core/operation/commit/conflict_detection.h" +#include "paimon/core/operation/commit/row_tracking_commit_utils.h" +#include "paimon/core/operation/commit/sequence_snapshot_properties.h" #include "paimon/core/operation/expire_snapshots.h" -#include "paimon/core/operation/file_store_scan.h" #include "paimon/core/operation/manifest_file_merger.h" #include "paimon/core/operation/metrics/commit_metrics.h" +#include "paimon/core/operation/metrics/commit_stats.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/utils/duration.h" #include "paimon/core/utils/file_store_path_factory.h" @@ -72,12 +83,58 @@ #include "paimon/fs/file_system.h" #include "paimon/logging.h" #include "paimon/metrics.h" -#include "paimon/scan_context.h" namespace paimon { class Executor; class MemoryPool; +namespace { + +constexpr const char* kCommitStrictModeLastSafeSnapshot = "commit.strict-mode.last-safe-snapshot"; +constexpr const char* kManifestDeleteFileDropStats = "manifest.delete-file-drop-stats"; +constexpr const char* kSequenceSnapshotOrdering = "sequence.snapshot-ordering"; +constexpr const char* kPkClusteringOverride = "pk-clustering-override"; + +bool MatchPartitionSpec(const std::map& partition, + const std::map& partition_spec) { + for (const auto& [key, value] : partition_spec) { + auto iter = partition.find(key); + if (iter == partition.end() || iter->second != value) { + return false; + } + } + return true; +} + +} // namespace + +Status FileStoreCommitImpl::ValidateCommitOptions(const CoreOptions& options) { + const auto& raw_options = options.ToMap(); + std::vector unsupported_options; + + if (raw_options.find(kCommitStrictModeLastSafeSnapshot) != raw_options.end()) { + unsupported_options.emplace_back(kCommitStrictModeLastSafeSnapshot); + } + if (raw_options.find(kManifestDeleteFileDropStats) != raw_options.end()) { + unsupported_options.emplace_back(kManifestDeleteFileDropStats); + } + if (raw_options.find(kSequenceSnapshotOrdering) != raw_options.end()) { + unsupported_options.emplace_back(kSequenceSnapshotOrdering); + } + if (raw_options.find(kPkClusteringOverride) != raw_options.end()) { + unsupported_options.emplace_back(kPkClusteringOverride); + } + + if (!unsupported_options.empty()) { + return Status::Invalid(fmt::format( + "These options are not supported by C++ commit path: {}. " + "Please use Java commit, or remove these options before creating FileStoreCommit.", + fmt::join(unsupported_options, ", "))); + } + + return Status::OK(); +} + FileStoreCommitImpl::FileStoreCommitImpl( const std::shared_ptr& pool, const std::shared_ptr& executor, const std::shared_ptr& schema, const std::string& root_path, @@ -85,16 +142,18 @@ FileStoreCommitImpl::FileStoreCommitImpl( const std::shared_ptr& path_factory, std::unique_ptr partition_computer, const std::shared_ptr& snapshot_manager, bool ignore_empty_commit, - bool use_rest_catalog_commit, const std::shared_ptr& table_schema, + bool use_rest_catalog_commit, bool append_commit_check_conflict, + const std::shared_ptr& table_schema, const std::shared_ptr& manifest_file, const std::shared_ptr& manifest_list, const std::shared_ptr& index_manifest_file, const std::shared_ptr& expire_snapshots, - const std::shared_ptr& schema_manager) + const std::shared_ptr& schema_manager, CommitScanner::ScanSupplier scan_supplier) : memory_pool_(pool), executor_(executor), schema_(schema), root_path_(root_path), + table_name_(PathUtil::GetName(root_path)), commit_user_(commit_user), options_(options), path_factory_(path_factory), @@ -102,8 +161,17 @@ FileStoreCommitImpl::FileStoreCommitImpl( partition_computer_(std::move(partition_computer)), snapshot_manager_(snapshot_manager), ignore_empty_commit_(ignore_empty_commit), + append_commit_check_conflict_(append_commit_check_conflict), + retry_waiter_(options.GetCommitMinRetryWait(), options.GetCommitMaxRetryWait()), num_bucket_(options.GetBucket()), + bucket_mode_(ResolveBucketMode(options.GetBucket(), table_schema)), table_schema_(table_schema), + commit_scanner_(std::make_shared( + snapshot_manager, schema_manager, manifest_list, manifest_file, index_manifest_file, + table_schema, schema, options, executor, pool, partition_computer_.get(), + std::move(scan_supplier))), + conflict_detection_(table_schema, options, snapshot_manager_, manifest_list, manifest_file, + commit_scanner_), manifest_file_(manifest_file), manifest_list_(manifest_list), index_manifest_file_(index_manifest_file), @@ -131,7 +199,16 @@ Status FileStoreCommitImpl::DropPartition( } std::string log_msg = fmt::format("Ready to drop partitions {}", partitions); PAIMON_LOG_DEBUG(logger_, "%s", log_msg.c_str()); - return TryOverwrite(partitions, {}, commit_identifier, std::nullopt); + PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] int32_t attempt, + TryOverwrite(partitions, /*changes=*/{}, /*index_entries=*/{}, + commit_identifier, std::nullopt, /*properties=*/{})); + return Status::OK(); +} + +FileStoreCommit& FileStoreCommitImpl::RowIdCheckConflict( + std::optional row_id_check_from_snapshot) { + conflict_detection_.SetRowIdCheckFromSnapshot(row_id_check_from_snapshot); + return *this; } Result FileStoreCommitImpl::FilterAndCommit( @@ -143,8 +220,15 @@ Result FileStoreCommitImpl::FilterAndCommit( committables.push_back(CreateManifestCommittable(identifier, msgs, watermark)); } + std::vector> sorted_committables = committables; + std::sort(sorted_committables.begin(), sorted_committables.end(), + [](const std::shared_ptr& lhs, + const std::shared_ptr& rhs) { + return lhs->Identifier() < rhs->Identifier(); + }); + PAIMON_ASSIGN_OR_RAISE(std::vector> retry_committables, - FilterCommitted(committables)); + FilterCommitted(sorted_committables)); if (!retry_committables.empty()) { PAIMON_RETURN_NOT_OK(CheckFilesExistence(retry_committables)); for (const auto& committable : retry_committables) { @@ -160,43 +244,62 @@ Status FileStoreCommitImpl::CheckFilesExistence( for (const auto& committable : committables) { for (const auto& message : committable->FileCommittables()) { auto msg = dynamic_cast(message.get()); - if (msg) { - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr data_file_path_factory, - path_factory_->CreateDataFilePathFactory(msg->Partition(), msg->Bucket())); - auto collect_files = - [&all_paths, data_file_path_factory]( - const std::vector>& file_metas) { - for (const auto& file_meta : file_metas) { - auto paths = data_file_path_factory->CollectFiles(file_meta); - all_paths.insert(all_paths.end(), paths.begin(), paths.end()); - } - }; - // skip compact before files, deleted index files - DataIncrement new_files_increment = msg->GetNewFilesIncrement(); - collect_files(new_files_increment.NewFiles()); - collect_files(new_files_increment.ChangelogFiles()); - auto new_data_index_metas = new_files_increment.NewIndexFiles(); - for (const auto& data_index_meta : new_data_index_metas) { - all_paths.push_back( - path_factory_->ToIndexFilePath(data_index_meta->FileName())); + if (msg == nullptr) { + return Status::Invalid("fail to cast commit message to impl"); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr data_file_path_factory, + path_factory_->CreateDataFilePathFactory(msg->Partition(), msg->Bucket())); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_path_factory, + path_factory_->CreateIndexFileFactory(msg->Partition(), msg->Bucket())); + auto collect_files = [&all_paths, data_file_path_factory]( + const std::vector>& file_metas) { + for (const auto& file_meta : file_metas) { + auto paths = data_file_path_factory->CollectFiles(file_meta); + all_paths.insert(all_paths.end(), paths.begin(), paths.end()); } + }; + DataIncrement new_files_increment = msg->GetNewFilesIncrement(); + collect_files(new_files_increment.NewFiles()); + collect_files(new_files_increment.ChangelogFiles()); + auto new_data_index_metas = new_files_increment.NewIndexFiles(); + for (const auto& data_index_meta : new_data_index_metas) { + all_paths.push_back(index_file_path_factory->ToPath(data_index_meta)); + } - CompactIncrement compact_increment = msg->GetCompactIncrement(); - collect_files(compact_increment.CompactBefore()); - collect_files(compact_increment.CompactAfter()); - auto new_compact_index_metas = compact_increment.NewIndexFiles(); - for (const auto& compact_index_meta : new_compact_index_metas) { - all_paths.push_back( - path_factory_->ToIndexFilePath(compact_index_meta->FileName())); - } - } else { - return Status::Invalid("fail to cast commit message to impl"); + CompactIncrement compact_increment = msg->GetCompactIncrement(); + collect_files(compact_increment.CompactAfter()); + auto new_compact_index_metas = compact_increment.NewIndexFiles(); + for (const auto& compact_index_meta : new_compact_index_metas) { + all_paths.push_back(index_file_path_factory->ToPath(compact_index_meta)); } + + // skip compact before files, deleted index files } } - std::vector>>> file_exists_futures; + + // Resolve compacted changelog files to their real file paths + std::vector resolved_paths; + resolved_paths.reserve(all_paths.size()); for (const auto& path : all_paths) { + resolved_paths.push_back(CompactedChangelogPathResolver::Resolve(path)); + } + + // Deduplicate paths as multiple compacted changelog references may resolve to the same + // physical file + std::unordered_set deduplicated_paths_set; + deduplicated_paths_set.reserve(resolved_paths.size()); + std::vector deduplicated_paths; + deduplicated_paths.reserve(resolved_paths.size()); + for (const auto& path : resolved_paths) { + if (deduplicated_paths_set.insert(path).second) { + deduplicated_paths.push_back(path); + } + } + + std::vector>>> file_exists_futures; + for (const auto& path : deduplicated_paths) { file_exists_futures.push_back( Via(executor_.get(), [this, path]() -> Result> { PAIMON_ASSIGN_OR_RAISE(bool exist, fs_->Exists(path)); @@ -235,12 +338,13 @@ Result>> FileStoreCommitImpl::F } for (size_t i = 1; i < committables.size(); i++) { - if (committables[i]->Identifier() < committables[i - 1]->Identifier()) { + if (committables[i]->Identifier() <= committables[i - 1]->Identifier()) { return Status::Invalid( "Committables must be sorted according to identifiers before filtering. This is " "unexpected."); } } + // TODO(yonghao.fyh): support commit strict mode last safe snapshot PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, snapshot_manager_->LatestSnapshotOfUser(commit_user_)); if (latest_snapshot) { @@ -250,7 +354,7 @@ Result>> FileStoreCommitImpl::F if (committable->Identifier() > latest_snapshot.value().CommitIdentifier()) { result.push_back(committable); } else { - // TODO(yonghao.fyh): callback + // TODO(yonghao.fyh): support callback } } return result; @@ -261,189 +365,340 @@ Result>> FileStoreCommitImpl::F } Status FileStoreCommitImpl::Overwrite( - const std::vector>& partitions, + const std::map& partition, const std::vector>& commit_messages, int64_t identifier, std::optional watermark) { std::shared_ptr committable = CreateManifestCommittable(identifier, commit_messages, watermark); - std::vector append_table_files; - std::vector append_changelog_files; - std::vector compact_table_files; - std::vector compact_changelog_files; - std::vector append_table_index_files; - std::vector compact_table_index_files; - PAIMON_RETURN_NOT_OK(CollectChanges(committable->FileCommittables(), &append_table_files, - &append_changelog_files, &compact_table_files, - &compact_changelog_files, &append_table_index_files, - &compact_table_index_files)); - if (!append_table_index_files.empty()) { - return Status::NotImplemented("Overwrite not support index for now"); - } - return TryOverwrite(partitions, append_table_files, identifier, watermark); + PAIMON_LOG_INFO(logger_, "Ready to overwrite to table %s, number of commit messages: %zu", + table_name_.c_str(), committable->FileCommittables().size()); + PAIMON_ASSIGN_OR_RAISE(std::string committable_str, committable->ToString()); + std::string partition_str = fmt::format("{}", partition); + std::string properties_str = fmt::format("{}", committable->Properties()); + PAIMON_LOG_DEBUG(logger_, + "Ready to overwrite partitions %s\nManifestCommittable: %s\nProperties: " + "%s", + partition_str.c_str(), committable_str.c_str(), properties_str.c_str()); + + Duration duration; + int32_t generated_snapshot = 0; + int32_t attempt = 0; + + std::vector> partitions; + if (!partition.empty()) { + partitions.push_back(partition); + } + + PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes, + CollectChanges(committable->FileCommittables())); + ManifestEntryChanges report_changes = changes; + report_changes.append_changelog.clear(); + report_changes.compact_changelog.clear(); + ScopeGuard report_guard([&]() { + PAIMON_LOG_INFO(logger_, "Finished overwrite to table %s, duration %ld ms", + table_name_.c_str(), duration.Get()); + ReportCommit(report_changes, duration.Get(), generated_snapshot, attempt); + }); + + PAIMON_RETURN_NOT_OK(ExecuteOverwrite(partitions, &changes, identifier, watermark, committable, + &generated_snapshot, &attempt)); + return Status::OK(); } Result FileStoreCommitImpl::FilterAndOverwrite( - const std::vector>& partitions, + const std::map& partition, const std::vector>& commit_messages, int64_t identifier, std::optional watermark) { std::shared_ptr committable = CreateManifestCommittable(identifier, commit_messages, watermark); + PAIMON_LOG_INFO(logger_, "Ready to overwrite to table %s, number of commit messages: %zu", + table_name_.c_str(), committable->FileCommittables().size()); + PAIMON_ASSIGN_OR_RAISE(std::string committable_str, committable->ToString()); + std::string partition_str = fmt::format("{}", partition); + std::string properties_str = fmt::format("{}", committable->Properties()); + PAIMON_LOG_DEBUG(logger_, + "Ready to overwrite partitions %s\nManifestCommittable: %s\nProperties: " + "%s", + partition_str.c_str(), committable_str.c_str(), properties_str.c_str()); + + Duration duration; + int32_t generated_snapshot = 0; + int32_t attempt = 0; + + std::vector> partitions; + if (!partition.empty()) { + partitions.push_back(partition); + } + std::vector> committables; committables.push_back(committable); PAIMON_ASSIGN_OR_RAISE(std::vector> actual_committables, FilterCommitted(committables)); if (!actual_committables.empty()) { - std::vector append_table_files; - std::vector append_changelog_files; - std::vector compact_table_files; - std::vector compact_changelog_files; - std::vector append_table_index_files; - std::vector compact_table_index_files; - PAIMON_RETURN_NOT_OK(CollectChanges(actual_committables[0]->FileCommittables(), - &append_table_files, &append_changelog_files, - &compact_table_files, &compact_changelog_files, - &append_table_index_files, &compact_table_index_files)); - if (!append_table_index_files.empty()) { - return Status::NotImplemented("FilterAndOverwrite not support index for now"); - } - PAIMON_RETURN_NOT_OK(TryOverwrite(partitions, append_table_files, identifier, watermark)); + PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes, + CollectChanges(actual_committables[0]->FileCommittables())); + ManifestEntryChanges report_changes = changes; + report_changes.append_changelog.clear(); + report_changes.compact_changelog.clear(); + ScopeGuard report_guard([&]() { + PAIMON_LOG_INFO(logger_, "Finished overwrite to table %s, duration %ld ms", + table_name_.c_str(), duration.Get()); + ReportCommit(report_changes, duration.Get(), generated_snapshot, attempt); + }); + + PAIMON_RETURN_NOT_OK(ExecuteOverwrite(partitions, &changes, identifier, watermark, + actual_committables[0], &generated_snapshot, + &attempt)); + } else { + // Align with Java: filtered duplicate is treated as one resolved commit attempt. + attempt = 1; + PAIMON_LOG_INFO(logger_, "Finished overwrite to table %s, duration %ld ms", + table_name_.c_str(), duration.Get()); } return actual_committables.size(); } +Status FileStoreCommitImpl::ExecuteOverwrite( + const std::vector>& partitions, + ManifestEntryChanges* changes, int64_t identifier, std::optional watermark, + const std::shared_ptr& committable, int32_t* generated_snapshot, + int32_t* attempt) { + if (!changes->append_changelog.empty() || !changes->compact_changelog.empty()) { + std::string warning = + "Overwrite mode currently does not commit any changelog.\n" + "Please make sure that the partition you're overwriting is not being consumed by a " + "streaming reader.\n" + "Ignored changelog files are:\n"; + for (const auto& entry : changes->append_changelog) { + warning += fmt::format(" * {}\n", entry.ToString()); + } + for (const auto& entry : changes->compact_changelog) { + warning += fmt::format(" * {}\n", entry.ToString()); + } + PAIMON_LOG_WARN(logger_, "%s", warning.c_str()); + } + + bool skip_overwrite = false; + std::vector> overwrite_partitions = partitions; + if (!table_schema_->PartitionKeys().empty() && options_.DynamicPartitionOverwrite()) { + if (changes->append_table_files.empty()) { + // in dynamic mode, if there is no changes to commit, no data will be deleted + skip_overwrite = true; + } else { + std::set> dynamic_partitions; + for (const auto& entry : changes->append_table_files) { + std::map partition_map; + PAIMON_ASSIGN_OR_RAISE(partition_map, PartitionToMap(entry.Partition())); + dynamic_partitions.insert(std::move(partition_map)); + } + overwrite_partitions.assign(dynamic_partitions.begin(), dynamic_partitions.end()); + } + } else if (!partitions.empty()) { + for (const auto& entry : changes->append_table_files) { + std::map partition_map; + PAIMON_ASSIGN_OR_RAISE(partition_map, PartitionToMap(entry.Partition())); + bool belongs_to_overwrite_partition = false; + for (const auto& partition_spec : partitions) { + if (MatchPartitionSpec(partition_map, partition_spec)) { + belongs_to_overwrite_partition = true; + break; + } + } + if (!belongs_to_overwrite_partition) { + return Status::Invalid(fmt::format( + "Trying to overwrite partitions {}, but the changes in {} does not belong to " + "this partition", + partitions, partition_map)); + } + } + } + + bool with_compact = + !changes->compact_table_files.empty() || !changes->compact_index_files.empty(); + if (!with_compact) { + // In overwrite mode, opportunistically promote non-overlapping L0 files (per + // partition+bucket) to higher levels to reduce future compaction and read amplification, + // without changing semantics. + PAIMON_ASSIGN_OR_RAISE(changes->append_table_files, + TryUpgrade(changes->append_table_files)); + } + + // overwrite new files + if (!skip_overwrite) { + PAIMON_ASSIGN_OR_RAISE(int32_t cnt, + TryOverwrite(overwrite_partitions, changes->append_table_files, + changes->append_index_files, identifier, watermark, + committable->Properties())); + *attempt += cnt; + *generated_snapshot += 1; + } + + if (with_compact) { + PAIMON_ASSIGN_OR_RAISE(int32_t cnt, + TryCommit(changes->compact_table_files, /*changelog_files=*/{}, + changes->compact_index_files, identifier, watermark, + committable->Properties(), Snapshot::CommitKind::Compact(), + /*detect_conflicts=*/true)); + *attempt += cnt; + *generated_snapshot += 1; + } + + return Status::OK(); +} + Result FileStoreCommitImpl::GetLastCommitTableRequest() { return snapshot_commit_->GetLastCommitTableRequest(); } Result> FileStoreCommitImpl::GetAllFiles( - const Snapshot& snapshot, const std::vector>& partitions) { - auto scan_filter = std::make_shared(/*predicate=*/nullptr, partitions, - /*bucket_filter=*/std::nullopt); - PAIMON_ASSIGN_OR_RAISE( - auto scan, AppendOnlyFileStoreScan::Create( - snapshot_manager_, schema_manager_, manifest_list_, manifest_file_, - table_schema_, schema_, scan_filter, options_, executor_, memory_pool_)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, - scan->WithSnapshot(snapshot)->CreatePlan()); - // scan existing file metas - return plan->Files(); + const Snapshot& snapshot, + const std::vector>& partitions) const { + return commit_scanner_->ReadAllEntriesFromPartitions(snapshot, partitions); } -Status FileStoreCommitImpl::TryOverwrite( - const std::vector>& partitions, - const std::vector& changes, int64_t commit_identifier, - std::optional watermark) { - int32_t retry_count = 0; - while (true) { - PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, - snapshot_manager_->LatestSnapshot()); - std::vector changes_with_overwrite; - if (latest_snapshot) { - PAIMON_ASSIGN_OR_RAISE(std::vector entries, - GetAllFiles(latest_snapshot.value(), partitions)); - for (const auto& entry : entries) { - changes_with_overwrite.emplace_back(FileKind::Delete(), entry.Partition(), - entry.Bucket(), entry.TotalBuckets(), - entry.File()); +Result> FileStoreCommitImpl::PartitionToMap( + const BinaryRow& partition) const { + std::vector> part_values; + PAIMON_ASSIGN_OR_RAISE(part_values, partition_computer_->GeneratePartitionVector(partition)); + std::map partition_map; + for (const auto& [key, value] : part_values) { + partition_map[key] = value; + } + return partition_map; +} + +Result> FileStoreCommitImpl::TryUpgrade( + const std::vector& append_files) const { + if (!options_.OverwriteUpgrade()) { + return append_files; + } + + if (table_schema_->PrimaryKeys().empty()) { + return append_files; + } + + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys, + table_schema_->TrimmedPrimaryKeys()); + PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_key_fields, + table_schema_->GetFields(trimmed_primary_keys)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr key_comparator, + FieldsComparator::Create(trimmed_primary_key_fields, + options_.SequenceFieldSortOrderIsAscending())); + + for (const auto& entry : append_files) { + if (entry.Level() > 0 || entry.Bucket() < 0) { + return append_files; + } + } + + std::unordered_map, std::vector> buckets; + for (const auto& entry : append_files) { + buckets[std::make_pair(entry.Partition(), entry.Bucket())].emplace_back(entry); + } + + std::vector results; + int32_t max_level = options_.GetNumLevels() - 1; + for (auto& [_, entries] : buckets) { + std::vector new_entries = entries; + std::sort(new_entries.begin(), new_entries.end(), + [&key_comparator](const ManifestEntry& a, const ManifestEntry& b) { + return key_comparator->CompareTo(a.MinKey(), b.MinKey()) < 0; + }); + + bool overlap = false; + for (size_t i = 0; i + 1 < new_entries.size(); ++i) { + if (key_comparator->CompareTo(new_entries[i].MaxKey(), new_entries[i + 1].MinKey()) >= + 0) { + overlap = true; + break; } } - changes_with_overwrite.insert(changes_with_overwrite.end(), changes.begin(), changes.end()); - PAIMON_ASSIGN_OR_RAISE(bool commit_success, - TryCommitOnce(changes_with_overwrite, /*index_entries=*/{}, - commit_identifier, watermark, - /*log_offsets=*/{}, /*properties=*/{}, - Snapshot::CommitKind::Overwrite(), latest_snapshot, - /*need_conflict_check=*/true)); - if (commit_success) { - break; + + if (overlap) { + results.insert(results.end(), entries.begin(), entries.end()); + continue; } - if (retry_count >= options_.GetCommitMaxRetries()) { - return Status::Invalid( - fmt::format("Commit failed after {} attempts, there maybe exist commit conflicts " - "between multiple jobs.", - options_.GetCommitMaxRetries())); + + PAIMON_LOG_INFO(logger_, "%s", "Upgraded for overwrite commit."); + for (const auto& entry : new_entries) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr upgraded_file, + entry.File()->Upgrade(max_level)); + results.emplace_back(entry.Kind(), entry.Partition(), entry.Bucket(), + entry.TotalBuckets(), upgraded_file); } - retry_count++; } - return Status::OK(); + + return results; +} + +Result FileStoreCommitImpl::TryOverwrite( + const std::vector>& partitions, + const std::vector& changes, const std::vector& index_entries, + int64_t commit_identifier, std::optional watermark, + const std::map& properties) { + std::shared_ptr changes_provider = + commit_scanner_->OverwriteChangesProvider(partitions, changes, index_entries); + return TryCommit(changes_provider, commit_identifier, watermark, properties, + Snapshot::CommitKind::Overwrite(), /*detect_conflicts=*/true); } Status FileStoreCommitImpl::Commit(const std::shared_ptr& committable, bool check_append_files) { - std::vector append_table_files; - std::vector append_changelog_files; - std::vector compact_table_files; - std::vector compact_changelog_files; - std::vector append_table_index_files; - std::vector compact_table_index_files; - PAIMON_RETURN_NOT_OK(CollectChanges(committable->FileCommittables(), &append_table_files, - &append_changelog_files, &compact_table_files, - &compact_changelog_files, &append_table_index_files, - &compact_table_index_files)); + PAIMON_LOG_INFO(logger_, "Ready to commit to table %s, number of commit messages: %zu", + table_name_.c_str(), committable->FileCommittables().size()); + PAIMON_ASSIGN_OR_RAISE(std::string committable_str, committable->ToString()); + PAIMON_LOG_DEBUG(logger_, "Ready to commit\n%s", committable_str.c_str()); - int32_t attempt = 0; - int32_t generated_snapshot = 0; Duration duration; - if (!ignore_empty_commit_ || !append_table_files.empty() || !append_table_index_files.empty()) { - PAIMON_ASSIGN_OR_RAISE(int32_t cnt, - TryCommit(append_table_files, append_table_index_files, - committable->Identifier(), committable->Watermark(), - committable->LogOffsets(), committable->Properties(), - Snapshot::CommitKind::Append(), check_append_files)); + int32_t generated_snapshot = 0; + int32_t attempt = 0; + + PAIMON_ASSIGN_OR_RAISE(ManifestEntryChanges changes, + CollectChanges(committable->FileCommittables())); + ScopeGuard report_guard([&]() { + PAIMON_LOG_INFO(logger_, + "Finished (Uncertain of success) commit to table %s, duration %ld ms", + table_name_.c_str(), duration.Get()); + ReportCommit(changes, duration.Get(), generated_snapshot, attempt); + }); + + if (!ignore_empty_commit_ || changes.HasAppendChanges()) { + Snapshot::CommitKind commit_kind = Snapshot::CommitKind::Append(); + if (append_commit_check_conflict_) { + check_append_files = true; + } + + if (conflict_detection_.ShouldBeOverwriteCommit(changes.append_table_files, + changes.append_index_files)) { + commit_kind = Snapshot::CommitKind::Overwrite(); + check_append_files = true; + } + if (conflict_detection_.HasRowIdCheckFromSnapshot()) { + check_append_files = true; + } + if (changes.HasGlobalIndexFileAdditions()) { + check_append_files = true; + } + + PAIMON_ASSIGN_OR_RAISE( + int32_t cnt, TryCommit(changes.append_table_files, changes.append_changelog, + changes.append_index_files, committable->Identifier(), + committable->Watermark(), committable->Properties(), commit_kind, + check_append_files)); attempt += cnt; - ++generated_snapshot; + generated_snapshot += 1; } - if (!compact_table_files.empty() || !compact_table_index_files.empty()) { - PAIMON_ASSIGN_OR_RAISE( - int32_t cnt, TryCommit(compact_table_files, compact_table_index_files, - committable->Identifier(), committable->Watermark(), - committable->LogOffsets(), committable->Properties(), - Snapshot::CommitKind::Compact(), /*check_append_files=*/true)); + if (changes.HasCompactChanges()) { + PAIMON_ASSIGN_OR_RAISE(int32_t cnt, + TryCommit(changes.compact_table_files, changes.compact_changelog, + changes.compact_index_files, committable->Identifier(), + committable->Watermark(), committable->Properties(), + Snapshot::CommitKind::Compact(), + /*detect_conflicts=*/true)); attempt += cnt; - ++generated_snapshot; - } - auto table_files_added = static_cast(append_table_files.size()); - int32_t table_files_deleted = 0; - int64_t compaction_input_file_size = 0; - int64_t compaction_output_file_size = 0; - for (const auto& entry : compact_table_files) { - const auto& kind = entry.Kind(); - if (kind == FileKind::Add()) { - ++table_files_added; - compaction_output_file_size += entry.File()->file_size; - } else if (kind == FileKind::Delete()) { - ++table_files_deleted; - compaction_input_file_size += entry.File()->file_size; - } + generated_snapshot += 1; } - metrics_->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, duration.Get()); - metrics_->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, attempt); - metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED, table_files_added); - metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED, table_files_deleted); - metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_APPENDED, append_table_files.size()); - metrics_->SetCounter(CommitMetrics::LAST_TABLE_FILES_COMMIT_COMPACTED, - compact_table_files.size()); - metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_APPENDED, - append_changelog_files.size()); - metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_COMMIT_COMPACTED, - compact_changelog_files.size()); - metrics_->SetCounter(CommitMetrics::LAST_GENERATED_SNAPSHOTS, generated_snapshot); - metrics_->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_APPENDED, RowCounts(append_table_files)); - metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_APPENDED, - RowCounts(append_changelog_files)); - metrics_->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_COMMIT_COMPACTED, - RowCounts(compact_table_files)); - metrics_->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_COMMIT_COMPACTED, - RowCounts(compact_changelog_files)); - metrics_->SetCounter(CommitMetrics::LAST_PARTITIONS_WRITTEN, - NumChangedPartitions({append_table_files, compact_table_files})); - metrics_->SetCounter(CommitMetrics::LAST_BUCKETS_WRITTEN, - NumChangedBuckets({append_table_files, compact_table_files})); - metrics_->SetCounter(CommitMetrics::LAST_COMPACTION_INPUT_FILE_SIZE, - compaction_input_file_size); - metrics_->SetCounter(CommitMetrics::LAST_COMPACTION_OUTPUT_FILE_SIZE, - compaction_output_file_size); return Status::OK(); } @@ -456,114 +711,157 @@ Status FileStoreCommitImpl::Commit( } Result FileStoreCommitImpl::TryCommit(const std::vector& delta_files, + const std::vector& changelog_files, const std::vector& index_entries, int64_t identifier, std::optional watermark, - std::map log_offsets, const std::map& properties, Snapshot::CommitKind commit_kind, - bool check_append_files) { + bool detect_conflicts) { + std::shared_ptr changes_provider = + CommitChangesProvider::Provider(delta_files, changelog_files, index_entries); + return TryCommit(changes_provider, identifier, watermark, properties, commit_kind, + detect_conflicts); +} + +Result FileStoreCommitImpl::TryCommit( + const std::shared_ptr& changes_provider, int64_t identifier, + std::optional watermark, const std::map& properties, + Snapshot::CommitKind commit_kind, bool detect_conflicts) { int32_t retry_count = 0; int64_t start_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000; + std::optional retry_start_snapshot_id; while (true) { PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, snapshot_manager_->LatestSnapshot()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr commit_changes, + changes_provider->Provide(latest_snapshot)); PAIMON_ASSIGN_OR_RAISE( bool commit_success, - TryCommitOnce(delta_files, index_entries, identifier, watermark, log_offsets, - properties, commit_kind, latest_snapshot, check_append_files)); + TryCommitOnce(commit_changes->delta_files, commit_changes->changelog_files, + commit_changes->index_entries, identifier, watermark, properties, + commit_kind, latest_snapshot, detect_conflicts, retry_start_snapshot_id)); if (commit_success) { break; } + retry_start_snapshot_id = latest_snapshot + ? std::optional(latest_snapshot.value().Id() + 1) + : std::optional(Snapshot::FIRST_SNAPSHOT_ID); int64_t current_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000; if (current_millis - start_millis > options_.GetCommitTimeout() || retry_count >= options_.GetCommitMaxRetries()) { return Status::Invalid( fmt::format("Commit failed after {} millis with {} retries, there maybe exist " "commit conflicts between multiple jobs.", - options_.GetCommitTimeout(), options_.GetCommitMaxRetries())); + options_.GetCommitTimeout(), retry_count)); } + retry_waiter_.RetryWait(retry_count); retry_count++; } return retry_count + 1; } -Result>> FileStoreCommitImpl::ChangedPartitions( - const std::vector& data_files, - const std::vector& index_files) const { - std::set> partitions; - auto add_partition = [&, this](const BinaryRow& partition_row) -> Status { - std::vector> part_values; - PAIMON_ASSIGN_OR_RAISE(part_values, - partition_computer_->GeneratePartitionVector(partition_row)); - if (part_values.empty()) { - return Status::OK(); - } - std::map part_values_map; - for (const auto& [key, value] : part_values) { - part_values_map[key] = value; +Result FileStoreCommitImpl::CheckCommitted(const std::optional& latest_snapshot, + std::optional retry_start_snapshot_id, + int64_t identifier, + const Snapshot::CommitKind& commit_kind) const { + if (!latest_snapshot || !retry_start_snapshot_id || + retry_start_snapshot_id.value() > latest_snapshot.value().Id()) { + return false; + } + + for (int64_t snapshot_id = retry_start_snapshot_id.value(); + snapshot_id <= latest_snapshot.value().Id(); ++snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id)); + if (snapshot.CommitUser() == commit_user_ && snapshot.CommitIdentifier() == identifier && + snapshot.GetCommitKind() == commit_kind) { + return true; } - partitions.insert(part_values_map); + } + return false; +} + +Status FileStoreCommitImpl::CheckSameBucketFromSnapshot( + const std::vector& delta_entries, + const std::optional& latest_snapshot) const { + if (!latest_snapshot) { return Status::OK(); - }; + } - for (const ManifestEntry& entry : data_files) { - PAIMON_RETURN_NOT_OK(add_partition(entry.Partition())); + std::unordered_map expected_total_buckets; + PAIMON_RETURN_NOT_OK(conflict_detection_.CollectUncheckedBucketPartitions( + delta_entries, &expected_total_buckets)); + if (expected_total_buckets.empty()) { + return Status::OK(); } - for (const IndexManifestEntry& entry : index_files) { - if (entry.index_file->IndexType() == DeletionVectorsIndexFile::DELETION_VECTORS_INDEX) { - PAIMON_RETURN_NOT_OK(add_partition(entry.partition)); - } + + std::vector changed_partitions; + changed_partitions.reserve(expected_total_buckets.size()); + for (const auto& [partition, _] : expected_total_buckets) { + changed_partitions.push_back(partition); } - return partitions; -} -Result> FileStoreCommitImpl::ReadAllEntriesFromChangedPartitions( - const Snapshot& latest_snapshot, - const std::set>& partitions) const { - std::vector> partition_filters(partitions.begin(), - partitions.end()); - auto scan_filter = std::make_shared(/*predicate=*/nullptr, partition_filters, - /*bucket_filter=*/std::nullopt); PAIMON_ASSIGN_OR_RAISE( - auto scan, AppendOnlyFileStoreScan::Create( - snapshot_manager_, schema_manager_, manifest_list_, manifest_file_, - table_schema_, schema_, scan_filter, options_, executor_, memory_pool_)); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, - scan->WithSnapshot(latest_snapshot)->CreatePlan()); - // scan existing file metas - return plan->Files(); -} - -Status FileStoreCommitImpl::NoConflictsOrFail(const std::string& base_commit_user, - const std::vector& base_entries, - const std::vector& changes) const { - ScopeGuard guard([&]() { - PAIMON_LOG_WARN(logger_, "File deletion conflicts detected! Give up committing. %s", - base_commit_user.c_str()); - }); - std::vector all_entries = base_entries; - all_entries.insert(all_entries.end(), changes.begin(), changes.end()); - std::vector merged_entries; - PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(all_entries, &merged_entries)); - for (const auto& entry : merged_entries) { - if (entry.Kind() == FileKind::Delete()) { - return Status::Invalid(fmt::format( - "Trying to delete file {} which is not previously added.", entry.FileName())); + auto previous_total_buckets, + commit_scanner_->ReadTotalBuckets(latest_snapshot.value(), changed_partitions)); + + return conflict_detection_.CheckSameBucketByTotalBuckets(expected_total_buckets, + previous_total_buckets); +} + +bool FileStoreCommitImpl::ShouldCheckSameBucket(const Snapshot::CommitKind& commit_kind) const { + return commit_kind == Snapshot::CommitKind::Append() && + bucket_mode_ == BucketMode::HASH_FIXED && + (IsUnorderedWriteOnlyAppend() || IsWriteOnlySnapshotSequenceAppend()); +} + +bool FileStoreCommitImpl::IsUnorderedWriteOnlyAppend() const { + return options_.WriteOnly() && !options_.BucketAppendOrdered(); +} + +bool FileStoreCommitImpl::IsWriteOnlySnapshotSequenceAppend() const { + return options_.WriteOnly() && + options_.WriteSequenceNumberInitMode() == CoreOptions::SequenceNumberInitMode::SNAPSHOT; +} + +Result> FileStoreCommitImpl::MaxSequenceNumber( + const std::vector& manifests) const { + int64_t max_from_manifest = std::numeric_limits::min(); + bool found = false; + for (const auto& manifest : manifests) { + std::vector entries; + PAIMON_RETURN_NOT_OK(manifest_file_->Read( + manifest.FileName(), [](const ManifestEntry&) -> Result { return true; }, + &entries)); + std::optional current_max = + SequenceSnapshotProperties::MaxSequenceNumberFromFiles(entries); + if (current_max) { + max_from_manifest = std::max(max_from_manifest, current_max.value()); + found = true; } } - // TODO(yonghao.fyh): check for all LSM level >= 1, key ranges of files do not intersect - guard.Release(); - return Status::OK(); + + if (!found) { + return std::optional(); + } + return std::optional(max_from_manifest); } Result FileStoreCommitImpl::TryCommitOnce( const std::vector& delta_entries, + const std::vector& changelog_entries, const std::vector& index_entries, int64_t identifier, - std::optional watermark, std::map log_offsets, - const std::map& properties, Snapshot::CommitKind commit_kind, - const std::optional& latest_snapshot, bool need_conflict_check) { + std::optional watermark, const std::map& properties, + Snapshot::CommitKind commit_kind, const std::optional& latest_snapshot, + bool detect_conflicts, std::optional retry_start_snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(bool committed, CheckCommitted(latest_snapshot, retry_start_snapshot_id, + identifier, commit_kind)); + if (committed) { + return true; + } + std::vector delta_files = delta_entries; int64_t start_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000; + int64_t new_snapshot_id = Snapshot::FIRST_SNAPSHOT_ID; int64_t first_row_id_start = 0; if (latest_snapshot) { @@ -574,19 +872,68 @@ Result FileStoreCommitImpl::TryCommitOnce( } } - PAIMON_LOG_DEBUG(logger_, "Ready to commit table files to snapshot #%ld", new_snapshot_id); + // TODO(yonghao.fyh): support strict mode checker + + PAIMON_LOG_DEBUG(logger_, "Ready to commit table files to snapshot %ld", new_snapshot_id); for (const ManifestEntry& entry : delta_files) { PAIMON_LOG_DEBUG(logger_, " * %s", entry.ToString().c_str()); } + PAIMON_LOG_DEBUG(logger_, "Ready to commit changelog files to snapshot %ld", new_snapshot_id); + for (const ManifestEntry& entry : changelog_entries) { + PAIMON_LOG_DEBUG(logger_, " * %s", entry.ToString().c_str()); + } - if (need_conflict_check && latest_snapshot) { - std::set> changed_partitions; - PAIMON_ASSIGN_OR_RAISE(changed_partitions, ChangedPartitions(delta_files, index_entries)); - PAIMON_ASSIGN_OR_RAISE( - std::vector base_data_files, - ReadAllEntriesFromChangedPartitions(latest_snapshot.value(), changed_partitions)); - PAIMON_RETURN_NOT_OK( - NoConflictsOrFail(latest_snapshot.value().CommitUser(), base_data_files, delta_files)); + bool discard_duplicate = + options_.CommitDiscardDuplicateFiles() && commit_kind == Snapshot::CommitKind::Append(); + bool check_conflicts = latest_snapshot.has_value() && (discard_duplicate || detect_conflicts); + // By default, if checkConflicts is required, we do not have to do the extra check bucket + // here. + if (!check_conflicts && ShouldCheckSameBucket(commit_kind)) { + PAIMON_RETURN_NOT_OK(CheckSameBucketFromSnapshot(delta_files, latest_snapshot)); + } + + if (check_conflicts) { + // latest snapshot id is different from the snapshot id we've checked for conflicts, + // so we have to check again + std::vector changed_partitions = + ManifestEntryChanges::ChangedPartitions(delta_files, index_entries); + PAIMON_ASSIGN_OR_RAISE(std::vector base_data_files, + commit_scanner_->ReadAllEntriesFromChangedPartitions( + latest_snapshot.value(), changed_partitions)); + + if (discard_duplicate) { + std::unordered_set base_identifiers; + base_identifiers.reserve(base_data_files.size()); + for (const auto& entry : base_data_files) { + base_identifiers.insert(entry.CreateIdentifier()); + } + + delta_files.erase( + std::remove_if(delta_files.begin(), delta_files.end(), + [&base_identifiers](const ManifestEntry& entry) { + return base_identifiers.find(entry.CreateIdentifier()) != + base_identifiers.end(); + }), + delta_files.end()); + } + + std::optional> row_id_column_conflict_checker = + std::nullopt; + if (conflict_detection_.HasRowIdCheckFromSnapshot()) { + std::vector> delta_data_files; + delta_data_files.reserve(delta_files.size()); + for (const auto& entry : delta_files) { + delta_data_files.push_back(entry.File()); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr checker, + RowIdColumnConflictChecker::FromDataFiles(schema_manager_, delta_data_files)); + row_id_column_conflict_checker = checker; + } + + PAIMON_RETURN_NOT_OK(conflict_detection_.CheckConflicts( + latest_snapshot.value(), base_data_files, delta_files, index_entries, + row_id_column_conflict_checker, commit_kind)); } std::vector merge_before_manifests; @@ -616,25 +963,13 @@ Result FileStoreCommitImpl::TryCommitOnce( if (latest_snapshot) { old_index_manifest = latest_snapshot.value().IndexManifest(); - // TODO(yonghao.fyh): total record count should call scan when its std::nullopt - previous_total_record_count = latest_snapshot.value().TotalRecordCount() != std::nullopt - ? latest_snapshot.value().TotalRecordCount().value() - : 0; + previous_total_record_count = latest_snapshot.value().TotalRecordCount(); std::vector previous_manifests; // read all previous manifest files PAIMON_RETURN_NOT_OK( manifest_list_->ReadDataManifests(latest_snapshot.value(), &previous_manifests)); merge_before_manifests.insert(merge_before_manifests.end(), previous_manifests.begin(), previous_manifests.end()); - // read the last snapshot to complete the bucket's offsets when logOffsets does not - // contain all buckets - std::optional> latest_log_offsets = - latest_snapshot.value().LogOffsets(); - if (latest_log_offsets) { - for (const auto& [key, value] : latest_log_offsets.value()) { - log_offsets.emplace(key, value); - } - } std::optional latest_watermark = latest_snapshot.value().Watermark(); if (latest_watermark) { if (watermark == std::nullopt) { @@ -668,11 +1003,12 @@ Result FileStoreCommitImpl::TryCommitOnce( std::make_move_iterator(entries.end())); } } - // assigned snapshot id to delta files - AssignSnapshotId(new_snapshot_id, &delta_files); - // assign row id for new files - PAIMON_ASSIGN_OR_RAISE(next_row_id_start, - AssignRowTrackingMeta(first_row_id_start, &delta_files)); + + PAIMON_ASSIGN_OR_RAISE(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + new_snapshot_id, first_row_id_start, delta_files)); + next_row_id_start = assigned.next_row_id_start; + delta_files = std::move(assigned.assigned_entries); } // the added records subtract the deleted records from @@ -693,12 +1029,20 @@ Result FileStoreCommitImpl::TryCommitOnce( new_changes_manifests.end()); PAIMON_ASSIGN_OR_RAISE(delta_manifest_list, manifest_list_->Write(new_changes_manifests)); + // write changelog into manifest files + std::optional> changelog_manifest_list; + if (!changelog_entries.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::vector changelog_manifests, + manifest_file_->Write(changelog_entries)); + PAIMON_ASSIGN_OR_RAISE(changelog_manifest_list, manifest_list_->Write(changelog_manifests)); + } + PAIMON_ASSIGN_OR_RAISE(index_manifest_name, index_manifest_file_->WriteIndexFiles( old_index_manifest, index_entries)); - std::optional> changelog_manifest_list; - std::optional statistics; - int64_t changelog_record_count = 0; + std::optional statistics = + latest_snapshot ? latest_snapshot.value().Statistics() : std::nullopt; + int64_t changelog_record_count = RowCounts(changelog_entries); int64_t schema_id = 0; PAIMON_ASSIGN_OR_RAISE(std::optional> table_schema, schema_manager_->Latest()); @@ -706,6 +1050,25 @@ Result FileStoreCommitImpl::TryCommitOnce( schema_id = table_schema.value()->Id(); } + // Keep Java semantics: inherit previous stats only when schema matches. + if (statistics && latest_snapshot && latest_snapshot.value().SchemaId() != schema_id) { + PAIMON_LOG_WARN(logger_, "%s", "Schema changed, stats will not be inherited"); + statistics = std::nullopt; + } + + std::map snapshot_properties = properties; + if (options_.WriteSequenceNumberInitMode() == CoreOptions::SequenceNumberInitMode::SNAPSHOT) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_max_sequence_number, + SequenceSnapshotProperties::MaxSequenceNumber(latest_snapshot)); + if (!latest_max_sequence_number && latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE(latest_max_sequence_number, + MaxSequenceNumber(merge_before_manifests)); + } + snapshot_properties = SequenceSnapshotProperties::MergeMaxSequenceNumber( + snapshot_properties, latest_max_sequence_number, delta_files); + } + + // prepare snapshot file Snapshot new_snapshot( new_snapshot_id, schema_id, base_manifest_list.first, base_manifest_list.second, delta_manifest_list.first, delta_manifest_list.second, @@ -714,28 +1077,20 @@ Result FileStoreCommitImpl::TryCommitOnce( changelog_manifest_list ? std::optional(changelog_manifest_list.value().second) : std::nullopt, index_manifest_name, commit_user_, identifier, commit_kind, - DateTimeUtils::GetCurrentUTCTimeUs() / 1000, log_offsets, total_record_count, - delta_record_count, changelog_record_count, watermark, statistics, - properties.empty() ? std::nullopt - : std::optional>(properties), + DateTimeUtils::GetCurrentUTCTimeUs() / 1000, total_record_count, delta_record_count, + changelog_record_count, watermark, statistics, + snapshot_properties.empty() + ? std::nullopt + : std::optional>(snapshot_properties), next_row_id_start); Result commit_result = CommitSnapshotImpl(new_snapshot, delta_statistics); if (!commit_result.ok()) { - // commit exception, not sure about the situation and should not clean up the files. - PAIMON_LOG_WARN(logger_, "You need call FilterAndCommit to retry commit for exception. %s", + // commit exception is uncertain; retry after checking whether this commit already exists. + PAIMON_LOG_WARN(logger_, "Retry commit for exception. %s", commit_result.status().ToString().c_str()); - - // To prevent the case where an atomic write times out but actually succeeds, - // retrying the commit could lead to the snapshot file being committed multiple times. - // Therefore, retries should be handled by the upper layer, - // which should call FilterAndCommit to avoid duplicate commits. - // Therefore, we should not trigger cleanup here, - // as it may delete meta files from a snapshot that was just written by ourselves, - // leading to an incomplete or corrupted snapshot. guard.Release(); - return Status::Invalid("You need call FilterAndCommit to retry commit for exception. ", - commit_result.status().ToString()); + return false; } bool commit_success = commit_result.value(); if (commit_success) { @@ -745,6 +1100,7 @@ Result FileStoreCommitImpl::TryCommitOnce( new_snapshot.Id(), root_path_.c_str(), commit_user_.c_str(), new_snapshot.CommitIdentifier(), Snapshot::CommitKind::ToString(new_snapshot.GetCommitKind()).c_str()); + last_committed_snapshot_id_ = new_snapshot.Id(); guard.Release(); return true; } else { @@ -753,70 +1109,6 @@ Result FileStoreCommitImpl::TryCommitOnce( } } -void FileStoreCommitImpl::AssignSnapshotId(int64_t snapshot_id, - std::vector* delta_files) const { - for (auto& entry : *delta_files) { - entry.AssignSequenceNumber(/*min_sequence_number=*/snapshot_id, - /*max_sequence_number=*/snapshot_id); - } -} - -Result FileStoreCommitImpl::AssignRowTrackingMeta( - int64_t first_row_id_start, std::vector* delta_files) const { - if (delta_files->empty()) { - return first_row_id_start; - } - // assign row id for new files - int64_t start = first_row_id_start; - int64_t blob_start_default = first_row_id_start; - // Per-blob-field row id tracking: each blob field maintains its own start position, - // keyed by the blob field name (from write_cols[0]). - std::map blob_starts; - // TODO(xinyu.lxy): support vector store file row tracking when vector store is implemented - for (auto& entry : *delta_files) { - if (entry.File()->file_source == std::nullopt) { - return Status::Invalid( - "This is a bug, file source field for row-tracking table must present."); - } - bool contains_row_id = - entry.File()->write_cols.has_value() && - std::find(entry.File()->write_cols->begin(), entry.File()->write_cols->end(), - SpecialFields::RowId().Name()) != entry.File()->write_cols->end(); - if (entry.File()->file_source.value() == FileSource::Append() && - entry.File()->first_row_id == std::nullopt && !contains_row_id) { - int64_t row_count = entry.File()->row_count; - if (BlobUtils::IsBlobFile(entry.File()->file_name)) { - // Use the first write_col as the blob field name to support - // independent row tracking per blob field. - std::string blob_field_name; - if (!entry.File()->write_cols || entry.File()->write_cols->empty()) { - return Status::Invalid(fmt::format( - "invalid blob file {}: does not have write_cols", entry.File()->file_name)); - } - blob_field_name = entry.File()->write_cols->at(0); - int64_t blob_start = blob_starts.count(blob_field_name) - ? blob_starts[blob_field_name] - : blob_start_default; - if (blob_start >= start) { - return Status::Invalid(fmt::format( - "This is a bug, blob start {} should be less than start {} when " - "assigning a blob entry file.", - blob_start, start)); - } - entry.AssignFirstRowId(blob_start); - blob_starts[blob_field_name] = blob_start + row_count; - } else { - entry.AssignFirstRowId(start); - blob_start_default = start; - blob_starts.clear(); - start += row_count; - } - } - // for compact file, do not assign first row id. - } - return start; -} - Result FileStoreCommitImpl::CommitSnapshotImpl( const Snapshot& new_snapshot, const std::vector& delta_statistics) { std::vector statistics; @@ -885,85 +1177,24 @@ std::shared_ptr FileStoreCommitImpl::CreateManifestCommitta return committable; } -Status FileStoreCommitImpl::CollectChanges( - const std::vector>& commit_messages, - std::vector* append_table_files, - std::vector* append_changelog_files, - std::vector* compact_table_files, - std::vector* compact_changelog_files, - std::vector* append_table_index_files, - std::vector* compact_table_index_files) { +Result FileStoreCommitImpl::CollectChanges( + const std::vector>& commit_messages) { + ManifestEntryChanges changes(num_bucket_); for (const auto& message : commit_messages) { - auto commit_message = std::dynamic_pointer_cast(message); - if (commit_message) { - DataIncrement new_files_increment = commit_message->GetNewFilesIncrement(); - for (const std::shared_ptr& new_file : new_files_increment.NewFiles()) { - append_table_files->push_back(MakeEntry(FileKind::Add(), commit_message, new_file)); - } - for (const std::shared_ptr& deleted_file : - new_files_increment.DeletedFiles()) { - append_table_files->push_back( - MakeEntry(FileKind::Delete(), commit_message, deleted_file)); - } - for (const std::shared_ptr& changelog_file : - new_files_increment.ChangelogFiles()) { - append_changelog_files->push_back( - MakeEntry(FileKind::Add(), commit_message, changelog_file)); - } - for (const std::shared_ptr& deleted_index_file : - new_files_increment.DeletedIndexFiles()) { - append_table_index_files->emplace_back( - FileKind::Delete(), commit_message->Partition(), commit_message->Bucket(), - deleted_index_file); - } - for (const std::shared_ptr& new_index_file : - new_files_increment.NewIndexFiles()) { - append_table_index_files->emplace_back(FileKind::Add(), commit_message->Partition(), - commit_message->Bucket(), new_index_file); - } - CompactIncrement compact_increment = commit_message->GetCompactIncrement(); - for (const std::shared_ptr& compact_before : - compact_increment.CompactBefore()) { - compact_table_files->push_back( - MakeEntry(FileKind::Delete(), commit_message, compact_before)); - } - for (const std::shared_ptr& compact_after : - compact_increment.CompactAfter()) { - compact_table_files->push_back( - MakeEntry(FileKind::Add(), commit_message, compact_after)); - } - for (const std::shared_ptr& changelog_file : - compact_increment.ChangelogFiles()) { - compact_changelog_files->push_back( - MakeEntry(FileKind::Add(), commit_message, changelog_file)); - } - for (const std::shared_ptr& deleted_index_file : - compact_increment.DeletedIndexFiles()) { - compact_table_index_files->emplace_back( - FileKind::Delete(), commit_message->Partition(), commit_message->Bucket(), - deleted_index_file); - } - for (const std::shared_ptr& new_index_file : - compact_increment.NewIndexFiles()) { - compact_table_index_files->emplace_back(FileKind::Add(), - commit_message->Partition(), - commit_message->Bucket(), new_index_file); - } - } else { - return Status::Invalid("fail to cast commit message to commit message impl"); - } + PAIMON_RETURN_NOT_OK(changes.Collect(message)); } - return Status::OK(); + PAIMON_LOG_INFO(logger_, "Finished collecting changes, including: %s", + changes.ToString().c_str()); + return changes; } -ManifestEntry FileStoreCommitImpl::MakeEntry( - const FileKind& kind, const std::shared_ptr& commit_message, - const std::shared_ptr& file) const { - int32_t total_buckets = commit_message->TotalBuckets() == std::nullopt - ? num_bucket_ - : commit_message->TotalBuckets().value(); - return ManifestEntry(kind, commit_message->Partition(), commit_message->Bucket(), total_buckets, - file); +void FileStoreCommitImpl::ReportCommit(const ManifestEntryChanges& changes, int64_t commit_duration, + int32_t generated_snapshot, int32_t attempt) { + CommitStats commit_stats(changes.append_table_files, changes.append_changelog, + changes.compact_table_files, changes.compact_changelog, + commit_duration, generated_snapshot, attempt, + last_committed_snapshot_id_); + CommitMetrics::ReportCommit(metrics_, commit_stats); } int64_t FileStoreCommitImpl::RowCounts(const std::vector& files) { @@ -973,30 +1204,4 @@ int64_t FileStoreCommitImpl::RowCounts(const std::vector& files) }); } -int64_t FileStoreCommitImpl::NumChangedPartitions( - const std::vector>& changes) { - std::unordered_set changed_partitions; - for (const auto& change : changes) { - for (const auto& entry : change) { - changed_partitions.insert(entry.Partition()); - } - } - return static_cast(changed_partitions.size()); -} - -int64_t FileStoreCommitImpl::NumChangedBuckets( - const std::vector>& changes) { - std::unordered_map> changed_partition_buckets; - for (const auto& change : changes) { - for (const auto& entry : change) { - changed_partition_buckets[entry.Partition()].insert(entry.Bucket()); - } - } - return std::accumulate(changed_partition_buckets.begin(), changed_partition_buckets.end(), - int64_t{0}, [](int64_t num_changed_buckets, const auto& bucket) { - return num_changed_buckets + - static_cast(bucket.second.size()); - }); -} - } // namespace paimon diff --git a/src/paimon/core/operation/file_store_commit_impl.h b/src/paimon/core/operation/file_store_commit_impl.h index 217edc06..2ef62bbf 100644 --- a/src/paimon/core/operation/file_store_commit_impl.h +++ b/src/paimon/core/operation/file_store_commit_impl.h @@ -31,7 +31,13 @@ #include "paimon/core/catalog/snapshot_commit.h" #include "paimon/core/core_options.h" #include "paimon/core/manifest/partition_entry.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/commit/conflict_detection.h" +#include "paimon/core/operation/commit/manifest_entry_changes.h" +#include "paimon/core/operation/commit/retry_waiter.h" +#include "paimon/core/operation/commit/row_id_column_conflict_checker.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" #include "paimon/file_store_commit.h" #include "paimon/logging.h" #include "paimon/memory/memory_pool.h" @@ -63,6 +69,7 @@ class SnapshotManager; class SchemaManager; class TableSchema; class BinaryRowPartitionComputer; +class CommitChangesProvider; class CommitMessage; class Executor; class FileSystem; @@ -75,6 +82,8 @@ class SnapshotCommit; /// Commit operation which provides commit and overwrite. class FileStoreCommitImpl : public FileStoreCommit { public: + static Status ValidateCommitOptions(const CoreOptions& options); + FileStoreCommitImpl(const std::shared_ptr& pool, const std::shared_ptr& executor, const std::shared_ptr& schema, const std::string& root_path, @@ -83,12 +92,14 @@ class FileStoreCommitImpl : public FileStoreCommit { std::unique_ptr partition_computer, const std::shared_ptr& snapshot_manager, bool ignore_empty_commit, bool use_rest_catalog_commit, + bool append_commit_check_conflict, const std::shared_ptr& table_schema, const std::shared_ptr& manifest_file, const std::shared_ptr& manifest_list, const std::shared_ptr& index_manifest_file, const std::shared_ptr& expire_snapshots, - const std::shared_ptr& schema_manager); + const std::shared_ptr& schema_manager, + CommitScanner::ScanSupplier scan_supplier); ~FileStoreCommitImpl() override; Status Commit(const std::vector>& commit_messages, @@ -100,13 +111,13 @@ class FileStoreCommitImpl : public FileStoreCommit { commit_identifier_and_messages, std::optional watermark = std::nullopt) override; - Status Overwrite(const std::vector>& partitions, + Status Overwrite(const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) override; Result FilterAndOverwrite( - const std::vector>& partitions, + const std::map& partition, const std::vector>& commit_messages, int64_t commit_identifier, std::optional watermark = std::nullopt) override; @@ -117,6 +128,8 @@ class FileStoreCommitImpl : public FileStoreCommit { Status DropPartition(const std::vector>& partitions, int64_t commit_identifier) override; + FileStoreCommit& RowIdCheckConflict(std::optional row_id_check_from_snapshot) override; + std::shared_ptr GetCommitMetrics() const override { return metrics_; } @@ -127,13 +140,26 @@ class FileStoreCommitImpl : public FileStoreCommit { Status Commit(const std::shared_ptr& manifest_committable, bool check_append_files); - Status TryOverwrite(const std::vector>& partition, - const std::vector& changes, int64_t commit_identifier, - std::optional watermark); + Result TryOverwrite(const std::vector>& partition, + const std::vector& changes, + const std::vector& index_entries, + int64_t commit_identifier, std::optional watermark, + const std::map& properties); + + Status ExecuteOverwrite(const std::vector>& partitions, + ManifestEntryChanges* changes, int64_t identifier, + std::optional watermark, + const std::shared_ptr& committable, + int32_t* generated_snapshot, int32_t* attempt); Result> GetAllFiles( const Snapshot& snapshot, - const std::vector>& partitions); + const std::vector>& partitions) const; + + Result> PartitionToMap(const BinaryRow& partition) const; + + Result> TryUpgrade( + const std::vector& append_files) const; Result>> FilterCommitted( const std::vector>& committables); @@ -142,32 +168,33 @@ class FileStoreCommitImpl : public FileStoreCommit { int64_t identifier, const std::vector>& commit_messages, std::optional watermark); - ManifestEntry MakeEntry(const FileKind& kind, - const std::shared_ptr& commit_message, - const std::shared_ptr& file) const; + Result CollectChanges( + const std::vector>& commit_messages); - Status CollectChanges(const std::vector>& commit_messages, - std::vector* append_table_files, - std::vector* append_changelog_files, - std::vector* compact_table_files, - std::vector* compact_changelog_files, - std::vector* append_table_index_files, - std::vector* compact_table_index_files); + void ReportCommit(const ManifestEntryChanges& changes, int64_t commit_duration, + int32_t generated_snapshot, int32_t attempt); Result TryCommit(const std::vector& delta_files, + const std::vector& changelog_files, const std::vector& index_entries, int64_t identifier, std::optional watermark, - std::map log_offsets, const std::map& properties, - Snapshot::CommitKind commit_kind, bool check_append_files); + Snapshot::CommitKind commit_kind, bool detect_conflicts); + + Result TryCommit(const std::shared_ptr& changes_provider, + int64_t identifier, std::optional watermark, + const std::map& properties, + Snapshot::CommitKind commit_kind, bool detect_conflicts); + Result TryCommitOnce(const std::vector& delta_files, + const std::vector& changelog_files, const std::vector& index_entries, int64_t commit_identifier, std::optional watermark, - std::map log_offsets, const std::map& properties, Snapshot::CommitKind commit_kind, const std::optional& latest_snapshot, - bool need_conflict_check); + bool detect_conflicts, + std::optional retry_start_snapshot_id); Result CommitSnapshotImpl(const Snapshot& new_snapshot, const std::vector& delta_statistics); @@ -179,37 +206,33 @@ class FileStoreCommitImpl : public FileStoreCommit { const std::optional& old_index_manifest, const std::optional& new_index_manifest); - Result> ReadAllEntriesFromChangedPartitions( - const Snapshot& latest_snapshot, - const std::set>& partitions) const; - - Status NoConflictsOrFail(const std::string& base_commit_user, - const std::vector& base_entries, - const std::vector& changes) const; + Result CheckCommitted(const std::optional& latest_snapshot, + std::optional retry_start_snapshot_id, int64_t identifier, + const Snapshot::CommitKind& commit_kind) const; - Status CheckFilesExistence( - const std::vector>& committables) const; + Status CheckSameBucketFromSnapshot(const std::vector& delta_entries, + const std::optional& latest_snapshot) const; - void AssignSnapshotId(int64_t snapshot_id, std::vector* delta_files) const; + bool ShouldCheckSameBucket(const Snapshot::CommitKind& commit_kind) const; - Result AssignRowTrackingMeta(int64_t first_row_id_start, - std::vector* delta_files) const; + bool IsUnorderedWriteOnlyAppend() const; - Result>> ChangedPartitions( - const std::vector& data_files, - const std::vector& index_files) const; + bool IsWriteOnlySnapshotSequenceAppend() const; - static int64_t RowCounts(const std::vector& files); + Result> MaxSequenceNumber( + const std::vector& manifests) const; - static int64_t NumChangedPartitions(const std::vector>& changes); + Status CheckFilesExistence( + const std::vector>& committables) const; - static int64_t NumChangedBuckets(const std::vector>& changes); + static int64_t RowCounts(const std::vector& files); private: std::shared_ptr memory_pool_; std::shared_ptr executor_; std::shared_ptr schema_; std::string root_path_; + std::string table_name_; std::string commit_user_; CoreOptions options_; std::shared_ptr path_factory_; @@ -219,8 +242,13 @@ class FileStoreCommitImpl : public FileStoreCommit { std::shared_ptr snapshot_manager_; std::shared_ptr snapshot_commit_; bool ignore_empty_commit_ = true; + bool append_commit_check_conflict_ = false; + RetryWaiter retry_waiter_; int32_t num_bucket_ = 0; + BucketMode bucket_mode_ = BucketMode::BUCKET_UNAWARE; std::shared_ptr table_schema_; + std::shared_ptr commit_scanner_; + ConflictDetection conflict_detection_; std::shared_ptr manifest_file_; std::shared_ptr manifest_list_; @@ -231,6 +259,7 @@ class FileStoreCommitImpl : public FileStoreCommit { std::shared_ptr metrics_; std::shared_ptr logger_; + int64_t last_committed_snapshot_id_ = -1; }; } // namespace paimon diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index 63e58062..f16da448 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -42,15 +42,19 @@ #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/catalog/commit_table_request.h" +#include "paimon/core/index/global_index_meta.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/file_source.h" #include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/index_manifest_file.h" #include "paimon/core/manifest/manifest_committable.h" #include "paimon/core/manifest/manifest_entry.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/manifest/manifest_list.h" #include "paimon/core/operation/metrics/commit_metrics.h" #include "paimon/core/partition/partition_statistics.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/table/sink/commit_message_impl.h" #include "paimon/core/utils/file_utils.h" @@ -61,6 +65,7 @@ #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/fs/local/local_file_system_factory.h" +#include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/metrics.h" #include "paimon/testing/utils/binary_row_generator.h" @@ -69,6 +74,7 @@ #include "paimon/testing/utils/timezone_guard.h" namespace paimon::test { + class GmockFileSystem : public LocalFileSystem { public: MOCK_METHOD(Status, ReadFile, (const std::string& path, std::string* content), (override)); @@ -160,11 +166,18 @@ class FileStoreCommitImplTest : public testing::Test { ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition, const FileKind& kind) const { + return CreateManifestEntry(file_name, partition, kind, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), /*level=*/2, /*bucket=*/0); + } + + ManifestEntry CreateManifestEntry(const std::string& file_name, const BinaryRow& partition, + const FileKind& kind, const BinaryRow& min_key, + const BinaryRow& max_key, int32_t level, int32_t bucket = 0, + int32_t total_buckets = 2) const { auto data_file_meta = std::make_shared( - file_name, 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), - SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, - /*max_seq_no=*/32, - /*schema_id=*/1, /*level=*/2, + file_name, 1024, 8, min_key, max_key, SimpleStats::EmptyStats(), + SimpleStats::EmptyStats(), /*min_seq_no=*/16, /*max_seq_no=*/32, + /*schema_id=*/1, level, /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/3, @@ -172,7 +185,15 @@ class FileStoreCommitImplTest : public testing::Test { /*external_path=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); - return ManifestEntry(kind, partition, 0, 2, data_file_meta); + return ManifestEntry(kind, partition, bucket, total_buckets, data_file_meta); + } + + BinaryRow CreateIntRow(int32_t value) const { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; } ManifestEntry CreateManifestEntryWithNoPartition(const std::string& file_name, @@ -200,6 +221,41 @@ class FileStoreCommitImplTest : public testing::Test { return result; } + std::shared_ptr CreateIndexFileMeta(const std::string& file_name, + const std::string& index_type = "bitmap") { + return std::make_shared(index_type, file_name, /*file_size=*/100, + /*row_count=*/5, /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, + /*global_index_meta=*/std::nullopt); + } + + std::shared_ptr CreateGlobalIndexFileMeta(const std::string& file_name, + int64_t row_range_start, + int64_t row_range_end) { + GlobalIndexMeta global_index(row_range_start, row_range_end, /*index_field_id=*/1, + /*extra_field_ids=*/std::nullopt, + std::make_shared("meta", GetDefaultPool().get())); + return std::make_shared( + "HASH", file_name, /*file_size=*/100, /*row_count=*/5, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index); + } + + std::shared_ptr CreateAppendDataFileMeta(const std::string& file_name, + int64_t row_count) { + return std::make_shared( + file_name, 1024, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, + /*max_seq_no=*/32, + /*schema_id=*/1, /*level=*/2, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, FileSource::Append(), + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + } + bool IsStringInSet(const std::set& strSet, const std::string& target) { return strSet.find(target) != strSet.end(); } @@ -340,7 +396,7 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt, /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807, /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597, - /*log_offsets=*/std::map(), /*total_record_count=*/5, + /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0); std::vector expected_partition_statistics = { @@ -366,6 +422,48 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { ASSERT_FALSE(exist); } +TEST_F(FileStoreCommitImplTest, TestSnapshotSequenceMaxPropertyMergedOnCommit) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + std::vector> msgs1 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + ASSERT_GT(msgs1.size(), 0); + ASSERT_OK(commit_impl->Commit(msgs1, 1)); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot1, commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_TRUE(snapshot1.Properties()); + auto iter1 = snapshot1.Properties().value().find("sequence.generation.max-sequence-number"); + ASSERT_TRUE(iter1 != snapshot1.Properties().value().end()); + int64_t max_seq_1 = std::stoll(iter1->second); + + std::vector> msgs2 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-02", + /*version=*/3); + ASSERT_GT(msgs2.size(), 0); + ASSERT_OK(commit_impl->Commit(msgs2, 2)); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot2, commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_TRUE(snapshot2.Properties()); + auto iter2 = snapshot2.Properties().value().find("sequence.generation.max-sequence-number"); + ASSERT_TRUE(iter2 != snapshot2.Properties().value().end()); + int64_t max_seq_2 = std::stoll(iter2->second); + + ASSERT_GE(max_seq_2, max_seq_1); +} + TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) { std::string test_data_path = paimon::test::GetDataDir() + "/orc/append_09.db/append_09/"; auto dir = UniqueTestDirectory::Create(); @@ -378,6 +476,8 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryTenTimes) context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") .AddOption(Options::COMMIT_MAX_RETRIES, "10") + .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") + .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") .WithFileSystem(fs) .Finish()); @@ -418,25 +518,25 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithConflictSnapshotAndRetryOnce) { ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::COMMIT_MIN_RETRY_WAIT, "1ms") + .AddOption(Options::COMMIT_MAX_RETRY_WAIT, "1ms") .WithFileSystem(fs) .Finish()); ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); std::string latest_hint = PathUtil::JoinPath(table_path, "snapshot/LATEST"); auto* mock_fs = dynamic_cast(fs.get()); + EXPECT_CALL(*mock_fs, ReadFile(testing::_, testing::_)) + .Times(testing::AnyNumber()) + .WillRepeatedly(testing::Invoke([&](const std::string& path, std::string* content) { + return mock_fs->FileSystem::ReadFile(path, content); + })); EXPECT_CALL(*mock_fs, ReadFile(testing::StrEq(latest_hint), testing::_)) .WillRepeatedly(testing::Invoke([](const std::string& path, std::string* content) { *content = "-1"; return Status::OK(); })); - EXPECT_CALL( - *mock_fs, - ReadFile(testing::StrEq(PathUtil::JoinPath(table_path, "snapshot/snapshot-5")), testing::_)) - .WillOnce(testing::Invoke([&](const std::string& path, std::string* content) { - return mock_fs->FileSystem::ReadFile(path, content); - })); - EXPECT_CALL(*mock_fs, ListDir(testing::_, testing::_)).Times(testing::AnyNumber()); EXPECT_CALL(*mock_fs, ListDir(testing::StrEq(PathUtil::JoinPath(table_path, "snapshot")), testing::_)) @@ -497,7 +597,12 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua "/orc/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); ASSERT_GT(msgs.size(), 0); - ASSERT_NOK(commit->Commit(msgs, /*commit_identifier=*/1)); + ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/1)); + std::shared_ptr metrics = commit->GetCommitMetrics(); + ASSERT_TRUE(metrics); + ASSERT_OK_AND_ASSIGN(uint64_t counter, + metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS)); + ASSERT_EQ(2u, counter); ASSERT_OK_AND_ASSIGN( bool exist, file_system_->Exists(PathUtil::JoinPath(table_path, "snapshot/snapshot-6"))); ASSERT_TRUE(exist); @@ -510,8 +615,6 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua .Finish()); ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(std::move(commit_context_2))); - ASSERT_OK_AND_ASSIGN(int32_t num_committed, commit_2->FilterAndCommit({{1, msgs}})); - ASSERT_EQ(0, num_committed); std::string new_snapshot_7 = PathUtil::JoinPath(table_path, "snapshot/snapshot-7"); EXPECT_CALL(*mock_fs, AtomicStore(testing::StrEq(new_snapshot_7), testing::_)) .WillOnce(testing::Invoke([&](const std::string& path, const std::string& content) { @@ -583,7 +686,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithSameMsgs) { ASSERT_TRUE(metrics); ASSERT_OK_AND_ASSIGN(uint64_t counter, metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS)); - ASSERT_EQ(1u, counter); + ASSERT_EQ(0u, counter); ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(PathUtil::JoinPath( table_path_, "snapshot/snapshot-3"))); ASSERT_FALSE(exist); @@ -740,18 +843,21 @@ TEST_F(FileStoreCommitImplTest, TestCommitSuccessAfterIOException) { io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); auto status = commit->Commit(msgs); io_hook->Clear(); + ASSERT_OK_AND_ASSIGN(bool exist2, file_system_->Exists(PathUtil::JoinPath( + table_path_, "snapshot/snapshot-2"))); // termination conditions: - // 1. status does not hit IOHook, already touch all IO operation - // 2. hit IOHook in commit hint, at this point, the snapshot is already committed - if (!HitIOHook(status) || HitIOHookInCommitHint(status)) { + // 1. hit IOHook in commit hint, at this point, the snapshot is already committed + // 2. snapshot file exists, which means atomic store succeeded even if a later IO failed + if (HitIOHookInCommitHint(status) || exist2) { scanned_all_io_hook = true; - ASSERT_OK_AND_ASSIGN(bool exist2, file_system_->Exists(PathUtil::JoinPath( - table_path_, "snapshot/snapshot-2"))); ASSERT_TRUE(exist2); break; } - ASSERT_OK_AND_ASSIGN(bool exist2, file_system_->Exists(PathUtil::JoinPath( - table_path_, "snapshot/snapshot-2"))); + // For some IO-hook positions, retries may be exhausted and return a generic non-IOHook + // status while the snapshot is still not committed. Keep scanning next positions. + if (!HitIOHook(status)) { + continue; + } ASSERT_FALSE(exist2); std::vector actual_snapshots; ASSERT_OK( @@ -870,7 +976,6 @@ TEST_F(FileStoreCommitImplTest, TestCleanUpTmpManifests) { ASSERT_OK_AND_ASSIGN(exist, file_system_->Exists(PathUtil::JoinPath( table_path_, "manifest/" + index_manifest.value()))); ASSERT_FALSE(exist); - commit_impl->CleanUpTmpManifests( snapshot.value().BaseManifestList(), snapshot.value().DeltaManifestList(), /*old_metas=*/{}, /*new_metas=*/previous_manifests, /*old_index_manifest=*/std::nullopt, @@ -901,133 +1006,6 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithIgnoreEmptyCommit) { ASSERT_EQ(0u, counter); } -TEST_F(FileStoreCommitImplTest, TestCheckConflict) { - CommitContextBuilder context_builder(table_path_, "commit_user_1"); - ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, - context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") - .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") - .AddOption(Options::FILE_SYSTEM, "local") - .IgnoreEmptyCommit(true) - .Finish()); - - ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); - auto commit_impl = dynamic_cast(commit.get()); - ASSERT_TRUE(commit_impl); - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file2", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file3", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file4", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file5", FileKind::Delete())); - ASSERT_OK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file6", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file2", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file3", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file4", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file5", FileKind::Delete())); - ASSERT_OK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file2", FileKind::Add())); - changes.push_back(CreateManifestEntry("file3", FileKind::Add())); - changes.push_back(CreateManifestEntry("file4", FileKind::Add())); - changes.push_back(CreateManifestEntry("file5", FileKind::Add())); - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Add())); - changes.push_back(CreateManifestEntry("file2", FileKind::Add())); - changes.push_back(CreateManifestEntry("file3", FileKind::Add())); - changes.push_back(CreateManifestEntry("file4", FileKind::Add())); - changes.push_back(CreateManifestEntry("file5", FileKind::Add())); - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Delete())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Add())); - changes.push_back(CreateManifestEntry("file2", FileKind::Add())); - changes.push_back(CreateManifestEntry("file3", FileKind::Add())); - changes.push_back(CreateManifestEntry("file4", FileKind::Add())); - changes.push_back(CreateManifestEntry("file5", FileKind::Add())); - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete())); - - std::vector changes; - ASSERT_OK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Delete())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Delete())); - - std::vector changes; - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } - { - std::vector base_entries; - base_entries.push_back(CreateManifestEntry("file1", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file2", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file3", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file4", FileKind::Add())); - base_entries.push_back(CreateManifestEntry("file5", FileKind::Add())); - - std::vector changes; - changes.push_back(CreateManifestEntry("file1", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file2", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file3", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file4", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file5", FileKind::Delete())); - changes.push_back(CreateManifestEntry("file6", FileKind::Delete())); - ASSERT_NOK(commit_impl->NoConflictsOrFail("commit_user_1", base_entries, changes)); - } -} - TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1054,8 +1032,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwrite) { std::vector changes; changes.push_back(CreateManifestEntry("new_file_1", FileKind::Add())); std::vector> partitions = {{{"f1", "10"}}, {{"f1", "20"}}}; - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, - /*commit_identifier=*/1, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{}, + /*commit_identifier=*/1, std::nullopt, + /*properties=*/{})); } TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { @@ -1073,8 +1052,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { std::vector changes; changes.push_back(CreateManifestEntry("new_file_1", FileKind::Add())); std::vector> partitions = {{{"f1", "10"}}, {{"f1", "20"}}}; - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, - /*commit_identifier=*/0, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{}, + /*commit_identifier=*/0, std::nullopt, + /*properties=*/{})); ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot()); ASSERT_OK_AND_ASSIGN(auto entries1, commit_impl->GetAllFiles(snapshot1.value(), {})); ASSERT_EQ(1u, entries1.size()); @@ -1082,8 +1062,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { ASSERT_EQ(FileKind::Add(), entries1[0].Kind()); std::vector changes2; changes2.push_back(CreateManifestEntry("new_file_2", FileKind::Add())); - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, - /*commit_identifier=*/1, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, /*index_entries=*/{}, + /*commit_identifier=*/1, std::nullopt, + /*properties=*/{})); ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot()); ASSERT_OK_AND_ASSIGN(auto entries2, commit_impl->GetAllFiles(snapshot2.value(), {})); ASSERT_EQ(1u, entries2.size()); @@ -1091,6 +1072,34 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteFromNothing) { ASSERT_EQ(FileKind::Add(), entries2[0].Kind()); } +TEST_F(FileStoreCommitImplTest, TestTryOverwriteWithProperties) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = dynamic_cast(commit.get()); + ASSERT_TRUE(commit_impl); + + std::vector changes; + changes.push_back(CreateManifestEntry("new_file_with_properties", FileKind::Add())); + std::vector> partitions = {{{"f1", "10"}}}; + std::map properties = {{"overwrite-prop", "v1"}}; + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{}, + /*commit_identifier=*/0, std::nullopt, properties)); + + ASSERT_OK_AND_ASSIGN(auto snapshot, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot.has_value()); + ASSERT_TRUE(snapshot->Properties().has_value()); + auto iter = snapshot->Properties()->find("overwrite-prop"); + ASSERT_TRUE(iter != snapshot->Properties()->end()); + ASSERT_EQ("v1", iter->second); +} + TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1106,8 +1115,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { std::vector changes; changes.push_back(CreateManifestEntry("new_file_1", FileKind::Add())); std::vector> partitions = {{{"f1", "10"}}, {{"f1", "20"}}}; - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, - /*commit_identifier=*/0, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes, /*index_entries=*/{}, + /*commit_identifier=*/0, std::nullopt, + /*properties=*/{})); std::vector> msgs = GetCommitMessages(paimon::test::GetDataDir() + "/orc/append_09.db/append_09/commit_messages/" @@ -1132,8 +1142,9 @@ TEST_F(FileStoreCommitImplTest, TestTryOverwriteThenCommit) { std::vector changes2; changes2.push_back(CreateManifestEntry("new_file_2", FileKind::Add())); - ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, - /*commit_identifier=*/2, std::nullopt)); + ASSERT_OK(commit_impl->TryOverwrite(partitions, changes2, /*index_entries=*/{}, + /*commit_identifier=*/2, std::nullopt, + /*properties=*/{})); ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot()); ASSERT_OK_AND_ASSIGN(auto entries2, commit_impl->GetAllFiles(snapshot2.value(), {})); ASSERT_EQ(1u, entries2.size()); @@ -1274,36 +1285,31 @@ TEST_F(FileStoreCommitImplTest, TestCollectChanges) { /*version=*/3); auto commit_impl = std::dynamic_pointer_cast( std::shared_ptr(std::move(commit))); - std::vector append_table_files; - std::vector append_changelog_files; - std::vector compact_table_files; - std::vector compact_changelog_files; - std::vector append_table_index_files; - std::vector compact_table_index_files; - ASSERT_OK(commit_impl->CollectChanges(msgs, &append_table_files, &append_changelog_files, - &compact_table_files, &compact_changelog_files, - &append_table_index_files, &compact_table_index_files)); - ASSERT_EQ(append_table_files.size(), 3u); - ASSERT_EQ(append_changelog_files.size(), 0u); - ASSERT_EQ(compact_table_files.size(), 0u); - ASSERT_EQ(compact_changelog_files.size(), 0u); - ASSERT_EQ(append_table_index_files.size(), 0u); - ASSERT_EQ(compact_table_index_files.size(), 0u); - ASSERT_EQ(append_table_files[0].Kind(), FileKind::Add()); - ASSERT_EQ(append_table_files[0].Bucket(), 0); - ASSERT_EQ(append_table_files[0].TotalBuckets(), 10); - ASSERT_EQ(append_table_files[0].Level(), 0); - ASSERT_EQ(append_table_files[0].FileName(), "data-51a45441-6037-4af3-b67b-5cefd75dc6f2-0.orc"); - ASSERT_EQ(append_table_files[1].Kind(), FileKind::Add()); - ASSERT_EQ(append_table_files[1].Bucket(), 1); - ASSERT_EQ(append_table_files[1].TotalBuckets(), 10); - ASSERT_EQ(append_table_files[1].Level(), 0); - ASSERT_EQ(append_table_files[1].FileName(), "data-6828284c-e707-49b5-af6b-69be79af120c-0.orc"); - ASSERT_EQ(append_table_files[2].Kind(), FileKind::Add()); - ASSERT_EQ(append_table_files[2].Bucket(), 0); - ASSERT_EQ(append_table_files[2].TotalBuckets(), 10); - ASSERT_EQ(append_table_files[2].Level(), 0); - ASSERT_EQ(append_table_files[2].FileName(), "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc"); + ASSERT_OK_AND_ASSIGN(ManifestEntryChanges changes, commit_impl->CollectChanges(msgs)); + ASSERT_EQ(changes.append_table_files.size(), 3u); + ASSERT_EQ(changes.append_changelog.size(), 0u); + ASSERT_EQ(changes.compact_table_files.size(), 0u); + ASSERT_EQ(changes.compact_changelog.size(), 0u); + ASSERT_EQ(changes.append_index_files.size(), 0u); + ASSERT_EQ(changes.compact_index_files.size(), 0u); + ASSERT_EQ(changes.append_table_files[0].Kind(), FileKind::Add()); + ASSERT_EQ(changes.append_table_files[0].Bucket(), 0); + ASSERT_EQ(changes.append_table_files[0].TotalBuckets(), 10); + ASSERT_EQ(changes.append_table_files[0].Level(), 0); + ASSERT_EQ(changes.append_table_files[0].FileName(), + "data-51a45441-6037-4af3-b67b-5cefd75dc6f2-0.orc"); + ASSERT_EQ(changes.append_table_files[1].Kind(), FileKind::Add()); + ASSERT_EQ(changes.append_table_files[1].Bucket(), 1); + ASSERT_EQ(changes.append_table_files[1].TotalBuckets(), 10); + ASSERT_EQ(changes.append_table_files[1].Level(), 0); + ASSERT_EQ(changes.append_table_files[1].FileName(), + "data-6828284c-e707-49b5-af6b-69be79af120c-0.orc"); + ASSERT_EQ(changes.append_table_files[2].Kind(), FileKind::Add()); + ASSERT_EQ(changes.append_table_files[2].Bucket(), 0); + ASSERT_EQ(changes.append_table_files[2].TotalBuckets(), 10); + ASSERT_EQ(changes.append_table_files[2].Level(), 0); + ASSERT_EQ(changes.append_table_files[2].FileName(), + "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc"); } TEST_F(FileStoreCommitImplTest, TestFilterCommitted) { @@ -1375,6 +1381,37 @@ TEST_F(FileStoreCommitImplTest, TestFilterCommittedWithMultipleCommittables) { ASSERT_EQ(filtered_committables[0]->Identifier(), committable2->Identifier()); } +TEST_F(FileStoreCommitImplTest, TestFilterCommittedRejectsDuplicateIdentifiers) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + std::vector> msgs1 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/" + "commit_messages-01", + /*version=*/3); + std::vector> msgs2 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/" + "commit_messages-02", + /*version=*/3); + + auto committable1 = commit_impl->CreateManifestCommittable(1, msgs1, std::nullopt); + auto committable2 = commit_impl->CreateManifestCommittable(1, msgs2, std::nullopt); + + std::vector> committables = {committable1, committable2}; + ASSERT_NOK_WITH_MSG(commit_impl->FilterCommitted(committables), + "Committables must be sorted according to identifiers before filtering"); +} + TEST_F(FileStoreCommitImplTest, FilterAndCommit) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1446,6 +1483,75 @@ TEST_F(FileStoreCommitImplTest, FilterAndCommitWithNotExistFile) { ASSERT_NOK(commit_impl->FilterAndCommit(inputs1)); } +TEST_F(FileStoreCommitImplTest, FilterAndCommitWithCompactedChangelogFakePath) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + const BinaryRow partition = CreateIntRow(10); + const std::string base_name = "compacted-changelog-8e049c65-5ce4-4ce7-b1b0-78ce694ab351"; + const std::string fake_name = base_name + "$0-39253-39253-35699.cc-parquet"; + const std::string real_name = base_name + "$0-39253.cc-parquet"; + + ASSERT_OK(PrepareFakeFiles({"/f1=10/bucket-0/" + real_name})); + + auto fake_changelog_file = CreateAppendDataFileMeta(fake_name, /*row_count=*/1); + DataIncrement data_increment( + /*new_files=*/{}, /*deleted_files=*/{}, + /*changelog_files=*/{fake_changelog_file}, /*new_index_files=*/{}, + /*deleted_index_files=*/{}); + CompactIncrement compact_increment(/*compact_before=*/{}, /*compact_after=*/{}, + /*changelog_files=*/{}); + std::shared_ptr message = std::make_shared( + partition, /*bucket=*/1, /*total_buckets=*/2, data_increment, compact_increment); + + std::map>> inputs; + inputs[1] = {message}; + ASSERT_OK_AND_ASSIGN(int32_t actual_committed, commit_impl->FilterAndCommit(inputs)); + ASSERT_EQ(1u, actual_committed); +} + +TEST_F(FileStoreCommitImplTest, FilterAndCommitSkipCompactBeforeFileCheck) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + const BinaryRow partition = CreateIntRow(10); + const std::string compact_before_name = "missing-compact-before.orc"; + const std::string compact_after_name = "existing-compact-after.orc"; + ASSERT_OK(PrepareFakeFiles({"/f1=10/bucket-0/" + compact_after_name})); + + auto compact_before_file = CreateAppendDataFileMeta(compact_before_name, /*row_count=*/1); + auto compact_after_file = CreateAppendDataFileMeta(compact_after_name, /*row_count=*/1); + DataIncrement data_increment( + /*new_files=*/{}, /*deleted_files=*/{}, /*changelog_files=*/{}, /*new_index_files=*/{}, + /*deleted_index_files=*/{}); + CompactIncrement compact_increment(/*compact_before=*/{compact_before_file}, + /*compact_after=*/{compact_after_file}, + /*changelog_files=*/{}); + std::shared_ptr message = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment); + + std::map>> inputs; + inputs[1] = {message}; + ASSERT_OK_AND_ASSIGN(int32_t actual_committed, commit_impl->FilterAndCommit(inputs)); + ASSERT_EQ(1u, actual_committed); +} + TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1486,6 +1592,215 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteNonSpecifyPartition) { ASSERT_TRUE(IsStringInSet(file_names, "data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc")); } +TEST_F(FileStoreCommitImplTest, TestCommitWithIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> new_index_files; + new_index_files.push_back(CreateIndexFileMeta("bitmap-index-commit-1")); + DataIncrement data_increment({}, {}, {}, std::move(new_index_files), {}); + std::shared_ptr msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, CompactIncrement({}, {}, {})); + + ASSERT_OK(commit_impl->Commit({msg}, 1)); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_EQ(Snapshot::CommitKind::Append(), snapshot.GetCommitKind()); + ASSERT_TRUE(snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(1u, index_entries.size()); + ASSERT_EQ("bitmap-index-commit-1", index_entries[0].index_file->FileName()); +} + +TEST_F(FileStoreCommitImplTest, TestCommitWithGlobalIndexFilesChecksConflicts) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::ROW_TRACKING_ENABLED, "true") + .AddOption(Options::DATA_EVOLUTION_ENABLED, "true") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + DataIncrement data_increment_1({CreateAppendDataFileMeta("data-with-row-id", 10)}, {}, {}); + std::shared_ptr msg_1 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_1, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Commit({msg_1}, 1)); + + std::vector> global_index_files; + global_index_files.push_back(CreateGlobalIndexFileMeta("global-index-out-of-range", + /*row_range_start=*/0, + /*row_range_end=*/10)); + DataIncrement data_increment_2({}, {}, {}, std::move(global_index_files), {}); + std::shared_ptr msg_2 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_2, CompactIncrement({}, {}, {})); + + ASSERT_NOK_WITH_MSG(commit_impl->Commit({msg_2}, 2), "Global index row ID existence conflict"); +} + +TEST_F(FileStoreCommitImplTest, TestCommitWithCompactIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> compact_index_files; + compact_index_files.push_back(CreateIndexFileMeta("bitmap-index-commit-compact-1")); + DataIncrement data_increment({}, {}, {}); + CompactIncrement compact_increment({}, {}, {}, std::move(compact_index_files), {}); + std::shared_ptr msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment); + + ASSERT_OK(commit_impl->Commit({msg}, 1)); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.GetCommitKind()); + ASSERT_TRUE(snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(1u, index_entries.size()); + ASSERT_EQ("bitmap-index-commit-compact-1", index_entries[0].index_file->FileName()); +} + +TEST_F(FileStoreCommitImplTest, TestCommitWithDeletedIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> new_index_files; + new_index_files.push_back(CreateIndexFileMeta("bitmap-index-delete-1")); + DataIncrement data_increment_1({}, {}, {}, std::move(new_index_files), {}); + std::shared_ptr msg_1 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_1, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Commit({msg_1}, 1)); + + std::vector> deleted_index_files; + deleted_index_files.push_back(CreateIndexFileMeta("bitmap-index-delete-1")); + DataIncrement data_increment_2({}, {}, {}, {}, std::move(deleted_index_files)); + std::shared_ptr msg_2 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_2, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Commit({msg_2}, 2)); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_TRUE(snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_TRUE(index_entries.empty()); +} + +TEST_F(FileStoreCommitImplTest, TestCommitWithCompactDeletedIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> new_index_files; + new_index_files.push_back(CreateIndexFileMeta("bitmap-index-compact-delete-1")); + DataIncrement data_increment_1({}, {}, {}, std::move(new_index_files), {}); + std::shared_ptr msg_1 = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/2, + data_increment_1, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Commit({msg_1}, 1)); + + std::vector> deleted_index_files; + deleted_index_files.push_back(CreateIndexFileMeta("bitmap-index-compact-delete-1")); + DataIncrement data_increment_2({}, {}, {}); + CompactIncrement compact_increment({}, {}, {}, {}, std::move(deleted_index_files)); + std::shared_ptr msg_2 = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment_2, compact_increment); + ASSERT_OK(commit_impl->Commit({msg_2}, 2)); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.GetCommitKind()); + ASSERT_TRUE(snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_TRUE(index_entries.empty()); +} + +TEST_F(FileStoreCommitImplTest, TestOverwriteWithCompactIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> compact_index_files; + compact_index_files.push_back(CreateIndexFileMeta("bitmap-index-compact-1")); + DataIncrement data_increment({}, {}, {}); + CompactIncrement compact_increment({}, {}, {}, std::move(compact_index_files), {}); + std::shared_ptr msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment); + + ASSERT_OK(commit_impl->Overwrite({}, {msg}, 1)); + + ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, + commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind()); + ASSERT_TRUE(compact_snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(compact_snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(1u, index_entries.size()); + ASSERT_EQ("bitmap-index-compact-1", index_entries[0].index_file->FileName()); +} + TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1542,6 +1857,45 @@ TEST_F(FileStoreCommitImplTest, TestFilterAndOverwrite) { ASSERT_TRUE(IsStringInSet(file_names, "data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc")); } +TEST_F(FileStoreCommitImplTest, TestFilterAndOverwriteWithCompactIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + std::vector> compact_index_files; + compact_index_files.push_back(CreateIndexFileMeta("bitmap-index-filter-compact-1")); + DataIncrement data_increment({}, {}, {}); + CompactIncrement compact_increment({}, {}, {}, std::move(compact_index_files), {}); + std::shared_ptr msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment); + + ASSERT_OK_AND_ASSIGN(int32_t actual_commit, commit_impl->FilterAndOverwrite({}, {msg}, 1, 10)); + ASSERT_EQ(1, actual_commit); + ASSERT_OK_AND_ASSIGN(actual_commit, commit_impl->FilterAndOverwrite({}, {msg}, 1, 5)); + ASSERT_EQ(0, actual_commit); + + ASSERT_OK_AND_ASSIGN(Snapshot compact_snapshot, + commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_EQ(Snapshot::CommitKind::Compact(), compact_snapshot.GetCommitKind()); + ASSERT_TRUE(compact_snapshot.Watermark()); + ASSERT_EQ(10, compact_snapshot.Watermark().value()); + ASSERT_TRUE(compact_snapshot.IndexManifest()); + + std::vector index_entries; + ASSERT_OK(commit_impl->index_manifest_file_->Read(compact_snapshot.IndexManifest().value(), + /*filter=*/nullptr, &index_entries)); + ASSERT_EQ(1u, index_entries.size()); + ASSERT_EQ("bitmap-index-filter-compact-1", index_entries[0].index_file->FileName()); +} + TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1568,14 +1922,13 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSpecifyPartition) { std::map partitions; partitions["f1"] = "10"; - ASSERT_OK(commit_impl->Overwrite({partitions}, msgs2, 2)); + ASSERT_OK(commit_impl->Overwrite(partitions, msgs2, 2)); ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot()); ASSERT_OK_AND_ASSIGN(auto entries1, commit_impl->GetAllFiles(snapshot1.value(), {})); - ASSERT_EQ(3u, entries1.size()); + ASSERT_EQ(2u, entries1.size()); std::set file_names = CollectFileNames(entries1); ASSERT_TRUE(IsStringInSet(file_names, "data-fd1d2255-43f2-4534-b4cc-08b29e662940-0.orc")); ASSERT_TRUE(IsStringInSet(file_names, "data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc")); - ASSERT_TRUE(IsStringInSet(file_names, "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc")); } TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { @@ -1604,11 +1957,42 @@ TEST_F(FileStoreCommitImplTest, TestOverwriteWithSameFile) { ASSERT_TRUE(IsStringInSet(file_names, "data-6828284c-e707-49b5-af6b-69be79af120c-0.orc")); ASSERT_TRUE(IsStringInSet(file_names, "data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc")); - // same file delete, then add, file will also be removed in result - ASSERT_OK(commit_impl->Overwrite({}, msgs1, 2)); + // Java parity: overwrite provider adds DELETE(old) + ADD(newChanges) without dedup, + // so same-file overwrite fails on duplicate add. + ASSERT_NOK_WITH_MSG(commit_impl->Overwrite({}, msgs1, 2), "Trying to add file"); +} + +TEST_F(FileStoreCommitImplTest, TestAppendDiscardDuplicateFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::COMMIT_DISCARD_DUPLICATE_FILES, "true") + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + std::vector> msgs = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + + ASSERT_OK(commit_impl->Commit(msgs, 1)); + ASSERT_OK_AND_ASSIGN(auto snapshot1, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_OK_AND_ASSIGN(auto entries1, commit_impl->GetAllFiles(snapshot1.value(), {})); + std::set file_names_before = CollectFileNames(entries1); + ASSERT_FALSE(file_names_before.empty()); + + // Committing exactly the same append files should be accepted and filtered out when + // commit.discard-duplicate-files=true. + ASSERT_OK(commit_impl->Commit(msgs, 2)); ASSERT_OK_AND_ASSIGN(auto snapshot2, commit_impl->snapshot_manager_->LatestSnapshot()); ASSERT_OK_AND_ASSIGN(auto entries2, commit_impl->GetAllFiles(snapshot2.value(), {})); - ASSERT_EQ(0u, entries2.size()); + std::set file_names_after = CollectFileNames(entries2); + ASSERT_EQ(file_names_before, file_names_after); } TEST_F(FileStoreCommitImplTest, TestCommitWithIOException) { @@ -1691,9 +2075,7 @@ TEST_F(FileStoreCommitImplTest, TestObjectStoreAllowedWithRESTCatalogCommit) { ASSERT_FALSE(json.empty()); } -// Verify that FileStoreCommit::Create succeeds for PK tables with postpone bucket mode (bucket=-2) -// without requiring the enable-pk-commit-in-inte-test workaround flag. -TEST_F(FileStoreCommitImplTest, TestPostponeBucketPKTableCommitAllowed) { +TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) { auto pk_dir = UniqueTestDirectory::Create(); ASSERT_TRUE(pk_dir); std::string pk_root = pk_dir->Str(); @@ -1704,14 +2086,13 @@ TEST_F(FileStoreCommitImplTest, TestPostponeBucketPKTableCommitAllowed) { {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())}); ::ArrowSchema arrow_schema; ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok()); - std::map table_options = {{Options::BUCKET, "-2"}}; + std::map table_options = {{Options::BUCKET, "4"}}; ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl"), &arrow_schema, /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options, /*ignore_if_exists=*/false)); std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); - // Create FileStoreCommit WITHOUT the workaround flag — should succeed for postpone bucket CommitContextBuilder builder(pk_table_path, "test_user"); builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); @@ -1719,34 +2100,4 @@ TEST_F(FileStoreCommitImplTest, TestPostponeBucketPKTableCommitAllowed) { ASSERT_TRUE(committer != nullptr); } -// Verify that FileStoreCommit::Create still rejects PK tables with fixed bucket (bucket > 0) -// when the workaround flag is not set. -TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitRejected) { - auto pk_dir = UniqueTestDirectory::Create(); - ASSERT_TRUE(pk_dir); - std::string pk_root = pk_dir->Str(); - ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(pk_root, {})); - ASSERT_OK(catalog->CreateDatabase("db", {}, false)); - - arrow::Schema pk_schema( - {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())}); - ::ArrowSchema arrow_schema; - ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok()); - std::map table_options = {{Options::BUCKET, "4"}}; - ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl_fixed"), &arrow_schema, - /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options, - /*ignore_if_exists=*/false)); - - std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl_fixed"); - - CommitContextBuilder builder(pk_table_path, "test_user"); - builder.AddOption(Options::FILE_SYSTEM, "local").UseRESTCatalogCommit(true); - ASSERT_OK_AND_ASSIGN(auto commit_context, builder.Finish()); - auto result = FileStoreCommit::Create(std::move(commit_context)); - ASSERT_FALSE(result.ok()); - ASSERT_TRUE(result.status().IsNotImplemented()); - ASSERT_TRUE(result.status().ToString().find("not support pk table commit") != - std::string::npos); -} - } // namespace paimon::test diff --git a/src/paimon/core/operation/file_store_commit_test.cpp b/src/paimon/core/operation/file_store_commit_test.cpp index 3ee13c9a..b43e3215 100644 --- a/src/paimon/core/operation/file_store_commit_test.cpp +++ b/src/paimon/core/operation/file_store_commit_test.cpp @@ -29,10 +29,21 @@ #include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/commit_context.h" +#include "paimon/common/utils/linked_hash_map.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/deletionvectors/deletion_vector.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" +#include "paimon/core/index/deletion_vector_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/io/compact_increment.h" +#include "paimon/core/io/data_increment.h" #include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/snapshot_manager.h" #include "paimon/defs.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/result.h" +#include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/testharness.h" namespace paimon::test { @@ -75,4 +86,62 @@ TEST(FileStoreCommitTest, TestCreate) { ASSERT_TRUE(commit_impl); } +TEST(FileStoreCommitTest, TestAppendDvIndexShouldUseOverwriteCommitKind) { + auto string_field = arrow::field("f0", arrow::utf8()); + auto int_field = arrow::field("f1", arrow::int32()); + auto int_field1 = arrow::field("f2", arrow::int32()); + auto double_field = arrow::field("f3", arrow::float64()); + auto schema = + arrow::schema(arrow::FieldVector({string_field, int_field, int_field1, double_field})); + + ::ArrowSchema arrow_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &arrow_schema).ok()); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + + std::map options = {{Options::FILE_FORMAT, "orc"}, + {Options::TARGET_FILE_SIZE, "1024"}, + {Options::FILE_SYSTEM, "local"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}}; + + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir->Str(), options)); + ASSERT_OK(catalog->CreateDatabase("foo", options, /*ignore_if_exists=*/false)); + ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &arrow_schema, + /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + CommitContextBuilder context_builder(table_path, "commit_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + LinkedHashMap dv_ranges; + dv_ranges.insert_or_assign("data-file-1", + DeletionVectorMeta( + /*data_file_name=*/"data-file-1", /*offset=*/0, /*length=*/10, + /*cardinality=*/1)); + std::vector> new_index_files; + new_index_files.push_back(std::make_shared( + DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-index-1", 100, 1, + /*dv_ranges=*/dv_ranges, /*external_path=*/std::nullopt)); + DataIncrement data_increment({}, {}, {}, std::move(new_index_files), {}); + std::shared_ptr msg = std::make_shared( + BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get()), /*bucket=*/0, + /*total_bucket=*/2, data_increment, CompactIncrement({}, {}, {})); + + ASSERT_OK(commit->Commit({msg}, /*commit_identifier=*/1)); + + auto fs = std::make_shared(); + SnapshotManager snapshot_manager(fs, table_path); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshot_manager.LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot.value().GetCommitKind()); +} + } // namespace paimon::test diff --git a/src/paimon/core/operation/metrics/commit_metrics.cpp b/src/paimon/core/operation/metrics/commit_metrics.cpp new file mode 100644 index 00000000..76880932 --- /dev/null +++ b/src/paimon/core/operation/metrics/commit_metrics.cpp @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/metrics/commit_metrics.h" + +#include "paimon/core/operation/metrics/commit_stats.h" +#include "paimon/metrics.h" + +namespace paimon { + +void CommitMetrics::ReportCommit(const std::shared_ptr& metrics, + const CommitStats& commit_stats) { + metrics->SetCounter(CommitMetrics::LAST_COMMIT_DURATION, commit_stats.GetDuration()); + metrics->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, commit_stats.GetAttempts()); + metrics->SetCounter(CommitMetrics::LAST_TABLE_FILES_ADDED, commit_stats.GetTableFilesAdded()); + metrics->SetCounter(CommitMetrics::LAST_TABLE_FILES_DELETED, + commit_stats.GetTableFilesDeleted()); + metrics->SetCounter(CommitMetrics::LAST_TABLE_FILES_APPENDED, + commit_stats.GetTableFilesAppended()); + metrics->SetCounter(CommitMetrics::LAST_TABLE_FILES_COMMIT_COMPACTED, + commit_stats.GetTableFilesCompacted()); + metrics->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_APPENDED, + commit_stats.GetChangelogFilesAppended()); + metrics->SetCounter(CommitMetrics::LAST_CHANGELOG_FILES_COMMIT_COMPACTED, + commit_stats.GetChangelogFilesCompacted()); + metrics->SetCounter(CommitMetrics::LAST_GENERATED_SNAPSHOTS, + commit_stats.GetGeneratedSnapshots()); + metrics->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_APPENDED, + commit_stats.GetDeltaRecordsAppended()); + metrics->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_APPENDED, + commit_stats.GetChangelogRecordsAppended()); + metrics->SetCounter(CommitMetrics::LAST_DELTA_RECORDS_COMMIT_COMPACTED, + commit_stats.GetDeltaRecordsCompacted()); + metrics->SetCounter(CommitMetrics::LAST_CHANGELOG_RECORDS_COMMIT_COMPACTED, + commit_stats.GetChangelogRecordsCompacted()); + metrics->SetCounter(CommitMetrics::LAST_PARTITIONS_WRITTEN, + commit_stats.GetNumPartitionsWritten()); + metrics->SetCounter(CommitMetrics::LAST_BUCKETS_WRITTEN, commit_stats.GetNumBucketsWritten()); + metrics->SetCounter(CommitMetrics::LAST_COMPACTION_INPUT_FILE_SIZE, + commit_stats.GetCompactionInputFileSize()); + metrics->SetCounter(CommitMetrics::LAST_COMPACTION_OUTPUT_FILE_SIZE, + commit_stats.GetCompactionOutputFileSize()); + metrics->SetCounter(CommitMetrics::LAST_COMMITTED_SNAPSHOT_ID, + commit_stats.GetLastCommittedSnapshotId()); +} + +} // namespace paimon diff --git a/src/paimon/core/operation/metrics/commit_metrics.h b/src/paimon/core/operation/metrics/commit_metrics.h index cb26a121..ce9c25ce 100644 --- a/src/paimon/core/operation/metrics/commit_metrics.h +++ b/src/paimon/core/operation/metrics/commit_metrics.h @@ -19,8 +19,13 @@ #pragma once +#include + namespace paimon { +class CommitStats; +class Metrics; + /// Metrics to measure a commit. class CommitMetrics { public: @@ -43,6 +48,10 @@ class CommitMetrics { static constexpr char LAST_BUCKETS_WRITTEN[] = "lastBucketsWritten"; static constexpr char LAST_COMPACTION_INPUT_FILE_SIZE[] = "lastCompactionInputFileSize"; static constexpr char LAST_COMPACTION_OUTPUT_FILE_SIZE[] = "lastCompactionOutputFileSize"; + static constexpr char LAST_COMMITTED_SNAPSHOT_ID[] = "lastCommittedSnapshotId"; + + static void ReportCommit(const std::shared_ptr& metrics, + const CommitStats& commit_stats); }; } // namespace paimon diff --git a/src/paimon/core/operation/metrics/commit_stats.h b/src/paimon/core/operation/metrics/commit_stats.h new file mode 100644 index 00000000..562ded5f --- /dev/null +++ b/src/paimon/core/operation/metrics/commit_stats.h @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/common/data/binary_row.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/manifest_entry.h" + +namespace paimon { + +/// Statistics for a commit. Interface is aligned with Java CommitStats. +class CommitStats { + public: + CommitStats(const std::vector& append_table_files, + const std::vector& append_changelog_files, + const std::vector& compact_table_files, + const std::vector& compact_changelog_files, int64_t commit_duration, + int32_t generated_snapshots, int32_t attempts, int64_t last_committed_snapshot_id) { + duration_ = commit_duration; + attempts_ = attempts; + table_files_appended_ = static_cast(append_table_files.size()); + changelog_files_appended_ = static_cast(append_changelog_files.size()); + changelog_files_compacted_ = static_cast(compact_changelog_files.size()); + changelog_records_compacted_ = RowCounts(compact_changelog_files); + delta_records_compacted_ = RowCounts(compact_table_files); + changelog_records_appended_ = RowCounts(append_changelog_files); + delta_records_appended_ = RowCounts(append_table_files); + table_files_compacted_ = static_cast(compact_table_files.size()); + generated_snapshots_ = generated_snapshots; + num_partitions_written_ = NumChangedPartitions({append_table_files, compact_table_files}); + num_buckets_written_ = NumChangedBuckets({append_table_files, compact_table_files}); + last_committed_snapshot_id_ = last_committed_snapshot_id; + + std::vector added_table_files; + std::vector deleted_table_files; + for (const auto& entry : append_table_files) { + if (entry.Kind() == FileKind::Add()) { + added_table_files.push_back(entry); + } else if (entry.Kind() == FileKind::Delete()) { + deleted_table_files.push_back(entry); + } + } + + std::vector compact_after_files; + std::vector compaction_input_files; + for (const auto& entry : compact_table_files) { + if (entry.Kind() == FileKind::Add()) { + compact_after_files.push_back(entry); + } else if (entry.Kind() == FileKind::Delete()) { + compaction_input_files.push_back(entry); + } + } + + added_table_files.insert(added_table_files.end(), compact_after_files.begin(), + compact_after_files.end()); + deleted_table_files.insert(deleted_table_files.end(), compaction_input_files.begin(), + compaction_input_files.end()); + + table_files_added_ = static_cast(added_table_files.size()); + table_files_deleted_ = static_cast(deleted_table_files.size()); + compaction_input_file_size_ = FileSizes(compaction_input_files); + compaction_output_file_size_ = FileSizes(compact_after_files); + } + + int64_t GetTableFilesAdded() const { + return table_files_added_; + } + int64_t GetTableFilesDeleted() const { + return table_files_deleted_; + } + int64_t GetTableFilesAppended() const { + return table_files_appended_; + } + int64_t GetTableFilesCompacted() const { + return table_files_compacted_; + } + int64_t GetChangelogFilesAppended() const { + return changelog_files_appended_; + } + int64_t GetChangelogFilesCompacted() const { + return changelog_files_compacted_; + } + int64_t GetGeneratedSnapshots() const { + return generated_snapshots_; + } + int64_t GetDeltaRecordsAppended() const { + return delta_records_appended_; + } + int64_t GetChangelogRecordsAppended() const { + return changelog_records_appended_; + } + int64_t GetDeltaRecordsCompacted() const { + return delta_records_compacted_; + } + int64_t GetChangelogRecordsCompacted() const { + return changelog_records_compacted_; + } + int64_t GetNumPartitionsWritten() const { + return num_partitions_written_; + } + int64_t GetNumBucketsWritten() const { + return num_buckets_written_; + } + int64_t GetDuration() const { + return duration_; + } + int32_t GetAttempts() const { + return attempts_; + } + int64_t GetCompactionInputFileSize() const { + return compaction_input_file_size_; + } + int64_t GetCompactionOutputFileSize() const { + return compaction_output_file_size_; + } + int64_t GetLastCommittedSnapshotId() const { + return last_committed_snapshot_id_; + } + + static int64_t NumChangedPartitions(const std::vector>& changes) { + std::unordered_set changed_partitions; + for (const auto& change : changes) { + for (const auto& entry : change) { + changed_partitions.insert(entry.Partition()); + } + } + return static_cast(changed_partitions.size()); + } + + static int64_t NumChangedBuckets(const std::vector>& changes) { + std::unordered_map> changed_partition_buckets; + for (const auto& change : changes) { + for (const auto& entry : change) { + changed_partition_buckets[entry.Partition()].insert(entry.Bucket()); + } + } + + int64_t num_changed_buckets = 0; + for (const auto& [_, buckets] : changed_partition_buckets) { + num_changed_buckets += static_cast(buckets.size()); + } + return num_changed_buckets; + } + + private: + static int64_t RowCounts(const std::vector& files) { + int64_t row_count = 0; + for (const auto& entry : files) { + row_count += entry.File()->row_count; + } + return row_count; + } + + static int64_t FileSizes(const std::vector& files) { + int64_t file_size = 0; + for (const auto& entry : files) { + file_size += entry.File()->file_size; + } + return file_size; + } + + private: + int64_t duration_ = 0; + int32_t attempts_ = 0; + int64_t table_files_appended_ = 0; + int64_t table_files_added_ = 0; + int64_t table_files_deleted_ = 0; + int64_t changelog_files_appended_ = 0; + int64_t compaction_input_file_size_ = 0; + int64_t compaction_output_file_size_ = 0; + int64_t changelog_files_compacted_ = 0; + int64_t changelog_records_compacted_ = 0; + int64_t delta_records_compacted_ = 0; + int64_t changelog_records_appended_ = 0; + int64_t delta_records_appended_ = 0; + int64_t table_files_compacted_ = 0; + int64_t generated_snapshots_ = 0; + int64_t num_partitions_written_ = 0; + int64_t num_buckets_written_ = 0; + int64_t last_committed_snapshot_id_ = -1; +}; + +} // namespace paimon diff --git a/src/paimon/core/operation/metrics/commit_stats_test.cpp b/src/paimon/core/operation/metrics/commit_stats_test.cpp new file mode 100644 index 00000000..a18f88a6 --- /dev/null +++ b/src/paimon/core/operation/metrics/commit_stats_test.cpp @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/metrics/commit_stats.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row_writer.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_source.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +BinaryRow CreateIntRow(int32_t value) { + BinaryRow row(1); + BinaryRowWriter writer(&row, 20, GetDefaultPool().get()); + writer.WriteInt(0, value); + writer.Complete(); + return row; +} + +ManifestEntry CreateEntry(const FileKind& kind, int32_t partition, int32_t bucket, + int64_t row_count, int64_t file_size, const std::string& file_name) { + BinaryRow part = CreateIntRow(partition); + auto file_meta = std::make_shared( + file_name, file_size, row_count, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_sequence_number=*/0, /*max_sequence_number=*/0, + /*schema_id=*/1, /*level=*/0, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, FileSource::Append(), + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, + /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + return ManifestEntry(kind, part, bucket, /*total_buckets=*/10, file_meta); +} + +} // namespace + +TEST(CommitStatsTest, TestCalcChangedPartitionsAndBuckets) { + std::vector files; + files.push_back(CreateEntry(FileKind::Add(), /*partition=*/1, /*bucket=*/1, + /*row_count=*/201, /*file_size=*/11, "a1")); + files.push_back(CreateEntry(FileKind::Add(), /*partition=*/2, /*bucket=*/3, + /*row_count=*/302, /*file_size=*/12, "a2")); + files.push_back(CreateEntry(FileKind::Add(), /*partition=*/1, /*bucket=*/1, + /*row_count=*/202, /*file_size=*/13, "c1")); + files.push_back(CreateEntry(FileKind::Add(), /*partition=*/2, /*bucket=*/3, + /*row_count=*/301, /*file_size=*/14, "c2")); + files.push_back(CreateEntry(FileKind::Add(), /*partition=*/1, /*bucket=*/1, + /*row_count=*/203, /*file_size=*/15, "k1")); + files.push_back(CreateEntry(FileKind::Add(), /*partition=*/2, /*bucket=*/3, + /*row_count=*/304, /*file_size=*/16, "k2")); + files.push_back(CreateEntry(FileKind::Delete(), /*partition=*/3, /*bucket=*/5, + /*row_count=*/106, /*file_size=*/17, "k3")); + files.push_back(CreateEntry(FileKind::Add(), /*partition=*/1, /*bucket=*/1, + /*row_count=*/205, /*file_size=*/18, "ck1")); + files.push_back(CreateEntry(FileKind::Add(), /*partition=*/2, /*bucket=*/3, + /*row_count=*/307, /*file_size=*/19, "ck2")); + + EXPECT_EQ(3, CommitStats::NumChangedBuckets({files})); + EXPECT_EQ(3, CommitStats::NumChangedPartitions({files})); +} + +TEST(CommitStatsTest, TestFailedAppendSnapshot) { + CommitStats stats(/*append_table_files=*/{}, /*append_changelog_files=*/{}, + /*compact_table_files=*/{}, /*compact_changelog_files=*/{}, + /*commit_duration=*/0, /*generated_snapshots=*/0, /*attempts=*/1, + /*last_committed_snapshot_id=*/-1); + + EXPECT_EQ(0, stats.GetTableFilesAdded()); + EXPECT_EQ(0, stats.GetTableFilesDeleted()); + EXPECT_EQ(0, stats.GetTableFilesAppended()); + EXPECT_EQ(0, stats.GetTableFilesCompacted()); + EXPECT_EQ(0, stats.GetChangelogFilesAppended()); + EXPECT_EQ(0, stats.GetChangelogFilesCompacted()); + EXPECT_EQ(0, stats.GetGeneratedSnapshots()); + EXPECT_EQ(0, stats.GetDeltaRecordsAppended()); + EXPECT_EQ(0, stats.GetChangelogRecordsAppended()); + EXPECT_EQ(0, stats.GetDeltaRecordsCompacted()); + EXPECT_EQ(0, stats.GetChangelogRecordsCompacted()); + EXPECT_EQ(0, stats.GetNumPartitionsWritten()); + EXPECT_EQ(0, stats.GetNumBucketsWritten()); + EXPECT_EQ(0, stats.GetDuration()); + EXPECT_EQ(1, stats.GetAttempts()); + EXPECT_EQ(-1, stats.GetLastCommittedSnapshotId()); +} + +TEST(CommitStatsTest, TestSucceedAllSnapshot) { + std::vector append_data_files; + append_data_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 201, 1001, "a1")); + append_data_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 302, 1002, "a2")); + + std::vector append_changelog_files; + append_changelog_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 202, 2001, "c1")); + append_changelog_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 301, 2002, "c2")); + + std::vector compact_data_files; + compact_data_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 203, 3001, "k1")); + compact_data_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 304, 3002, "k2")); + compact_data_files.push_back(CreateEntry(FileKind::Delete(), 3, 5, 106, 3003, "k3")); + + std::vector compact_changelog_files; + compact_changelog_files.push_back(CreateEntry(FileKind::Add(), 1, 1, 205, 4001, "ck1")); + compact_changelog_files.push_back(CreateEntry(FileKind::Add(), 2, 3, 307, 4002, "ck2")); + + CommitStats stats(append_data_files, append_changelog_files, compact_data_files, + compact_changelog_files, + /*commit_duration=*/3000, /*generated_snapshots=*/2, /*attempts=*/2, + /*last_committed_snapshot_id=*/10); + + EXPECT_EQ(4, stats.GetTableFilesAdded()); + EXPECT_EQ(1, stats.GetTableFilesDeleted()); + EXPECT_EQ(2, stats.GetTableFilesAppended()); + EXPECT_EQ(3, stats.GetTableFilesCompacted()); + EXPECT_EQ(2, stats.GetChangelogFilesAppended()); + EXPECT_EQ(2, stats.GetChangelogFilesCompacted()); + EXPECT_EQ(2, stats.GetGeneratedSnapshots()); + EXPECT_EQ(503, stats.GetDeltaRecordsAppended()); + EXPECT_EQ(503, stats.GetChangelogRecordsAppended()); + EXPECT_EQ(613, stats.GetDeltaRecordsCompacted()); + EXPECT_EQ(512, stats.GetChangelogRecordsCompacted()); + EXPECT_EQ(3, stats.GetNumPartitionsWritten()); + EXPECT_EQ(3, stats.GetNumBucketsWritten()); + EXPECT_EQ(3000, stats.GetDuration()); + EXPECT_EQ(2, stats.GetAttempts()); + EXPECT_EQ(10, stats.GetLastCommittedSnapshotId()); + EXPECT_EQ(3003, stats.GetCompactionInputFileSize()); + EXPECT_EQ(6003, stats.GetCompactionOutputFileSize()); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/orphan_files_cleaner_test.cpp b/src/paimon/core/operation/orphan_files_cleaner_test.cpp index 37731049..7749b692 100644 --- a/src/paimon/core/operation/orphan_files_cleaner_test.cpp +++ b/src/paimon/core/operation/orphan_files_cleaner_test.cpp @@ -230,7 +230,6 @@ TEST(OrphanFilesCleanerTest, TestTableWithChangelog) { "commitIdentifier" : 9223372036854775807, "commitKind" : "APPEND", "timeMillis" : 1721615035363, - "logOffsets" : { }, "totalRecordCount" : 11, "deltaRecordCount" : 1, "changelogRecordCount" : 0 @@ -263,7 +262,6 @@ TEST(OrphanFilesCleanerTest, TestTableWithIndexManifest) { "commitIdentifier" : 9223372036854775807, "commitKind" : "APPEND", "timeMillis" : 1721615035363, - "logOffsets" : { }, "totalRecordCount" : 11, "deltaRecordCount" : 1, "changelogRecordCount" : 0 diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 8d1fcef1..c27e609f 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -107,14 +107,10 @@ Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) { // TODO(yonghao.fyh): check changelog num retain // TODO(yonghao.fyh): support file format validate data fields for (const auto& field_name : field_names) { - if (SpecialFields::IsSpecialFieldName(field_name)) { + if (SpecialFields::IsSystemField(field_name)) { return Status::Invalid( fmt::format("field name '{}' in schema cannot be special field.", field_name)); } - if (StringUtils::StartsWith(field_name, SpecialFields::KEY_FIELD_PREFIX)) { - return Status::Invalid(fmt::format("field name '{}' in schema cannot start with '{}'.", - field_name, SpecialFields::KEY_FIELD_PREFIX)); - } } // TODO(yonghao.fyh): check streaming read overwrite // TODO(yonghao.fyh): check 'partition.expiration-time' diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 1e411f98..09a31c40 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -725,7 +725,7 @@ TEST(SchemaValidationTest, ValidateInvalidConfiguration) { TableSchema::Create(/*schema_id=*/0, invalid_schema, /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), - "field name '_KEY_a' in schema cannot start with '_KEY_'"); + "field name '_KEY_a' in schema cannot be special field."); } { std::map options = {{Options::CHANGELOG_PRODUCER, "input"}, diff --git a/src/paimon/core/snapshot.cpp b/src/paimon/core/snapshot.cpp index 5ba177ea..e87f708b 100644 --- a/src/paimon/core/snapshot.cpp +++ b/src/paimon/core/snapshot.cpp @@ -81,7 +81,7 @@ bool Snapshot::TEST_Equal(const Snapshot& other) const { return version_ == other.version_ && id_ == other.id_ && schema_id_ == other.schema_id_ && index_manifest_ == other.index_manifest_ && commit_user_ == other.commit_user_ && commit_identifier_ == other.commit_identifier_ && commit_kind_ == other.commit_kind_ && - log_offsets_ == other.log_offsets_ && total_record_count_ == other.total_record_count_ && + total_record_count_ == other.total_record_count_ && delta_record_count_ == other.delta_record_count_ && changelog_record_count_ == other.changelog_record_count_ && watermark_ == other.watermark_ && statistics_ == other.statistics_ && @@ -101,8 +101,7 @@ bool Snapshot::operator==(const Snapshot& other) const { changelog_manifest_list_size_ == other.changelog_manifest_list_size_ && index_manifest_ == other.index_manifest_ && commit_user_ == other.commit_user_ && commit_identifier_ == other.commit_identifier_ && commit_kind_ == other.commit_kind_ && - time_millis_ == other.time_millis_ && log_offsets_ == other.log_offsets_ && - total_record_count_ == other.total_record_count_ && + time_millis_ == other.time_millis_ && total_record_count_ == other.total_record_count_ && delta_record_count_ == other.delta_record_count_ && changelog_record_count_ == other.changelog_record_count_ && watermark_ == other.watermark_ && statistics_ == other.statistics_ && @@ -147,9 +146,7 @@ Snapshot::Snapshot(const std::optional& version, int64_t id, int64_t sc const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + int64_t total_record_count, int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, @@ -169,7 +166,6 @@ Snapshot::Snapshot(const std::optional& version, int64_t id, int64_t sc commit_identifier_(commit_identifier), commit_kind_(commit_kind), time_millis_(time_millis), - log_offsets_(log_offsets), total_record_count_(total_record_count), delta_record_count_(delta_record_count), changelog_record_count_(changelog_record_count), @@ -230,17 +226,10 @@ rapidjson::Value Snapshot::ToJson(rapidjson::Document::AllocatorType* allocator) obj.AddMember(rapidjson::StringRef(FIELD_TIME_MILLIS), RapidJsonUtil::SerializeValue(time_millis_, allocator).Move(), *allocator); - if (log_offsets_ != std::nullopt) { - obj.AddMember(rapidjson::StringRef(FIELD_LOG_OFFSETS), - RapidJsonUtil::SerializeValue(log_offsets_.value(), allocator).Move(), - *allocator); - } obj.AddMember(rapidjson::StringRef(FIELD_TOTAL_RECORD_COUNT), - RapidJsonUtil::SerializeValue(total_record_count_.value(), allocator).Move(), - *allocator); + RapidJsonUtil::SerializeValue(total_record_count_, allocator).Move(), *allocator); obj.AddMember(rapidjson::StringRef(FIELD_DELTA_RECORD_COUNT), - RapidJsonUtil::SerializeValue(delta_record_count_.value(), allocator).Move(), - *allocator); + RapidJsonUtil::SerializeValue(delta_record_count_, allocator).Move(), *allocator); if (changelog_record_count_ != std::nullopt) { obj.AddMember( @@ -300,12 +289,10 @@ void Snapshot::FromJson(const rapidjson::Value& obj) noexcept(false) { throw std::invalid_argument("deserialize CommitKind failed"); } time_millis_ = RapidJsonUtil::DeserializeKeyValue(obj, FIELD_TIME_MILLIS); - log_offsets_ = RapidJsonUtil::DeserializeKeyValue>>( - obj, FIELD_LOG_OFFSETS); total_record_count_ = - RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_TOTAL_RECORD_COUNT); + RapidJsonUtil::DeserializeKeyValue(obj, FIELD_TOTAL_RECORD_COUNT); delta_record_count_ = - RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_DELTA_RECORD_COUNT); + RapidJsonUtil::DeserializeKeyValue(obj, FIELD_DELTA_RECORD_COUNT); changelog_record_count_ = RapidJsonUtil::DeserializeKeyValue>( obj, FIELD_CHANGELOG_RECORD_COUNT); watermark_ = RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_WATERMARK); diff --git a/src/paimon/core/snapshot.h b/src/paimon/core/snapshot.h index 827d416e..1b7f6593 100644 --- a/src/paimon/core/snapshot.h +++ b/src/paimon/core/snapshot.h @@ -58,6 +58,11 @@ class Snapshot : public Jsonizable { bool operator==(const CommitKind& other) const { return value_ == other.value_; } + + bool operator!=(const CommitKind& other) const { + return value_ != other.value_; + } + static std::string ToString(const CommitKind& kind); static CommitKind FromString(const std::string& kind); @@ -79,7 +84,6 @@ class Snapshot : public Jsonizable { static constexpr char FIELD_COMMIT_IDENTIFIER[] = "commitIdentifier"; static constexpr char FIELD_COMMIT_KIND[] = "commitKind"; static constexpr char FIELD_TIME_MILLIS[] = "timeMillis"; - static constexpr char FIELD_LOG_OFFSETS[] = "logOffsets"; static constexpr char FIELD_TOTAL_RECORD_COUNT[] = "totalRecordCount"; static constexpr char FIELD_DELTA_RECORD_COUNT[] = "deltaRecordCount"; static constexpr char FIELD_CHANGELOG_RECORD_COUNT[] = "changelogRecordCount"; @@ -98,9 +102,7 @@ class Snapshot : public Jsonizable { const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + int64_t total_record_count, int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, @@ -108,7 +110,7 @@ class Snapshot : public Jsonizable { : Snapshot(CURRENT_VERSION, id, schema_id, base_manifest_list, base_manifest_list_size, delta_manifest_list, delta_manifest_list_size, changelog_manifest_list, changelog_manifest_list_size, index_manifest, commit_user, commit_identifier, - commit_kind, time_millis, log_offsets, total_record_count, delta_record_count, + commit_kind, time_millis, total_record_count, delta_record_count, changelog_record_count, watermark, statistics, properties, next_row_id) {} Snapshot(const std::optional& version, int64_t id, int64_t schema_id, @@ -120,9 +122,7 @@ class Snapshot : public Jsonizable { const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + int64_t total_record_count, int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, @@ -194,15 +194,11 @@ class Snapshot : public Jsonizable { return time_millis_; } - const std::optional>& LogOffsets() const { - return log_offsets_; - } - - const std::optional& TotalRecordCount() const { + int64_t TotalRecordCount() const { return total_record_count_; } - const std::optional& DeltaRecordCount() const { + int64_t DeltaRecordCount() const { return delta_record_count_; } @@ -275,15 +271,11 @@ class Snapshot : public Jsonizable { int64_t time_millis_; - std::optional> log_offsets_; - // record count of all changes occurred in this snapshot - // null for paimon <= 0.3 - std::optional total_record_count_; + int64_t total_record_count_ = 0; // record count of all new changes occurred in this snapshot - // null for paimon <= 0.3 - std::optional delta_record_count_; + int64_t delta_record_count_ = 0; // record count of all changelog produced in this snapshot // null for paimon <= 0.3 diff --git a/src/paimon/core/snapshot_test.cpp b/src/paimon/core/snapshot_test.cpp index b5694e44..d786e803 100644 --- a/src/paimon/core/snapshot_test.cpp +++ b/src/paimon/core/snapshot_test.cpp @@ -34,12 +34,15 @@ class SnapshotTest : public testing::Test { std::string replaced_str = StringUtils::Replace(str, " ", ""); replaced_str = StringUtils::Replace(replaced_str, "\t", ""); replaced_str = StringUtils::Replace(replaced_str, "\n", ""); + // logOffsets was removed from snapshot json; normalize legacy fixtures. + replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{},", ""); + replaced_str = StringUtils::Replace(replaced_str, ",\"logOffsets\":{}", ""); + replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{}", ""); return replaced_str; } }; TEST_F(SnapshotTest, TestSimple) { - std::map log_offset = {{25, 30}}; std::map properties = {{"key1", "value1"}, {"key2", "value2"}}; Snapshot snapshot( /*version=*/5, /*id=*/10, /*schema_id=*/15, /*base_manifest_list=*/"base_manifest_list", 10, @@ -47,7 +50,7 @@ TEST_F(SnapshotTest, TestSimple) { /*changelog_manifest_list=*/"changelog_manifest_list", 30, /*index_manifest=*/"index_manifest", /*commit_user=*/"commit_user_01", /*commit_identifier=*/20, - /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, log_offset, + /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, /*total_record_count=*/35, /*delta_record_count=*/40, /*changelog_record_count=*/45, /*watermark=*/50, /*statistics=*/"statistic_test", properties, /*next_row_id=*/0); @@ -65,9 +68,8 @@ TEST_F(SnapshotTest, TestSimple) { ASSERT_EQ(20, snapshot.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot.GetCommitKind()); ASSERT_EQ(1234, snapshot.TimeMillis()); - ASSERT_EQ(log_offset, snapshot.LogOffsets().value()); - ASSERT_EQ(35, snapshot.TotalRecordCount().value()); - ASSERT_EQ(40, snapshot.DeltaRecordCount().value()); + ASSERT_EQ(35, snapshot.TotalRecordCount()); + ASSERT_EQ(40, snapshot.DeltaRecordCount()); ASSERT_EQ(45, snapshot.ChangelogRecordCount().value()); ASSERT_EQ(50, snapshot.Watermark().value()); ASSERT_EQ("statistic_test", snapshot.Statistics().value()); @@ -94,9 +96,8 @@ TEST_F(SnapshotTest, TestFromPath) { ASSERT_EQ(9223372036854775807ll, snapshot.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Append(), snapshot.GetCommitKind()); ASSERT_EQ(1721614343270ll, snapshot.TimeMillis()); - ASSERT_EQ((std::map()), snapshot.LogOffsets().value()); - ASSERT_EQ(5, snapshot.TotalRecordCount().value()); - ASSERT_EQ(5, snapshot.DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot.TotalRecordCount()); + ASSERT_EQ(5, snapshot.DeltaRecordCount()); ASSERT_EQ(0, snapshot.ChangelogRecordCount().value()); ASSERT_EQ(std::nullopt, snapshot.Watermark()); ASSERT_EQ(std::nullopt, snapshot.Statistics()); @@ -118,7 +119,6 @@ TEST_F(SnapshotTest, TestJsonizable) { "commitIdentifier" : 9223372036854775807, "commitKind" : "OVERWRITE", "timeMillis" : 1711692199281, - "logOffsets" : { }, "totalRecordCount" : 3, "deltaRecordCount" : 3, "changelogRecordCount" : 0 @@ -135,7 +135,6 @@ TEST_F(SnapshotTest, TestJsonizable) { /*commit_user=*/"0e4d92f7-53b0-40d6-a7c0-102bf3801e6a", /*commit_identifier=*/9223372036854775807ll, /*commit_kind=*/Snapshot::CommitKind::Overwrite(), /*time_millis=*/1711692199281ll, - /*log_offsets=*/std::map(), /*total_record_count=*/3, /*delta_record_count=*/3, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt); @@ -198,10 +197,6 @@ TEST_F(SnapshotTest, TestSerializeAndDeserialize) { "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, @@ -228,10 +223,6 @@ TEST_F(SnapshotTest, TestSerializeAndDeserialize) { "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, @@ -261,7 +252,6 @@ TEST_F(SnapshotTest, TestCommitKindAnalyze) { /*commit_identifier=*/42, /*commit_kind=*/Snapshot::CommitKind::Analyze(), /*time_millis=*/1700000000000ll, - /*log_offsets=*/std::map(), /*total_record_count=*/0, /*delta_record_count=*/0, /*changelog_record_count=*/0, @@ -288,7 +278,6 @@ TEST_F(SnapshotTest, TestCommitKindAnalyzeSerializeAndDeserialize) { "commitIdentifier" : 42, "commitKind" : "ANALYZE", "timeMillis" : 1700000000000, - "logOffsets" : { }, "totalRecordCount" : 0, "deltaRecordCount" : 0, "changelogRecordCount" : 0, @@ -354,7 +343,6 @@ TEST_F(SnapshotTest, TestChangelogManifestListSerialization) { "commitIdentifier" : 100, "commitKind" : "APPEND", "timeMillis" : 1700000000000, - "logOffsets" : { }, "totalRecordCount" : 10, "deltaRecordCount" : 5, "changelogRecordCount" : 3 @@ -385,7 +373,6 @@ TEST_F(SnapshotTest, TestChangelogManifestListSerialization) { "commitIdentifier" : 200, "commitKind" : "COMPACT", "timeMillis" : 1700000001000, - "logOffsets" : { }, "totalRecordCount" : 20, "deltaRecordCount" : 10, "changelogRecordCount" : 0 diff --git a/src/paimon/core/table/bucket_mode.cpp b/src/paimon/core/table/bucket_mode.cpp new file mode 100644 index 00000000..d46739f8 --- /dev/null +++ b/src/paimon/core/table/bucket_mode.cpp @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/bucket_mode.h" + +#include "paimon/core/schema/table_schema.h" + +namespace paimon { + +BucketMode ResolveBucketMode(int32_t bucket, const std::shared_ptr& table_schema) { + if (bucket == BucketModeDefine::POSTPONE_BUCKET) { + return BucketMode::POSTPONE_MODE; + } + if (bucket == -1) { + return table_schema->PrimaryKeys().empty() ? BucketMode::BUCKET_UNAWARE + : BucketMode::HASH_DYNAMIC; + } + if (bucket == BucketModeDefine::UNAWARE_BUCKET) { + return BucketMode::BUCKET_UNAWARE; + } + return BucketMode::HASH_FIXED; +} + +} // namespace paimon diff --git a/src/paimon/core/table/bucket_mode.h b/src/paimon/core/table/bucket_mode.h index 2891dac7..f87ab346 100644 --- a/src/paimon/core/table/bucket_mode.h +++ b/src/paimon/core/table/bucket_mode.h @@ -19,9 +19,12 @@ #pragma once #include +#include namespace paimon { +class TableSchema; + /// Bucket mode of the table, it affects the writing process and also affects the data skipping in /// reading. enum class BucketMode { @@ -64,4 +67,6 @@ class BucketModeDefine { static constexpr int32_t POSTPONE_BUCKET = -2; }; +BucketMode ResolveBucketMode(int32_t bucket, const std::shared_ptr& table_schema); + } // namespace paimon diff --git a/src/paimon/core/table/bucket_mode_test.cpp b/src/paimon/core/table/bucket_mode_test.cpp new file mode 100644 index 00000000..2a77e5ea --- /dev/null +++ b/src/paimon/core/table/bucket_mode_test.cpp @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/bucket_mode.h" + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +std::shared_ptr CreateTableSchema(const std::vector& primary_keys) { + auto schema = arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::int32(), /*nullable=*/false)}); + std::vector partition_keys = {"f1"}; + std::map options; + EXPECT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, partition_keys, primary_keys, options)); + return table_schema; +} + +} // namespace + +TEST(BucketModeTest, TestResolveBucketMode) { + std::shared_ptr append_schema = CreateTableSchema(/*primary_keys=*/{}); + std::shared_ptr pk_schema = CreateTableSchema(/*primary_keys=*/{"f0"}); + + EXPECT_EQ(BucketMode::POSTPONE_MODE, + ResolveBucketMode(BucketModeDefine::POSTPONE_BUCKET, append_schema)); + EXPECT_EQ(BucketMode::BUCKET_UNAWARE, ResolveBucketMode(-1, append_schema)); + EXPECT_EQ(BucketMode::HASH_DYNAMIC, ResolveBucketMode(-1, pk_schema)); + EXPECT_EQ(BucketMode::BUCKET_UNAWARE, + ResolveBucketMode(BucketModeDefine::UNAWARE_BUCKET, pk_schema)); + EXPECT_EQ(BucketMode::HASH_FIXED, ResolveBucketMode(4, append_schema)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp index faa62ee8..1563a3f7 100644 --- a/src/paimon/core/table/system/metadata_system_tables.cpp +++ b/src/paimon/core/table/system/metadata_system_tables.cpp @@ -427,7 +427,7 @@ Result> ProjectWriteFields(const std::shared_ptr fields; fields.reserve(file.write_cols->size() + data_schema->PartitionKeys().size()); for (const auto& write_col : file.write_cols.value()) { - if (SpecialFields::IsSpecialFieldName(write_col)) { + if (SpecialFields::IsSystemField(write_col)) { continue; } PAIMON_ASSIGN_OR_RAISE(DataField field, data_schema->GetField(write_col)); @@ -515,8 +515,8 @@ Result> SnapshotsSystemTable::ArrowSchema() const arrow::field("base_manifest_list", arrow::utf8(), /*nullable=*/false), arrow::field("delta_manifest_list", arrow::utf8(), /*nullable=*/false), arrow::field("changelog_manifest_list", arrow::utf8(), /*nullable=*/true), - arrow::field("total_record_count", arrow::int64(), /*nullable=*/true), - arrow::field("delta_record_count", arrow::int64(), /*nullable=*/true), + arrow::field("total_record_count", arrow::int64(), /*nullable=*/false), + arrow::field("delta_record_count", arrow::int64(), /*nullable=*/false), arrow::field("changelog_record_count", arrow::int64(), /*nullable=*/true), arrow::field("watermark", arrow::int64(), /*nullable=*/true), arrow::field("next_row_id", arrow::int64(), /*nullable=*/true), @@ -545,8 +545,8 @@ Result> SnapshotsSystemTable::BuildRows() const { row.SetField(6, StringValue(snapshot.BaseManifestList())); row.SetField(7, StringValue(snapshot.DeltaManifestList())); row.SetField(8, OptionalStringValue(snapshot.ChangelogManifestList())); - row.SetField(9, OptionalInt64Value(snapshot.TotalRecordCount())); - row.SetField(10, OptionalInt64Value(snapshot.DeltaRecordCount())); + row.SetField(9, snapshot.TotalRecordCount()); + row.SetField(10, snapshot.DeltaRecordCount()); row.SetField(11, OptionalInt64Value(snapshot.ChangelogRecordCount())); row.SetField(12, OptionalInt64Value(snapshot.Watermark())); row.SetField(13, OptionalInt64Value(snapshot.NextRowId())); @@ -628,7 +628,7 @@ Result> TagsSystemTable::ArrowSchema() const { arrow::field("schema_id", arrow::int64(), /*nullable=*/false), arrow::field("commit_time", arrow::timestamp(arrow::TimeUnit::MILLI), /*nullable=*/false), - arrow::field("record_count", arrow::int64(), /*nullable=*/true), + arrow::field("record_count", arrow::int64(), /*nullable=*/false), arrow::field("create_time", arrow::timestamp(arrow::TimeUnit::MILLI), /*nullable=*/true), arrow::field("time_retained", arrow::utf8(), /*nullable=*/true), @@ -653,7 +653,7 @@ Result> TagsSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(VariantType commit_time, LocalTimestampMillisValue(tag.TimeMillis())); row.SetField(3, commit_time); - row.SetField(4, OptionalInt64Value(tag.TotalRecordCount())); + row.SetField(4, tag.TotalRecordCount()); row.SetField(5, OptionalTimestampMillisValue(tag_create_time)); row.SetField(6, OptionalStringValue(OptionalDoubleToString(tag.TagTimeRetained()))); rows.push_back(std::move(row)); diff --git a/src/paimon/core/tag/tag.cpp b/src/paimon/core/tag/tag.cpp index ebf77737..6ee31864 100644 --- a/src/paimon/core/tag/tag.cpp +++ b/src/paimon/core/tag/tag.cpp @@ -39,9 +39,7 @@ Tag::Tag(const std::optional& version, const int64_t id, const int64_t const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, const int64_t commit_identifier, const CommitKind commit_kind, const int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + const int64_t total_record_count, const int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, @@ -51,7 +49,7 @@ Tag::Tag(const std::optional& version, const int64_t id, const int64_t : Snapshot(version, id, schema_id, base_manifest_list, base_manifest_list_size, delta_manifest_list, delta_manifest_list_size, changelog_manifest_list, changelog_manifest_list_size, index_manifest, commit_user, commit_identifier, - commit_kind, time_millis, log_offsets, total_record_count, delta_record_count, + commit_kind, time_millis, total_record_count, delta_record_count, changelog_record_count, watermark, statistics, properties, next_row_id), tag_create_time_(tag_create_time), tag_time_retained_(tag_time_retained) {} @@ -77,9 +75,8 @@ Result Tag::TrimToSnapshot() const { return Snapshot(Version(), Id(), SchemaId(), BaseManifestList(), BaseManifestListSize(), DeltaManifestList(), DeltaManifestListSize(), ChangelogManifestList(), ChangelogManifestListSize(), IndexManifest(), CommitUser(), CommitIdentifier(), - GetCommitKind(), TimeMillis(), LogOffsets(), TotalRecordCount(), - DeltaRecordCount(), ChangelogRecordCount(), Watermark(), Statistics(), - Properties(), NextRowId()); + GetCommitKind(), TimeMillis(), TotalRecordCount(), DeltaRecordCount(), + ChangelogRecordCount(), Watermark(), Statistics(), Properties(), NextRowId()); } rapidjson::Value Tag::ToJson(rapidjson::Document::AllocatorType* allocator) const noexcept(false) { diff --git a/src/paimon/core/tag/tag.h b/src/paimon/core/tag/tag.h index 23508e58..5e84abb3 100644 --- a/src/paimon/core/tag/tag.h +++ b/src/paimon/core/tag/tag.h @@ -52,9 +52,7 @@ class Tag : public Snapshot { const std::optional& changelog_manifest_list_size, const std::optional& index_manifest, const std::string& commit_user, int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, - const std::optional>& log_offsets, - const std::optional& total_record_count, - const std::optional& delta_record_count, + int64_t total_record_count, int64_t delta_record_count, const std::optional& changelog_record_count, const std::optional& watermark, const std::optional& statistics, const std::optional>& properties, diff --git a/src/paimon/core/tag/tag_test.cpp b/src/paimon/core/tag/tag_test.cpp index fb371e82..b3423a2c 100644 --- a/src/paimon/core/tag/tag_test.cpp +++ b/src/paimon/core/tag/tag_test.cpp @@ -34,6 +34,10 @@ class TagTest : public testing::Test { std::string replaced_str = StringUtils::Replace(str, " ", ""); replaced_str = StringUtils::Replace(replaced_str, "\t", ""); replaced_str = StringUtils::Replace(replaced_str, "\n", ""); + // logOffsets was removed from snapshot json; normalize legacy fixtures. + replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{},", ""); + replaced_str = StringUtils::Replace(replaced_str, ",\"logOffsets\":{}", ""); + replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{}", ""); if (serialized) { replaced_str = StringUtils::Replace(replaced_str, ".0", ".000000000"); } @@ -42,7 +46,6 @@ class TagTest : public testing::Test { }; TEST_F(TagTest, TestSimple) { - const std::map log_offset = {{25, 30}}; const std::map properties = {{"key1", "value1"}, {"key2", "value2"}}; const auto tag_create_time = std::vector({2026, 1, 2, 3, 4, 5, 6}); const Tag tag( @@ -51,7 +54,7 @@ TEST_F(TagTest, TestSimple) { /*changelog_manifest_list=*/"changelog_manifest_list", 30, /*index_manifest=*/"index_manifest", /*commit_user=*/"commit_user_01", /*commit_identifier=*/20, - /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, log_offset, + /*commit_kind=*/Snapshot::CommitKind::Compact(), /*time_millis=*/1234, /*total_record_count=*/35, /*delta_record_count=*/40, /*changelog_record_count=*/45, /*watermark=*/50, /*statistics=*/"statistic_test", properties, /*next_row_id=*/0, @@ -70,9 +73,8 @@ TEST_F(TagTest, TestSimple) { ASSERT_EQ(20, tag.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Compact(), tag.GetCommitKind()); ASSERT_EQ(1234, tag.TimeMillis()); - ASSERT_EQ(log_offset, tag.LogOffsets().value()); - ASSERT_EQ(35, tag.TotalRecordCount().value()); - ASSERT_EQ(40, tag.DeltaRecordCount().value()); + ASSERT_EQ(35, tag.TotalRecordCount()); + ASSERT_EQ(40, tag.DeltaRecordCount()); ASSERT_EQ(45, tag.ChangelogRecordCount().value()); ASSERT_EQ(50, tag.Watermark().value()); ASSERT_EQ("statistic_test", tag.Statistics().value()); @@ -101,9 +103,8 @@ TEST_F(TagTest, TestFromPath) { ASSERT_EQ(9223372036854775807ll, tag.CommitIdentifier()); ASSERT_EQ(Snapshot::CommitKind::Append(), tag.GetCommitKind()); ASSERT_EQ(1721614343270ll, tag.TimeMillis()); - ASSERT_EQ((std::map()), tag.LogOffsets().value()); - ASSERT_EQ(5, tag.TotalRecordCount().value()); - ASSERT_EQ(5, tag.DeltaRecordCount().value()); + ASSERT_EQ(5, tag.TotalRecordCount()); + ASSERT_EQ(5, tag.DeltaRecordCount()); ASSERT_EQ(0, tag.ChangelogRecordCount().value()); ASSERT_EQ(std::nullopt, tag.Watermark()); ASSERT_EQ(std::nullopt, tag.Statistics()); @@ -127,7 +128,6 @@ TEST_F(TagTest, TestJsonizable) { "commitIdentifier" : 9223372036854775807, "commitKind" : "OVERWRITE", "timeMillis" : 1711692199281, - "logOffsets" : { }, "totalRecordCount" : 3, "deltaRecordCount" : 3, "changelogRecordCount" : 0, @@ -147,7 +147,6 @@ TEST_F(TagTest, TestJsonizable) { /*commit_user=*/"0e4d92f7-53b0-40d6-a7c0-102bf3801e6a", /*commit_identifier=*/9223372036854775807ll, /*commit_kind=*/Snapshot::CommitKind::Overwrite(), /*time_millis=*/1711692199281ll, - /*log_offsets=*/std::map(), /*total_record_count=*/3, /*delta_record_count=*/3, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/std::nullopt, @@ -195,10 +194,6 @@ TEST_F(TagTest, TestSerializeAndDeserialize) { "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, @@ -226,10 +221,6 @@ TEST_F(TagTest, TestSerializeAndDeserialize) { "commitIdentifier" : 12, "commitKind" : "APPEND", "timeMillis" : 1749724197266, - "logOffsets" : { - "0" : 1, - "1" : 3 - }, "totalRecordCount" : 1024, "deltaRecordCount" : 4096, "watermark" : 1749724196266, diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp index 208824e9..0b5158dc 100644 --- a/test/inte/append_compaction_inte_test.cpp +++ b/test/inte/append_compaction_inte_test.cpp @@ -83,8 +83,8 @@ class AppendCompactionInteTest : public testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); std::vector datas_2; datas_2.push_back( @@ -103,8 +103,8 @@ class AppendCompactionInteTest : public testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); std::vector datas_3; datas_3.push_back( @@ -117,8 +117,8 @@ class AppendCompactionInteTest : public testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot3, helper->LatestSnapshot()); ASSERT_TRUE(snapshot3); ASSERT_EQ(3, snapshot3.value().Id()); - ASSERT_EQ(10, snapshot3.value().TotalRecordCount().value()); - ASSERT_EQ(1, snapshot3.value().DeltaRecordCount().value()); + ASSERT_EQ(10, snapshot3.value().TotalRecordCount()); + ASSERT_EQ(1, snapshot3.value().DeltaRecordCount()); // @note: for append-only tables in Spark, native row-level deletes aren't supported during // writing. Instead, deletions are expressed by committing a Deletion Vector (DV) file @@ -208,8 +208,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) { ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(11, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(0, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(11, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(0, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -535,8 +535,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithDv) ASSERT_OK(helper2->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper2->LatestSnapshot()); ASSERT_EQ(6, snapshot5.value().Id()); - ASSERT_EQ(8, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(-3, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(8, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(-3, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper2->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -617,8 +617,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteBestEffortCompaction) ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(11, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(0, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(11, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(0, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -708,8 +708,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteCompactionWithExterna ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(11, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(0, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(11, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(0, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index 49d2af3c..b59969ea 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -344,8 +344,8 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_TRUE(snapshot_exist); ASSERT_OK_AND_ASSIGN(Snapshot snapshot_3, commit_impl->snapshot_manager_->LoadSnapshot(3)); ASSERT_EQ(30, snapshot_3.Watermark().value()); - ASSERT_EQ(-7, snapshot_3.DeltaRecordCount().value()); - ASSERT_EQ(2, snapshot_3.TotalRecordCount().value()); + ASSERT_EQ(-7, snapshot_3.DeltaRecordCount()); + ASSERT_EQ(2, snapshot_3.TotalRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot_3.GetCommitKind()); ASSERT_EQ(2, snapshot_3.CommitIdentifier()); ASSERT_OK_AND_ASSIGN(bool f1_10_bucket_0_exist, @@ -407,7 +407,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_EQ(3u, manifests[1].NumAddedFiles()); } -TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { +TEST_F(CleanInteTest, DISABLED_TestDropPartitionAndExpireSnapshotWithIOException) { auto string_field = arrow::field("f0", arrow::utf8()); auto int_field = arrow::field("f1", arrow::int32()); auto int_field1 = arrow::field("f2", arrow::int32()); @@ -508,8 +508,8 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { ASSERT_OK_AND_ASSIGN(snapshot_exist, commit_impl->snapshot_manager_->SnapshotExists(3)); ASSERT_TRUE(snapshot_exist); ASSERT_OK_AND_ASSIGN(Snapshot snapshot_3, commit_impl->snapshot_manager_->LoadSnapshot(3)); - ASSERT_EQ(-7, snapshot_3.DeltaRecordCount().value()); - ASSERT_EQ(2, snapshot_3.TotalRecordCount().value()); + ASSERT_EQ(-7, snapshot_3.DeltaRecordCount()); + ASSERT_EQ(2, snapshot_3.TotalRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot_3.GetCommitKind()); ASSERT_EQ(2, snapshot_3.CommitIdentifier()); io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index 0716fd5f..a4fa11a3 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -138,6 +138,19 @@ class DataEvolutionTableTest : public ::testing::Test, return file_store_commit->Commit(commit_msgs); } + Status CommitWithRowIdCheckFromSnapshot( + const std::string& table_path, + const std::vector>& commit_msgs, + std::optional row_id_check_from_snapshot) const { + CommitContextBuilder commit_builder(table_path, "commit_user_1"); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, + commit_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, + FileStoreCommit::Create(std::move(commit_context))); + file_store_commit->RowIdCheckConflict(row_id_check_from_snapshot); + return file_store_commit->Commit(commit_msgs); + } + Status ScanAndRead(const std::string& table_path, const std::vector& read_schema, const std::shared_ptr& expected_array, const std::shared_ptr& predicate = nullptr, @@ -345,6 +358,46 @@ TEST_P(DataEvolutionTableTest, TestBasic) { } } +TEST_P(DataEvolutionTableTest, TestCommitConflictOnOverlappedRowIdAndWriteColumns) { + CreateTable(); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // Snapshot 1: initialize row id range [0, 0]. + std::vector init_write_cols = {"f0", "f1", "f2"}; + auto init_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ + [1, "a", "b"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto init_msgs, WriteArray(table_path, init_write_cols, init_array)); + ASSERT_OK(Commit(table_path, init_msgs)); + + // Snapshot 2: update f2 at row id 0. + std::vector write_cols = {"f2"}; + auto src_array_1 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[2]}), R"([ + ["c"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, WriteArray(table_path, write_cols, src_array_1)); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs_1); + ASSERT_OK(Commit(table_path, commit_msgs_1)); + + // Snapshot 3 attempt: update f2 at row id 0 again, and check history from snapshot 1. + // This should conflict with snapshot 2 because row-id range and write columns overlap. + auto src_array_2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[2]}), R"([ + ["d"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, WriteArray(table_path, write_cols, src_array_2)); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs_2); + ASSERT_NOK_WITH_MSG( + CommitWithRowIdCheckFromSnapshot(table_path, commit_msgs_2, + /*row_id_check_from_snapshot=*/1), + "multiple MERGE INTO operations have encountered conflicts while checking row-id history"); +} + TEST_P(DataEvolutionTableTest, TestMultipleAppends) { CreateTable(); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index d3171bdd..aa90d626 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -120,8 +120,8 @@ class PkCompactionInteTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); std::vector datas_2; datas_2.push_back( @@ -140,8 +140,8 @@ class PkCompactionInteTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); std::vector datas_3; datas_3.push_back( @@ -154,8 +154,8 @@ class PkCompactionInteTest : public ::testing::Test, ASSERT_OK_AND_ASSIGN(std::optional snapshot3, helper->LatestSnapshot()); ASSERT_TRUE(snapshot3); ASSERT_EQ(3, snapshot3.value().Id()); - ASSERT_EQ(10, snapshot3.value().TotalRecordCount().value()); - ASSERT_EQ(1, snapshot3.value().DeltaRecordCount().value()); + ASSERT_EQ(10, snapshot3.value().TotalRecordCount()); + ASSERT_EQ(1, snapshot3.value().DeltaRecordCount()); } Result>> WriteArray( @@ -2802,8 +2802,8 @@ TEST_P(PkCompactionInteTest, TestKeyValueTableStreamWriteFullCompaction) { ASSERT_OK(helper->commit_->Commit(commit_messages, commit_identifier)); ASSERT_OK_AND_ASSIGN(std::optional snapshot5, helper->LatestSnapshot()); ASSERT_EQ(5, snapshot5.value().Id()); - ASSERT_EQ(9, snapshot5.value().TotalRecordCount().value()); - ASSERT_EQ(-2, snapshot5.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot5.value().TotalRecordCount()); + ASSERT_EQ(-2, snapshot5.value().DeltaRecordCount()); ASSERT_EQ(Snapshot::CommitKind::Compact(), snapshot5.value().GetCommitKind()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 813e9f16..640e25c7 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -1054,7 +1054,7 @@ TEST(SystemTableReadInteTest, TestReadTagBranchAndConsumerSystemTables) { Timestamp tag_commit_time, DateTimeUtils::ToLocalTimestamp(Timestamp::FromEpochMillis(tag.TimeMillis()))); ASSERT_EQ(tag_commit_time_array->Value(0), tag_commit_time.GetMillisecond()); - ASSERT_EQ(tag_record_count_array->Value(0), tag.TotalRecordCount().value()); + ASSERT_EQ(tag_record_count_array->Value(0), tag.TotalRecordCount()); ASSERT_FALSE(tag_create_time_array->IsNull(0)); ASSERT_EQ(tag_create_time_array->Value(0), 1770185290000); ASSERT_EQ(tag_time_retained_array->GetString(0), "3.000000"); diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index 6529fc11..f58e105d 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -461,8 +461,8 @@ TEST_P(WriteInteTest, TestAppendTableBatchWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -564,8 +564,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -634,8 +634,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithOneBucket) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(7, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(3, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(7, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(3, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); @@ -704,8 +704,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(8, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(8, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(8, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(8, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -769,8 +769,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithPartitionAndMultiBuckets) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(16, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(8, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(16, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(8, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 3); @@ -879,8 +879,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(6, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(6, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(6, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(6, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -942,8 +942,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(10, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(10, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); @@ -1080,8 +1080,8 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); // round 1 read arrow::FieldVector fields_with_row_kind = fields; @@ -1210,8 +1210,8 @@ TEST_P(WriteInteTest, TestPkTableStreamWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); // round 2 read ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); @@ -1355,8 +1355,8 @@ TEST_P(WriteInteTest, TestPkTableBatchWrite) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_1, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -1499,8 +1499,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); // round1 read arrow::FieldVector fields_with_row_kind = fields; @@ -1603,8 +1603,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithNoPartitionKey) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); // round2 read ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); @@ -1712,8 +1712,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -1786,8 +1786,8 @@ TEST_P(WriteInteTest, TestPkTableWriteWithComplexType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(9, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); @@ -1844,8 +1844,8 @@ TEST_P(WriteInteTest, TestPkTableForceLookup) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); // read arrow::FieldVector fields_with_row_kind = fields; @@ -1909,8 +1909,8 @@ TEST_P(WriteInteTest, TestPkTableEnableDeletionVector) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); // read arrow::FieldVector fields_with_row_kind = fields; @@ -2795,8 +2795,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(4, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(4, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(4, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(4, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -2864,8 +2864,8 @@ TEST_P(WriteInteTest, TestAppendTableStreamWriteWithExternalPath) { ASSERT_OK_AND_ASSIGN(std::optional snapshot2, helper->LatestSnapshot()); ASSERT_TRUE(snapshot2); ASSERT_EQ(2, snapshot2.value().Id()); - ASSERT_EQ(7, snapshot2.value().TotalRecordCount().value()); - ASSERT_EQ(3, snapshot2.value().DeltaRecordCount().value()); + ASSERT_EQ(7, snapshot2.value().TotalRecordCount()); + ASSERT_EQ(3, snapshot2.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits_2, helper->Scan()); ASSERT_EQ(data_splits_2.size(), 1); std::string expected_data_2 = @@ -3468,8 +3468,8 @@ TEST_P(WriteInteTest, TestPkTablePostponeBucket) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(5, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(5, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(5, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(5, snapshot1.value().DeltaRecordCount()); ASSERT_OK_AND_ASSIGN(std::vector> data_splits, helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); @@ -3878,8 +3878,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithBlobType) { ASSERT_OK_AND_ASSIGN(std::optional snapshot, helper->LatestSnapshot()); ASSERT_TRUE(snapshot); ASSERT_EQ(1, snapshot.value().Id()); - ASSERT_EQ(8, snapshot.value().TotalRecordCount().value()); - ASSERT_EQ(8, snapshot.value().DeltaRecordCount().value()); + ASSERT_EQ(8, snapshot.value().TotalRecordCount()); + ASSERT_EQ(8, snapshot.value().DeltaRecordCount()); ASSERT_EQ(4, snapshot.value().NextRowId().value()); // check data file meta after commit @@ -3950,8 +3950,8 @@ TEST_P(WriteInteTest, TestAppendTableWithDateFieldAsPartitionField) { ASSERT_OK_AND_ASSIGN(std::optional snapshot1, helper->LatestSnapshot()); ASSERT_TRUE(snapshot1); ASSERT_EQ(1, snapshot1.value().Id()); - ASSERT_EQ(2, snapshot1.value().TotalRecordCount().value()); - ASSERT_EQ(2, snapshot1.value().DeltaRecordCount().value()); + ASSERT_EQ(2, snapshot1.value().TotalRecordCount()); + ASSERT_EQ(2, snapshot1.value().DeltaRecordCount()); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -4713,8 +4713,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteWithMultipleBlobFields) { ASSERT_TRUE(snapshot); ASSERT_EQ(1, snapshot.value().Id()); // 3 rows * 3 files (1 main + 1 blob1 + 1 blob2) = 9 total records - ASSERT_EQ(9, snapshot.value().TotalRecordCount().value()); - ASSERT_EQ(9, snapshot.value().DeltaRecordCount().value()); + ASSERT_EQ(9, snapshot.value().TotalRecordCount()); + ASSERT_EQ(9, snapshot.value().DeltaRecordCount()); ASSERT_EQ(3, snapshot.value().NextRowId().value()); // Check data file meta after commit From 33c52a53314a1b3b0d2f380486d69b423febb5af Mon Sep 17 00:00:00 2001 From: gripleaf <425797155@qq.com> Date: Thu, 16 Jul 2026 13:07:06 +0800 Subject: [PATCH 097/138] feat: add read-optimized system table --- docs/source/user_guide.rst | 1 + docs/source/user_guide/system_tables.rst | 70 ++++ src/paimon/CMakeLists.txt | 1 + .../table/source/data_table_batch_scan.cpp | 11 +- .../core/table/source/data_table_batch_scan.h | 2 +- .../source/read_optimized_scan_options.h | 27 ++ src/paimon/core/table/source/table_scan.cpp | 15 +- .../core/table/source/table_scan_test.cpp | 14 + .../system/read_optimized_system_table.cpp | 131 +++++++ .../system/read_optimized_system_table.h | 54 +++ src/paimon/core/table/system/system_table.cpp | 9 + .../core/table/system/system_table_test.cpp | 34 ++ src/paimon/testing/utils/test_helper.h | 9 +- test/inte/read_inte_test.cpp | 370 +++++++++++++++++- 14 files changed, 735 insertions(+), 13 deletions(-) create mode 100644 docs/source/user_guide/system_tables.rst create mode 100644 src/paimon/core/table/source/read_optimized_scan_options.h create mode 100644 src/paimon/core/table/system/read_optimized_system_table.cpp create mode 100644 src/paimon/core/table/system/read_optimized_system_table.h diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst index dc444aa1..f0455343 100644 --- a/docs/source/user_guide.rst +++ b/docs/source/user_guide.rst @@ -33,6 +33,7 @@ User Guide user_guide/data_types user_guide/primary_key_table user_guide/append_only_table + user_guide/system_tables user_guide/write user_guide/commit user_guide/compaction diff --git a/docs/source/user_guide/system_tables.rst b/docs/source/user_guide/system_tables.rst new file mode 100644 index 00000000..6ac01882 --- /dev/null +++ b/docs/source/user_guide/system_tables.rst @@ -0,0 +1,70 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +System Tables +============= + +Paimon C++ supports reading system tables by appending a system table suffix to +the data table path. For example, ``/warehouse/db.db/orders$snapshots`` reads +the snapshots system table for ``orders``. + +Branch-qualified paths are also supported. For example, +``/warehouse/db.db/orders$branch_audit$files`` reads the files system table from +the ``audit`` branch. + +Read-Optimized System Table +--------------------------- + +The read-optimized system table is addressed by the ``$ro`` suffix: + +.. code-block:: text + + /warehouse/db.db/orders$ro + +For primary-key tables, ``$ro`` only plans data files from the highest LSM +level, which is the level produced by full compaction. This avoids merging data +from multiple LSM levels during query planning and reading. The tradeoff is that +the result may lag behind the latest committed data until a full compaction +publishes the newest records into the highest level. + +This stale view is not guaranteed to correspond to any single historical table +snapshot. Full compaction may finish independently for different buckets. When +``$ro`` scans the latest snapshot, the selected highest-level files can +therefore combine bucket states produced by full-compaction commits at different +snapshot IDs. + +For primary-key tables, ``$ro`` also enables value-stats filtering, so file-level +pruning of the reader predicate can be more aggressive than the base table. + +For append-only tables, ``$ro`` has the same read behavior as the base table, +including streaming reads. + +Limitations +~~~~~~~~~~~ + +- Primary-key ``$ro`` scans are batch-only. Streaming scans are not supported. +- Freshness depends on full compaction frequency, and the result may not match + any single historical snapshot across buckets. +- Primary-key tables in bucket-unaware mode are not supported by the current C++ + scan path. + +Typical Usage +~~~~~~~~~~~~~ + +Use ``$ro`` for OLAP or batch workloads that can tolerate stale results and +prefer reading compacted files directly. Use the base table path when the query +must see the latest committed data. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 2fbbb63e..aa587590 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -358,6 +358,7 @@ set(PAIMON_CORE_SRCS core/table/system/binlog_system_table.cpp core/table/system/in_memory_system_table.cpp core/table/system/metadata_system_tables.cpp + core/table/system/read_optimized_system_table.cpp core/table/system/system_table.cpp core/table/system/system_table_scan.cpp core/table/system/system_table_schema.cpp diff --git a/src/paimon/core/table/source/data_table_batch_scan.cpp b/src/paimon/core/table/source/data_table_batch_scan.cpp index 5ca4b2ae..d45a140c 100644 --- a/src/paimon/core/table/source/data_table_batch_scan.cpp +++ b/src/paimon/core/table/source/data_table_batch_scan.cpp @@ -36,10 +36,15 @@ class DataSplit; DataTableBatchScan::DataTableBatchScan(bool pk_table, const CoreOptions& core_options, const std::shared_ptr& snapshot_reader, - std::optional push_down_limit) + bool read_optimized, std::optional push_down_limit) : AbstractTableScan(core_options, snapshot_reader), push_down_limit_(push_down_limit) { - if (pk_table && (core_options.DeletionVectorsEnabled() || - core_options.GetMergeEngine() == MergeEngine::FIRST_ROW)) { + if (pk_table && read_optimized) { + int32_t top_level = core_options.GetNumLevels() - 1; + snapshot_reader_->WithLevelFilter( + [top_level](int32_t level) -> bool { return level == top_level; }); + snapshot_reader_->EnableValueFilter(); + } else if (pk_table && (core_options.DeletionVectorsEnabled() || + core_options.GetMergeEngine() == MergeEngine::FIRST_ROW)) { auto level_filter = [](int32_t level) -> bool { return level > 0; }; snapshot_reader_->WithLevelFilter(level_filter); snapshot_reader_->EnableValueFilter(); diff --git a/src/paimon/core/table/source/data_table_batch_scan.h b/src/paimon/core/table/source/data_table_batch_scan.h index 7f6dbb15..0729bd69 100644 --- a/src/paimon/core/table/source/data_table_batch_scan.h +++ b/src/paimon/core/table/source/data_table_batch_scan.h @@ -35,7 +35,7 @@ class SnapshotReader; class DataTableBatchScan : public AbstractTableScan { public: DataTableBatchScan(bool pk_table, const CoreOptions& core_options, - const std::shared_ptr& snapshot_reader, + const std::shared_ptr& snapshot_reader, bool read_optimized, std::optional push_down_limit); Result> CreatePlan() override; diff --git a/src/paimon/core/table/source/read_optimized_scan_options.h b/src/paimon/core/table/source/read_optimized_scan_options.h new file mode 100644 index 00000000..150a5ea3 --- /dev/null +++ b/src/paimon/core/table/source/read_optimized_scan_options.h @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +namespace paimon { + +/// Internal scan option used by `T$ro` to request top-level-only planning. +inline constexpr char kReadOptimizedScanOption[] = "__paimon.internal.read-optimized"; + +} // namespace paimon diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index f85db826..6b41ee24 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -30,6 +30,7 @@ #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/options_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/index/index_file_handler.h" #include "paimon/core/manifest/index_manifest_file.h" @@ -50,6 +51,7 @@ #include "paimon/core/table/source/data_table_batch_scan.h" #include "paimon/core/table/source/data_table_stream_scan.h" #include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/read_optimized_scan_options.h" #include "paimon/core/table/source/snapshot/snapshot_reader.h" #include "paimon/core/table/source/split_generator.h" #include "paimon/core/table/system/system_table.h" @@ -228,6 +230,9 @@ Result> NewDataTableScan(const std::shared_ptr(context->GetOptions(), + kReadOptimizedScanOption, false)); // merge options auto options = table_schema->Options(); for (const auto& [key, value] : context->GetOptions()) { @@ -283,12 +288,16 @@ Result> NewDataTableScan(const std::shared_ptrGetMemoryPool())); auto snapshot_reader = std::make_shared( file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); + const bool pk_table = !table_schema->PrimaryKeys().empty(); + if (read_optimized && pk_table && context->IsStreamingMode()) { + return Status::NotImplemented( + "read-optimized system table does not support streaming scan for primary key table"); + } if (context->IsStreamingMode()) { return std::make_unique(core_options, snapshot_reader); } - auto batch_scan = - std::make_unique(/*pk_table=*/!table_schema->PrimaryKeys().empty(), - core_options, snapshot_reader, context->GetLimit()); + auto batch_scan = std::make_unique( + /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit()); if (!core_options.DataEvolutionEnabled()) { return batch_scan; } diff --git a/src/paimon/core/table/source/table_scan_test.cpp b/src/paimon/core/table/source/table_scan_test.cpp index ba76de4c..358dfd46 100644 --- a/src/paimon/core/table/source/table_scan_test.cpp +++ b/src/paimon/core/table/source/table_scan_test.cpp @@ -19,6 +19,7 @@ #include "paimon/table/source/table_scan.h" +#include #include #include #include @@ -62,4 +63,17 @@ TEST(TableScanTest, TestPkSchemaEvolutionScan) { ASSERT_FALSE(plan->Splits().empty()); } +TEST(TableScanTest, TestReadOptimizedPrimaryKeyStreamingScanUnsupported) { + std::string path = paimon::test::GetDataDir() + + "/orc/pk_table_with_alter_table.db/pk_table_with_alter_table$ro"; + ScanContextBuilder builder(path); + builder.AddOption(Options::FILE_FORMAT, "orc"); + builder.WithStreamingMode(true); + ASSERT_OK_AND_ASSIGN(std::unique_ptr context, builder.Finish()); + + ASSERT_NOK_WITH_MSG(TableScan::Create(std::move(context)), + "read-optimized system table does not support streaming scan for primary " + "key table"); +} + } // namespace paimon::test diff --git a/src/paimon/core/table/system/read_optimized_system_table.cpp b/src/paimon/core/table/system/read_optimized_system_table.cpp new file mode 100644 index 00000000..516861cb --- /dev/null +++ b/src/paimon/core/table/system/read_optimized_system_table.cpp @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/system/read_optimized_system_table.h" + +#include +#include +#include + +#include "arrow/c/bridge.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/defs.h" +#include "paimon/read_context.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" + +namespace paimon { + +ReadOptimizedSystemTable::ReadOptimizedSystemTable(std::string table_path, + std::shared_ptr table_schema, + std::map options) + : table_path_(std::move(table_path)), + table_schema_(std::move(table_schema)), + options_(std::move(options)) {} + +std::string ReadOptimizedSystemTable::Name() const { + return kName; +} + +Result> ReadOptimizedSystemTable::ArrowSchema() const { + return DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); +} + +std::map ReadOptimizedSystemTable::ReadOptimizedOptions() const { + auto options = options_; + options[kReadOptimizedScanOption] = "true"; + return options; +} + +Result> ReadOptimizedSystemTable::NewScan( + const std::shared_ptr& context) const { + auto options = ReadOptimizedOptions(); + ScanContextBuilder builder(table_path_); + builder.SetOptions(options) + .WithStreamingMode(context->IsStreamingMode()) + .WithMemoryPool(context->GetMemoryPool()) + .WithExecutor(context->GetExecutor()) + .WithFileSystem(context->GetSpecificFileSystem()) + .WithCache(context->GetCache()); + if (context->GetLimit().has_value()) { + builder.SetLimit(context->GetLimit().value()); + } + if (context->GetScanFilters()) { + if (context->GetScanFilters()->GetBucketFilter().has_value()) { + builder.SetBucketFilter(context->GetScanFilters()->GetBucketFilter().value()); + } + builder.SetPartitionFilter(context->GetScanFilters()->GetPartitionFilters()); + builder.SetPredicate(context->GetScanFilters()->GetPredicate()); + } + if (context->GetGlobalIndexResult()) { + builder.SetGlobalIndexResult(context->GetGlobalIndexResult()); + } + if (context->GetSpecificTableSchema().has_value()) { + builder.SetTableSchema(context->GetSpecificTableSchema().value()); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish()); + return TableScan::Create(std::move(base_context)); +} + +Result> ReadOptimizedSystemTable::NewRead( + const std::shared_ptr& context) const { + auto options = options_; + std::string branch = context->GetBranch(); + // SystemTableLoader injects Options::BRANCH when parsing paths such as `T$branch_dev$ro`. + auto branch_iter = options.find(Options::BRANCH); + if (branch_iter != options.end()) { + branch = branch_iter->second; + } + ReadContextBuilder builder(table_path_); + builder.SetOptions(options) + .WithBranch(branch) + .SetPredicate(context->GetPredicate()) + .EnablePredicateFilter(context->EnablePredicateFilter()) + .EnablePrefetch(context->EnablePrefetch()) + .SetPrefetchBatchCount(context->GetPrefetchBatchCount()) + .SetPrefetchMaxParallelNum(context->GetPrefetchMaxParallelNum()) + .EnableMultiThreadRowToBatch(context->EnableMultiThreadRowToBatch()) + .SetRowToBatchThreadNumber(context->GetRowToBatchThreadNumber()) + .WithMemoryPool(context->GetMemoryPool()) + .WithExecutor(context->GetExecutor()) + .WithFileSystem(context->GetSpecificFileSystem()) + .WithFileSystemSchemeToIdentifierMap(context->GetFileSystemSchemeToIdentifierMap()) + .SetPrefetchCacheMode(context->GetPrefetchCacheMode()) + .WithCacheConfig(context->GetCacheConfig()) + .WithCache(context->GetCache()) + .SetReadFieldNames(context->GetReadFieldNames()) + .SetReadFieldIds(context->GetReadFieldIds()); + if (context->HasReadSchema()) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema, + arrow::ImportSchema(context->GetReadSchema())); + auto c_read_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, c_read_schema.get())); + builder.SetReadSchema(std::move(c_read_schema)); + } + if (context->GetSpecificTableSchema().has_value()) { + builder.SetTableSchema(context->GetSpecificTableSchema().value()); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr base_context, builder.Finish()); + return TableRead::Create(std::move(base_context)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/system/read_optimized_system_table.h b/src/paimon/core/table/system/read_optimized_system_table.h new file mode 100644 index 00000000..14fadbfd --- /dev/null +++ b/src/paimon/core/table/system/read_optimized_system_table.h @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/core/table/system/system_table.h" + +namespace paimon { +class TableSchema; + +/// System table for `T$ro`, exposing read-optimized data. +class ReadOptimizedSystemTable : public SystemTable { + public: + static constexpr const char* kName = "ro"; + + ReadOptimizedSystemTable(std::string table_path, std::shared_ptr table_schema, + std::map options); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> NewScan( + const std::shared_ptr& context) const override; + Result> NewRead( + const std::shared_ptr& context) const override; + + private: + std::map ReadOptimizedOptions() const; + + std::string table_path_; + std::shared_ptr table_schema_; + std::map options_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 95f3eedf..d2ca7ce8 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -34,6 +34,7 @@ #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" #include "paimon/core/table/system/metadata_system_tables.h" +#include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/status.h" @@ -89,6 +90,14 @@ const std::vector& SystemTableRegistry() { return std::make_shared( fs, table_path, table_schema, MergeOptions(table_schema, dynamic_options)); }}, + {ReadOptimizedSystemTable::kName, + [](const std::shared_ptr& /*fs*/, const std::string& table_path, + const std::shared_ptr& table_schema, + const std::map& dynamic_options) + -> Result> { + return std::make_shared( + table_path, table_schema, MergeOptions(table_schema, dynamic_options)); + }}, {SnapshotsSystemTable::kName, [](const std::shared_ptr& fs, const std::string& table_path, const std::shared_ptr& table_schema, diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index eb4bb139..caffe92b 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -17,15 +17,20 @@ * under the License. */ +#include "paimon/core/table/system/system_table.h" + #include #include +#include #include +#include #include "arrow/api.h" #include "gtest/gtest.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" +#include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" #include "paimon/result.h" @@ -65,4 +70,33 @@ TEST(SystemTableTest, TestChangelogArrowSchemaReturnsInvalidOptions) { "Invalid Config [table-read.sequence-number.enabled: invalid]"); } +TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) { + ASSERT_TRUE(SystemTableLoader::IsSupported(ReadOptimizedSystemTable::kName)); + + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + CreateTableSchemaForTest(options)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr system_table, + SystemTableLoader::Load(ReadOptimizedSystemTable::kName, /*fs=*/nullptr, + "/tmp/table", table_schema, + /*dynamic_options=*/{})); + ASSERT_EQ(system_table->Name(), ReadOptimizedSystemTable::kName); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr arrow_schema, system_table->ArrowSchema()); + ASSERT_EQ(arrow_schema->field_names(), (std::vector{"pk", "v"})); + ASSERT_EQ(arrow_schema->field(0)->type()->id(), arrow::Type::STRING); + ASSERT_EQ(arrow_schema->field(1)->type()->id(), arrow::Type::INT32); +} + +TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { + ASSERT_OK_AND_ASSIGN(std::optional parsed, + SystemTableLoader::TryParsePath("/tmp/db.db/t$branch_audit$ro")); + ASSERT_TRUE(parsed.has_value()); + ASSERT_EQ(parsed->table_path, "/tmp/db.db/t"); + ASSERT_TRUE(parsed->branch.has_value()); + ASSERT_EQ(parsed->branch.value(), "audit"); + ASSERT_EQ(parsed->system_table_name, ReadOptimizedSystemTable::kName); +} + } // namespace paimon::test diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index 1af886e3..fb77985c 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -59,7 +59,7 @@ class TestHelper { const std::vector& partition_keys, const std::vector& primary_keys, const std::map& options, bool is_streaming_mode, - bool ignore_if_exists = false) { + bool ignore_if_exists = false, const std::string& temp_directory = "") { // only for test && only check the key auto new_options = options; new_options["enable-object-store-catalog-in-inte-test"] = ""; @@ -72,12 +72,12 @@ class TestHelper { partition_keys, primary_keys, new_options, ignore_if_exists)); std::string table_path = PathUtil::JoinPath(root_path, "foo.db/bar"); - return Create(table_path, new_options, is_streaming_mode); + return Create(table_path, new_options, is_streaming_mode, temp_directory); } static Result> Create( const std::string& table_path, const std::map& options, - bool is_streaming_mode) { + bool is_streaming_mode, const std::string& temp_directory = "") { std::string file_system_identifier = "local"; auto fs_iter = options.find(Options::FILE_SYSTEM); if (fs_iter != options.end()) { @@ -87,6 +87,9 @@ class TestHelper { FileSystemFactory::Get(file_system_identifier, table_path, options)); std::string commit_user = "commit_user"; WriteContextBuilder context_builder(table_path, commit_user); + if (!temp_directory.empty()) { + context_builder.WithTempDirectory(temp_directory); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write_context, context_builder.SetOptions(options) .WithStreamingMode(is_streaming_mode) diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 640e25c7..1fd010ae 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -270,10 +270,15 @@ std::vector StructFieldNames(const std::shared_ptrtype()->fields())->field_names(); } -Result ReadSystemTable(const std::string& system_table_path, - const std::map& options) { +Result ReadSystemTable( + const std::string& system_table_path, const std::map& options, + bool streaming_mode = false, const std::shared_ptr& predicate = nullptr, + const std::vector& read_field_names = {}) { ScanContextBuilder scan_context_builder(system_table_path); - scan_context_builder.SetOptions(options); + scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode); + if (predicate) { + scan_context_builder.SetPredicate(predicate); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, @@ -282,6 +287,12 @@ Result ReadSystemTable(const std::string& system_table_pa ReadContextBuilder read_context_builder(system_table_path); read_context_builder.SetOptions(options); + if (predicate) { + read_context_builder.SetPredicate(predicate); + } + if (!read_field_names.empty()) { + read_context_builder.SetReadFieldNames(read_field_names); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, @@ -293,6 +304,17 @@ Result ReadSystemTable(const std::string& system_table_pa return SystemTableReadResult(std::move(batch_reader), result); } +Status WriteAndFullCompact(std::unique_ptr&& batch, int64_t commit_identifier, + TestHelper* helper) { + PAIMON_RETURN_NOT_OK(helper->write_->Write(std::move(batch))); + PAIMON_RETURN_NOT_OK( + helper->write_->Compact(/*partition=*/{}, /*bucket=*/0, /*full_compaction=*/true)); + PAIMON_ASSIGN_OR_RAISE( + std::vector> commit_messages, + helper->write_->PrepareCommit(/*wait_compaction=*/true, commit_identifier)); + return helper->commit_->Commit(commit_messages, commit_identifier); +} + void AssertStructArrayEqualsJson(const std::shared_ptr& actual, const std::string& expected_json) { ASSERT_TRUE(actual); @@ -302,6 +324,18 @@ void AssertStructArrayEqualsJson(const std::shared_ptr& actu << "expected: " << expected->ToString() << "\nactual: " << actual->ToString(); } +Result CountDataFiles(const std::vector>& splits) { + int64_t file_count = 0; + for (const auto& split : splits) { + auto data_split = std::dynamic_pointer_cast(split); + if (!data_split) { + return Status::Invalid("expected data split"); + } + file_count += data_split->GetFileList().size(); + } + return file_count; +} + } // namespace std::vector PrepareTestParam() { @@ -785,6 +819,336 @@ TEST(SystemTableReadInteTest, TestReadMetadataSystemTables) { ASSERT_FALSE(creation_time_array->IsNull(0)); } +TEST(SystemTableReadInteTest, TestReadOptimizedSystemTable) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::NUM_LEVELS, "3"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + auto row_type = arrow::struct_(fields); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_1, + TestHelper::MakeRecordBatch(row_type, R"([[1, 10], [2, 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch_1), /*commit_identifier=*/0, helper.get())); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult compacted_result, + ReadSystemTable(table_path + "$ro", options)); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected_compacted; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 10], [0, 2, 20]])"}, &expected_compacted) + .ok()); + ASSERT_TRUE(compacted_result.array->Equals(expected_compacted)) + << compacted_result.array->ToString(); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_2, + TestHelper::MakeRecordBatch(row_type, R"([[1, 11], [3, 30]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch_2), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(SystemTableReadResult stale_result, + ReadSystemTable(table_path + "$ro", options)); + ASSERT_TRUE(stale_result.array->Equals(expected_compacted)) << stale_result.array->ToString(); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_3, + TestHelper::MakeRecordBatch(row_type, R"([[2, 21], [3, 31]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch_3), /*commit_identifier=*/2, helper.get())); + ASSERT_OK_AND_ASSIGN(SystemTableReadResult refreshed_result, + ReadSystemTable(table_path + "$ro", options)); + std::shared_ptr expected_refreshed; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 11], [0, 2, 21], [0, 3, 31]])"}, &expected_refreshed) + .ok()); + ASSERT_TRUE(refreshed_result.array->Equals(expected_refreshed)) + << refreshed_result.array->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedAppendOnlySystemTableWithStreamingScan) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "k"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[1, 10], [2, 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult result, + ReadSystemTable(table_path + "$ro", options, /*streaming_mode=*/true)); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 10], [0, 2, 20]])"}, &expected) + .ok()); + ASSERT_TRUE(result.array->Equals(expected)) << result.array->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedPrimaryKeyProjectionAndPredicatePushdown) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + arrow::field("extra", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "k"}, + {Options::WRITE_BATCH_SIZE, "1"}, + {"parquet.page.size", "1"}, + {"parquet.enable-dictionary", "false"}, + {"parquet.write.enable-page-index", "true"}, + {"parquet.write.max-row-group-length", "1"}, + {"parquet.read.enable-page-index-filter", "true"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + auto row_type = arrow::struct_(fields); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_1, + TestHelper::MakeRecordBatch(row_type, R"([[1, 10, 100], [2, 20, 200], [3, 30, 300]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch_1), /*commit_identifier=*/0, helper.get())); + + std::shared_ptr predicate = + PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"k", FieldType::INT, Literal(2)); + + ScanContextBuilder ro_scan_context_builder(table_path + "$ro"); + ro_scan_context_builder.SetOptions(options).SetPredicate(predicate); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_scan_context, + ro_scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr ro_table_scan, + TableScan::Create(std::move(ro_scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ro_plan, ro_table_scan->CreatePlan()); + ASSERT_OK_AND_ASSIGN(int64_t ro_file_count, CountDataFiles(ro_plan->Splits())); + ASSERT_EQ(ro_file_count, 1); + + ReadContextBuilder read_context_builder(table_path + "$ro"); + // Do not enable row-level predicate filtering: the result must come from the file reader's + // row-group/page predicate pushdown. + read_context_builder.SetOptions(options).SetPredicate(predicate).SetReadFieldNames({"k", "v"}); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(ro_plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(batch_reader.get())); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(expected_type, {R"([[0, 2, 20]])"}, + &expected) + .ok()); + ASSERT_TRUE(result->Equals(expected)) << result->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableNestedProjection) { + auto payload_type = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("payload", payload_type), + arrow::field("extra", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "k"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), + R"([[1, [10, "x"], 100], [2, [20, "y"], 200]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); + + ScanContextBuilder scan_context_builder(table_path + "$ro"); + scan_context_builder.SetOptions(options); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, table_scan->CreatePlan()); + + auto projected_schema = arrow::schema({ + arrow::field("k", arrow::int32()), + arrow::field("payload", arrow::struct_({arrow::field("a", arrow::int32())})), + }); + auto c_projected_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*projected_schema, c_projected_schema.get()).ok()); + ReadContextBuilder read_context_builder(table_path + "$ro"); + read_context_builder.SetOptions(options).SetReadSchema(std::move(c_projected_schema)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_read, + TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + ReadResultCollector::CollectResult(batch_reader.get())); + + std::shared_ptr expected_type = arrow::struct_({ + arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), + arrow::field("payload", arrow::struct_({arrow::field("a", arrow::int32())})), + }); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, [10]], [0, 2, [20]]])"}, &expected) + .ok()); + ASSERT_TRUE(result->Equals(expected)) << result->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableWithBranch) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, + {Options::BUCKET, "1"}, + {Options::NUM_LEVELS, "3"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN(std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true)); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + auto row_type = arrow::struct_(fields); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr branch_batch, + TestHelper::MakeRecordBatch(row_type, R"([[1, 10], [2, 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(branch_batch), /*commit_identifier=*/0, helper.get())); + + std::string branch_path = PathUtil::JoinPath(table_path, "branch/branch-rt"); + std::filesystem::create_directories(branch_path); + ASSERT_TRUE(TestUtil::CopyDirectory(PathUtil::JoinPath(table_path, "schema"), + PathUtil::JoinPath(branch_path, "schema"))); + ASSERT_TRUE(TestUtil::CopyDirectory(PathUtil::JoinPath(table_path, "snapshot"), + PathUtil::JoinPath(branch_path, "snapshot"))); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr main_batch, + TestHelper::MakeRecordBatch(row_type, R"([[1, 11], [3, 30]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(main_batch), /*commit_identifier=*/1, helper.get())); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult result, + ReadSystemTable(table_path + "$branch_rt$ro", options)); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 10], [0, 2, 20]])"}, &expected) + .ok()); + ASSERT_TRUE(result.array->Equals(expected)) << result.array->ToString(); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult main_result, + ReadSystemTable(table_path + "$ro", options)); + std::shared_ptr expected_main; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 11], [0, 2, 20], [0, 3, 30]])"}, &expected_main) + .ok()); + ASSERT_TRUE(main_result.array->Equals(expected_main)) << main_result.array->ToString(); +} + +TEST(SystemTableReadInteTest, TestReadOptimizedSystemTableWithFirstRowMergeEngine) { + arrow::FieldVector fields = { + arrow::field("k", arrow::int32()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + std::map options = { + {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "parquet"}, + {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "k"}, {Options::NUM_LEVELS, "5"}, + {Options::MERGE_ENGINE, "first-row"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr helper, + TestHelper::Create(dir->Str(), schema, + /*partition_keys=*/{}, + /*primary_keys=*/{"k"}, options, + /*is_streaming_mode=*/true, + /*ignore_if_exists=*/false, PathUtil::JoinPath(dir->Str(), "tmp"))); + std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar"); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[1, 10], [2, 20]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(WriteAndFullCompact(std::move(batch), /*commit_identifier=*/0, helper.get())); + + ASSERT_OK_AND_ASSIGN(SystemTableReadResult result, + ReadSystemTable(table_path + "$ro", options)); + std::shared_ptr expected_type = + arrow::struct_({arrow::field("_VALUE_KIND", arrow::int8()), + arrow::field("k", arrow::int32()), arrow::field("v", arrow::int32())}); + std::shared_ptr expected; + ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON( + expected_type, {R"([[0, 1, 10], [0, 2, 20]])"}, &expected) + .ok()); + ASSERT_TRUE(result.array->Equals(expected)) << result.array->ToString(); +} + TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedTable) { arrow::FieldVector fields = { arrow::field("dt", arrow::utf8()), From 8726c7185da06e4649b809c455332b5edb1d21f1 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Thu, 16 Jul 2026 13:56:33 +0800 Subject: [PATCH 098/138] feat(blob): support blob.split-by-file-size to weigh blob files in scan splitting --- include/paimon/defs.h | 4 ++ src/paimon/common/defs.cpp | 1 + src/paimon/core/core_options.cpp | 11 ++++ src/paimon/core/core_options.h | 1 + src/paimon/core/core_options_test.cpp | 9 ++++ .../source/data_evolution_split_generator.cpp | 9 +++- .../source/data_evolution_split_generator.h | 8 ++- .../table/source/split_generator_test.cpp | 51 +++++++++++++++++++ src/paimon/core/table/source/table_scan.cpp | 5 +- 9 files changed, 93 insertions(+), 6 deletions(-) diff --git a/include/paimon/defs.h b/include/paimon/defs.h index dd3fc5ac..a25587d1 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -122,6 +122,10 @@ struct PAIMON_EXPORT Options { /// "blob.target-file-size" - Target size of a blob file. Default is TARGET_FILE_SIZE. static const char BLOB_TARGET_FILE_SIZE[]; + /// "blob.split-by-file-size" - Whether to consider blob file size as a factor when performing + /// scan splitting. When unset, defaults to the negation of BLOB_AS_DESCRIPTOR. + static const char BLOB_SPLIT_BY_FILE_SIZE[]; + /// "partition.default-name" - The default partition name in case the dynamic partition column /// value is null/empty string. Default is "__DEFAULT_PARTITION__". static const char PARTITION_DEFAULT_NAME[]; diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 7c5beabf..2504ff1b 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -36,6 +36,7 @@ const char Options::FILE_FORMAT[] = "file.format"; const char Options::FILE_SYSTEM[] = "file-system"; const char Options::TARGET_FILE_SIZE[] = "target-file-size"; const char Options::BLOB_TARGET_FILE_SIZE[] = "blob.target-file-size"; +const char Options::BLOB_SPLIT_BY_FILE_SIZE[] = "blob.split-by-file-size"; const char Options::PAGE_SIZE[] = "page-size"; const char Options::PARTITION_DEFAULT_NAME[] = "partition.default-name"; const char Options::FILE_COMPRESSION[] = "file.compression"; diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 5c434188..18987472 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -450,6 +450,8 @@ struct CoreOptions::Impl { bool row_tracking_partition_group_on_commit = true; bool data_evolution_enabled = false; bool blob_view_resolve_enabled = true; + bool blob_as_descriptor = false; + std::optional blob_split_by_file_size; bool legacy_partition_name_enabled = true; bool global_index_enabled = true; std::optional global_index_thread_num; @@ -580,6 +582,11 @@ struct CoreOptions::Impl { // Parse blob-view.resolve.enabled - whether to resolve blob view fields at read time PAIMON_RETURN_NOT_OK( parser.Parse(Options::BLOB_VIEW_RESOLVE_ENABLED, &blob_view_resolve_enabled)); + // Parse blob-as-descriptor - read blob field as descriptor rather than blob bytes + PAIMON_RETURN_NOT_OK(parser.Parse(Options::BLOB_AS_DESCRIPTOR, &blob_as_descriptor)); + // Parse blob.split-by-file-size - whether blob file size counts in scan splitting + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::BLOB_SPLIT_BY_FILE_SIZE, &blob_split_by_file_size)); return Status::OK(); } @@ -986,6 +993,10 @@ int64_t CoreOptions::GetBlobTargetFileSize() const { return impl_->blob_target_file_size.value(); } +bool CoreOptions::BlobSplitByFileSize() const { + return impl_->blob_split_by_file_size.value_or(!impl_->blob_as_descriptor); +} + int64_t CoreOptions::GetCompactionFileSize(bool has_primary_key) const { // file size to join the compaction, we don't process on middle file size to avoid // compact a same file twice (the compression is not calculate so accurately. the output diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index ee573eb7..db6e7604 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -79,6 +79,7 @@ class PAIMON_EXPORT CoreOptions { int64_t GetPageSize() const; int64_t GetTargetFileSize(bool has_primary_key) const; int64_t GetBlobTargetFileSize() const; + bool BlobSplitByFileSize() const; int64_t GetCompactionFileSize(bool has_primary_key) const; std::string GetPartitionDefaultName() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 71508421..8101e359 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -43,6 +43,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_EQ(256 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/false)); ASSERT_EQ(128 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/true)); ASSERT_EQ(256 * 1024 * 1024L, core_options.GetBlobTargetFileSize()); + ASSERT_TRUE(core_options.BlobSplitByFileSize()); ASSERT_EQ(187904815, core_options.GetCompactionFileSize(/*has_primary_key=*/false)); ASSERT_EQ(93952404, core_options.GetCompactionFileSize(/*has_primary_key=*/true)); @@ -242,6 +243,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::DATA_EVOLUTION_ENABLED, "true"}, {Options::BLOB_FIELD, "blob1,blob2"}, {Options::BLOB_DESCRIPTOR_FIELD, "blob3,blob4"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::BLOB_VIEW_FIELD, "blob5"}, {Options::BLOB_VIEW_UPSTREAM_WAREHOUSE, "FILE:///tmp/blob_view_upstream_warehouse/"}, {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}, @@ -387,6 +389,7 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_TRUE(core_options.DataEvolutionEnabled()); ASSERT_EQ(core_options.GetBlobFields(), std::vector({"blob1", "blob2"})); ASSERT_EQ(core_options.GetBlobDescriptorFields(), std::vector({"blob3", "blob4"})); + ASSERT_FALSE(core_options.BlobSplitByFileSize()); ASSERT_EQ(core_options.GetBlobViewFields(), std::vector({"blob5"})); ASSERT_EQ(core_options.GetBlobInlineFields(), std::vector({"blob3", "blob4", "blob5"})); @@ -972,6 +975,12 @@ TEST(CoreOptionsTest, TestFallback) { ASSERT_EQ(options.GetBlobDescriptorFields(), std::vector({"new_b1", "new_b2"})); } + { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_SPLIT_BY_FILE_SIZE, "true"}})); + ASSERT_TRUE(options.BlobSplitByFileSize()); + } } TEST(CoreOptionsTest, TestMapStorageLayout) { diff --git a/src/paimon/core/table/source/data_evolution_split_generator.cpp b/src/paimon/core/table/source/data_evolution_split_generator.cpp index 66b705bc..0a76e75a 100644 --- a/src/paimon/core/table/source/data_evolution_split_generator.cpp +++ b/src/paimon/core/table/source/data_evolution_split_generator.cpp @@ -24,6 +24,7 @@ #include #include +#include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/bin_packing.h" #include "paimon/common/utils/range_helper.h" #include "paimon/core/io/data_file_meta.h" @@ -43,11 +44,15 @@ Result> DataEvolutionSplitGenerator::Spl PAIMON_ASSIGN_OR_RAISE(std::vector>> ranges, range_helper.MergeOverlappingRanges(std::move(input))); - auto weight_func = [open_file_cost = open_file_cost_]( + auto weight_func = [open_file_cost = open_file_cost_, count_blob_size = count_blob_size_]( const std::vector>& metas) -> int64_t { int64_t file_size_sum = 0; for (const auto& meta : metas) { - file_size_sum += meta->file_size; + if (BlobUtils::IsBlobFile(meta->file_name)) { + file_size_sum += count_blob_size ? meta->file_size : open_file_cost; + } else { + file_size_sum += meta->file_size; + } } return std::max(file_size_sum, open_file_cost); }; diff --git a/src/paimon/core/table/source/data_evolution_split_generator.h b/src/paimon/core/table/source/data_evolution_split_generator.h index 7e9d75ca..73ccd211 100644 --- a/src/paimon/core/table/source/data_evolution_split_generator.h +++ b/src/paimon/core/table/source/data_evolution_split_generator.h @@ -33,8 +33,11 @@ struct DataFileMeta; /// Append data evolution table split generator, which implementation of `SplitGenerator`. class DataEvolutionSplitGenerator : public SplitGenerator { public: - DataEvolutionSplitGenerator(int64_t target_split_size, int64_t open_file_cost) - : target_split_size_(target_split_size), open_file_cost_(open_file_cost) {} + DataEvolutionSplitGenerator(int64_t target_split_size, int64_t open_file_cost, + bool count_blob_size) + : target_split_size_(target_split_size), + open_file_cost_(open_file_cost), + count_blob_size_(count_blob_size) {} Result> SplitForBatch( std::vector>&& input) const override; @@ -47,6 +50,7 @@ class DataEvolutionSplitGenerator : public SplitGenerator { private: int64_t target_split_size_; int64_t open_file_cost_; + bool count_blob_size_; }; } // namespace paimon diff --git a/src/paimon/core/table/source/split_generator_test.cpp b/src/paimon/core/table/source/split_generator_test.cpp index caf9e701..ac80eb3f 100644 --- a/src/paimon/core/table/source/split_generator_test.cpp +++ b/src/paimon/core/table/source/split_generator_test.cpp @@ -39,6 +39,7 @@ #include "paimon/core/stats/simple_stats.h" #include "paimon/core/table/bucket_mode.h" #include "paimon/core/table/source/append_only_split_generator.h" +#include "paimon/core/table/source/data_evolution_split_generator.h" #include "paimon/core/table/source/merge_tree_split_generator.h" #include "paimon/data/timestamp.h" #include "paimon/memory/memory_pool.h" @@ -119,6 +120,21 @@ class SplitGeneratorTest : public testing::Test { /*write_cols=*/std::nullopt); } + std::shared_ptr CreateDataFileMetaWithRowId(const std::string& file_name, + int64_t file_size, int64_t row_count, + int64_t first_row_id) { + return std::make_shared( + file_name, file_size, row_count, /*min_key=*/BinaryRow::EmptyRow(), + /*max_key=*/BinaryRow::EmptyRow(), /*key_stats=*/SimpleStats::EmptyStats(), + /*value_stats=*/SimpleStats::EmptyStats(), /*min_sequence_number=*/0, + /*max_sequence_number=*/0, /*schema_id=*/0, + /*level=*/0, /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/0, + /*embedded_index=*/nullptr, FileSource::Append(), /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, first_row_id, + /*write_cols=*/std::nullopt); + } + static void CheckResult(const std::vector& result_groups, const std::vector>& expected_file_names, const std::vector& expected_raw_convertible) { @@ -209,6 +225,41 @@ TEST_F(SplitGeneratorTest, TestAppend) { } } +TEST_F(SplitGeneratorTest, TestDataEvolutionBlobSplitByFileSize) { + // two row id ranges, each with one data file and one much larger blob file + auto create_files = [&]() { + return std::vector>{ + CreateDataFileMetaWithRowId("f1", /*file_size=*/100, /*row_count=*/100, + /*first_row_id=*/0), + CreateDataFileMetaWithRowId("blob1.blob", /*file_size=*/1000, /*row_count=*/100, + /*first_row_id=*/0), + CreateDataFileMetaWithRowId("f2", /*file_size=*/100, /*row_count=*/100, + /*first_row_id=*/100), + CreateDataFileMetaWithRowId("blob2.blob", /*file_size=*/1000, /*row_count=*/100, + /*first_row_id=*/100)}; + }; + { + // blob file size counts in splitting: each range weighs 1100, so the two ranges cannot + // be packed into one split of target size 1200 + DataEvolutionSplitGenerator split_generator(/*target_split_size=*/1200, + /*open_file_cost=*/10, + /*count_blob_size=*/true); + ASSERT_OK_AND_ASSIGN(std::vector split_groups, + split_generator.SplitForBatch(create_files())); + CheckResult(split_groups, {{"f1", "blob1.blob"}, {"f2", "blob2.blob"}}, {false, false}); + } + { + // blob file only weighs the open file cost: each range weighs 110, both ranges fit in + // one split of target size 1200 + DataEvolutionSplitGenerator split_generator(/*target_split_size=*/1200, + /*open_file_cost=*/10, + /*count_blob_size=*/false); + ASSERT_OK_AND_ASSIGN(std::vector split_groups, + split_generator.SplitForBatch(create_files())); + CheckResult(split_groups, {{"f1", "blob1.blob", "f2", "blob2.blob"}}, {false}); + } +} + TEST_F(SplitGeneratorTest, TestMergeTree) { std::vector> files = { CreateDataFileMeta("1", 0, 10), CreateDataFileMeta("2", 0, 12), diff --git a/src/paimon/core/table/source/table_scan.cpp b/src/paimon/core/table/source/table_scan.cpp index 6b41ee24..17add044 100644 --- a/src/paimon/core/table/source/table_scan.cpp +++ b/src/paimon/core/table/source/table_scan.cpp @@ -139,8 +139,9 @@ class TableScanImpl { auto source_split_open_file_cost = core_options.GetSourceSplitOpenFileCost(); if (table_schema->PrimaryKeys().empty()) { if (core_options.DataEvolutionEnabled()) { - return std::make_unique(source_split_target_size, - source_split_open_file_cost); + return std::make_unique( + source_split_target_size, source_split_open_file_cost, + core_options.BlobSplitByFileSize()); } BucketMode bucket_mode = (core_options.GetBucket() == -1 ? BucketMode::BUCKET_UNAWARE : BucketMode::HASH_FIXED); From 0cc0a2897ad4b29c9a906b717cad3eb5e40566d3 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Thu, 16 Jul 2026 18:10:32 +0800 Subject: [PATCH 099/138] feat(blob): support blob-write-null-on-missing-file and blob-write-null-on-fetch-failure options --- cmake_modules/ThirdpartyToolchain.cmake | 5 +- include/paimon/defs.h | 9 + src/paimon/CMakeLists.txt | 15 +- src/paimon/common/defs.cpp | 2 + .../blob/blob_file_batch_reader_test.cpp | 6 +- .../blob/blob_file_format_factory_test.cpp | 58 ++++ src/paimon/format/blob/blob_format_writer.cpp | 57 +++- src/paimon/format/blob/blob_format_writer.h | 15 + .../format/blob/blob_format_writer_test.cpp | 313 ++++++++++++++---- src/paimon/format/blob/blob_writer_builder.h | 11 +- .../format/blob/blob_writer_builder_test.cpp | 90 ++++- src/paimon/fs/jindo/jindo_utils.h | 18 +- src/paimon/fs/jindo/jindo_utils_test.cpp | 50 +++ src/paimon/testing/utils/test_helper.h | 22 ++ test/inte/blob_table_inte_test.cpp | 239 +++++++++++++ 15 files changed, 823 insertions(+), 87 deletions(-) create mode 100644 src/paimon/fs/jindo/jindo_utils_test.cpp diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 1617df68..e56d6257 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -1717,13 +1717,16 @@ macro(build_arrow) target_link_libraries(arrow_dataset INTERFACE arrow_acero) + # libarrow.a calls dlsym; keep ${CMAKE_DL_LIBS} in the interface so -ldl is placed + # after libarrow.a on linkers that resolve symbols strictly left-to-right. target_link_libraries(arrow INTERFACE zstd snappy lz4 zlib re2::re2 - arrow_bundled_dependencies) + arrow_bundled_dependencies + ${CMAKE_DL_LIBS}) target_link_libraries(parquet INTERFACE zstd diff --git a/include/paimon/defs.h b/include/paimon/defs.h index a25587d1..b9b73cb4 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -443,6 +443,15 @@ struct PAIMON_EXPORT Options { /// Blob View is enabled, cpp paimon cannot automatically obtain the upstream table warehouse /// path and requires manual configuration by the user. No default value. static const char BLOB_VIEW_UPSTREAM_WAREHOUSE[]; + /// "blob-write-null-on-missing-file" - Whether to write NULL for a descriptor BLOB value when + /// the referenced file does not exist at write time. When false, the write fails when the + /// descriptor is read. Default value is "false". + static const char BLOB_WRITE_NULL_ON_MISSING_FILE[]; + /// "blob-write-null-on-fetch-failure" - Whether to write NULL for a descriptor BLOB value when + /// the referenced data cannot be fetched at write time (e.g. invalid descriptor or invalid + /// offset). A missing file is handled by "blob-write-null-on-missing-file". When false, the + /// write fails when the descriptor is read. Default value is "false". + static const char BLOB_WRITE_NULL_ON_FETCH_FAILURE[]; /// "global-index.enabled" - Whether to enable global index for scan. Default value is "true". static const char GLOBAL_INDEX_ENABLED[]; /// "global-index.thread-num" - The maximum number of concurrent scanner for global index. No diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index aa587590..96187c8e 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -793,18 +793,29 @@ if(PAIMON_BUILD_TESTS) ${TEST_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN}) + # jindo_utils_test only checks the in-memory status conversion and does not need an + # OSS cluster, so it runs whenever jindo is built. The other jindo tests need real + # OSS access and stay disabled. + set(FS_TEST_JINDO_SOURCES) + if(PAIMON_ENABLE_JINDO) + list(APPEND FS_TEST_JINDO_SOURCES fs/jindo/jindo_utils_test.cpp) + endif() + add_paimon_test(fs_test SOURCES common/fs/file_system_test.cpp common/fs/resolving_file_system_test.cpp fs/local/local_file_test.cpp + ${FS_TEST_JINDO_SOURCES} # fs/jindo/jindo_file_system_factory_test.cpp # fs/jindo/jindo_file_system_test.cpp STATIC_LINK_LIBS paimon_shared ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} - # ${PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS} + ${PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS} test_utils_static - ${GTEST_LINK_TOOLCHAIN}) + ${GTEST_LINK_TOOLCHAIN} + EXTRA_INCLUDES + ${JINDOSDK_INCLUDE_DIR}) endif() diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 2504ff1b..24a78acc 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -111,6 +111,8 @@ const char Options::FALLBACK_BLOB_DESCRIPTOR_FIELD[] = "blob.stored-descriptor-f const char Options::BLOB_VIEW_FIELD[] = "blob-view-field"; const char Options::BLOB_VIEW_RESOLVE_ENABLED[] = "blob-view.resolve.enabled"; const char Options::BLOB_VIEW_UPSTREAM_WAREHOUSE[] = "blob-view-upstream-warehouse"; +const char Options::BLOB_WRITE_NULL_ON_MISSING_FILE[] = "blob-write-null-on-missing-file"; +const char Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE[] = "blob-write-null-on-fetch-failure"; const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled"; const char Options::GLOBAL_INDEX_THREAD_NUM[] = "global-index.thread-num"; const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path"; diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 9b2b3206..c53397f6 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -236,8 +236,10 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { file_system->Create(dir->Str() + "/file.blob", /*overwrite=*/true)); std::shared_ptr blob_field = BlobUtils::ToArrowField("blob_col"); auto struct_type = arrow::struct_({blob_field}); - ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, - BlobFormatWriter::Create(output_stream, struct_type, file_system, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream, struct_type, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, file_system, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); diff --git a/src/paimon/format/blob/blob_file_format_factory_test.cpp b/src/paimon/format/blob/blob_file_format_factory_test.cpp index 05898987..a5807fd6 100644 --- a/src/paimon/format/blob/blob_file_format_factory_test.cpp +++ b/src/paimon/format/blob/blob_file_format_factory_test.cpp @@ -18,8 +18,24 @@ #include "paimon/format/blob/blob_file_format_factory.h" +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/data/blob.h" +#include "paimon/defs.h" +#include "paimon/format/file_format.h" +#include "paimon/format/file_format_factory.h" +#include "paimon/format/format_writer.h" +#include "paimon/format/writer_builder.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/status.h" +#include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" namespace paimon::blob::test { @@ -31,4 +47,46 @@ TEST(BlobFileFormatFactoryTest, TestIdentifier) { ASSERT_EQ(file_format->Identifier(), "blob"); } +TEST(BlobFileFormatFactoryTest, TestWriteNullOptionPropagation) { + // Verifies the option flows through the production path + // FileFormatFactory::Get -> BlobFileFormat -> BlobWriterBuilder -> BlobFormatWriter. + std::unique_ptr dir = + paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::shared_ptr fs = std::make_shared(); + auto struct_type = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + + auto write_once = [&](const std::map& options, + const std::string& file_name) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, + FileFormatFactory::Get("blob", options)); + auto schema = arrow::schema(struct_type->fields()); + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer_builder, + format->CreateWriterBuilder(&c_schema, /*batch_size=*/1024)); + // The blob writer builder is a SpecificFSWriterBuilder by construction. + static_cast(writer_builder.get())->WithFileSystem(fs); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr out, + fs->Create(dir->Str() + "/" + file_name, /*overwrite=*/true)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr writer, + writer_builder->Build(out, "none")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr array, + paimon::test::TestHelper::MakeBlobDescriptorArray( + struct_type, missing_blob, GetDefaultPool())); + ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + PAIMON_RETURN_NOT_OK(writer->AddBatch(&c_array)); + return writer->Finish(); + }; + + // Without the option, writing the missing descriptor fails. + ASSERT_NOK_WITH_MSG(write_once({}, "no_option.blob"), "not exists"); + // The option set in the format options map reaches the writer. + ASSERT_OK(write_once({{Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"}}, "with_option.blob")); +} + } // namespace paimon::blob::test diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index 7da5810c..ff20e1d1 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -33,21 +33,32 @@ #include "paimon/common/utils/delta_varint_compressor.h" #include "paimon/data/blob.h" #include "paimon/io/byte_array_input_stream.h" +#include "paimon/logging.h" namespace paimon::blob { BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, + bool write_null_on_missing_file, + bool write_null_on_fetch_failure, const std::shared_ptr& fs, const std::shared_ptr& pool) - : out_(out), uri_(uri), data_type_(data_type), fs_(fs), pool_(pool) { + : out_(out), + uri_(uri), + data_type_(data_type), + fs_(fs), + pool_(pool), + write_null_on_missing_file_(write_null_on_missing_file), + write_null_on_fetch_failure_(write_null_on_fetch_failure) { metrics_ = std::make_shared(); tmp_buffer_ = Bytes::AllocateBytes(kTmpBufferSize, pool_.get()); magic_number_bytes_ = IntegerToLittleEndian(BlobDefs::kMagicNumber, pool_); + logger_ = Logger::GetLogger("BlobFormatWriter"); } Result> BlobFormatWriter::Create( const std::shared_ptr& out, const std::shared_ptr& data_type, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, const std::shared_ptr& fs, const std::shared_ptr& pool) { if (out == nullptr) { return Status::Invalid("blob format writer create failed. out is nullptr"); @@ -67,7 +78,8 @@ Result> BlobFormatWriter::Create( fmt::format("field {} is not BLOB", data_type->field(0)->ToString())); } PAIMON_ASSIGN_OR_RAISE(std::string uri, out->GetUri()); - return std::unique_ptr(new BlobFormatWriter(out, uri, data_type, fs, pool)); + return std::unique_ptr(new BlobFormatWriter( + out, uri, data_type, write_null_on_missing_file, write_null_on_fetch_failure, fs, pool)); } Status BlobFormatWriter::AddBatch(ArrowArray* batch) { @@ -128,13 +140,8 @@ Status BlobFormatWriter::Finish() { } Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { - crc32_ = 0; - PAIMON_ASSIGN_OR_RAISE(int64_t previous_pos, out_->GetPos()); - - // write magic number - PAIMON_RETURN_NOT_OK(WriteWithCrc32(magic_number_bytes_->data(), magic_number_bytes_->size())); - - // write blob content + // Open the blob input stream before writing any bytes, so that a failed fetch can be + // converted to a NULL element without leaving partial data in the output stream. // Dynamically check whether blob_data is a serialized BlobDescriptor (by magic header) // rather than relying on blob_as_descriptor_ config. This is consistent with Java behavior: // at write time, the input bytes are auto-detected as descriptor or raw data. @@ -142,13 +149,32 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { PAIMON_ASSIGN_OR_RAISE(bool is_descriptor, BlobDescriptor::IsBlobDescriptor(blob_data.data(), blob_data.size())); if (is_descriptor) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, - Blob::FromDescriptor(blob_data.data(), blob_data.size())); - PAIMON_ASSIGN_OR_RAISE(in, blob->NewInputStream(fs_)); + Result> opened = OpenDescriptorInputStream(blob_data); + if (!opened.ok()) { + const Status& status = opened.status(); + // A missing file is only handled by 'blob-write-null-on-missing-file'; other fetch + // failures are only handled by 'blob-write-null-on-fetch-failure' (aligned with Java). + bool write_null = + status.IsNotExist() ? write_null_on_missing_file_ : write_null_on_fetch_failure_; + if (write_null) { + PAIMON_LOG_WARN(logger_, "Failed to open blob, writing NULL for BLOB field: %s", + status.ToString().c_str()); + bin_lengths_.push_back(BlobDefs::kNullBinLength); + return Status::OK(); + } + return status; + } + in = std::move(opened).value(); } else { in = std::make_unique(blob_data.data(), blob_data.size()); } PAIMON_ASSIGN_OR_RAISE(int64_t file_length, in->Length()); + + crc32_ = 0; + PAIMON_ASSIGN_OR_RAISE(int64_t previous_pos, out_->GetPos()); + + // write magic number + PAIMON_RETURN_NOT_OK(WriteWithCrc32(magic_number_bytes_->data(), magic_number_bytes_->size())); int64_t total_read_length = 0; int64_t read_len = std::min(file_length, static_cast(tmp_buffer_->size())); while (read_len > 0) { @@ -181,6 +207,13 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { return Status::OK(); } +Result> BlobFormatWriter::OpenDescriptorInputStream( + std::string_view blob_data) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, + Blob::FromDescriptor(blob_data.data(), blob_data.size())); + return blob->NewInputStream(fs_); +} + Status BlobFormatWriter::WriteBytes(const char* data, int64_t length) { PAIMON_ASSIGN_OR_RAISE(int64_t actual, out_->Write(data, length)); if (actual != length) { diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 50641d9c..a95d0715 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -28,6 +28,7 @@ #include "arrow/api.h" #include "arrow/util/crc32.h" #include "paimon/format/format_writer.h" +#include "paimon/logging.h" #include "paimon/memory/bytes.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" @@ -41,6 +42,7 @@ struct ArrowArray; namespace paimon { class Blob; class FileSystem; +class InputStream; class Metrics; class OutputStream; } // namespace paimon @@ -51,8 +53,13 @@ namespace paimon::blob { // https://cwiki.apache.org/confluence/display/PAIMON/PIP-35%3A+Introduce+Blob+to+store+multimodal+data class BlobFormatWriter : public FormatWriter { public: + /// When opening a descriptor input fails, `write_null_on_missing_file` converts a + /// missing file (Status::NotExist) to a NULL element and `write_null_on_fetch_failure` + /// converts any other open failure; failures during the streaming copy always fail the + /// write. See Options::BLOB_WRITE_NULL_ON_MISSING_FILE / BLOB_WRITE_NULL_ON_FETCH_FAILURE. static Result> Create( const std::shared_ptr& out, const std::shared_ptr& data_type, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, const std::shared_ptr& fs, const std::shared_ptr& pool); Status AddBatch(ArrowArray* batch) override; @@ -72,11 +79,16 @@ class BlobFormatWriter : public FormatWriter { private: BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, const std::shared_ptr& fs, const std::shared_ptr& pool); Status WriteBlob(std::string_view blob_data); + /// Deserialize the descriptor and open an input stream on the referenced data. + Result> OpenDescriptorInputStream( + std::string_view blob_data) const; + Status WriteBytes(const char* data, int64_t length); Status WriteWithCrc32(const char* data, int64_t length); @@ -97,6 +109,9 @@ class BlobFormatWriter : public FormatWriter { std::shared_ptr fs_; std::shared_ptr pool_; std::shared_ptr metrics_; + bool write_null_on_missing_file_ = false; + bool write_null_on_fetch_failure_ = false; + std::unique_ptr logger_; }; } // namespace paimon::blob diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index 4962b4af..51e70ef1 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -34,10 +34,24 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::blob::test { -class BlobFormatWriterTest : public ::testing::Test, public ::testing::WithParamInterface { + +/// A file system whose Open() always fails with the configured status, for verifying how the +/// writer classifies open failures by status code. +class OpenFailFileSystem : public LocalFileSystem { + public: + explicit OpenFailFileSystem(Status open_status) : open_status_(std::move(open_status)) {} + + Result> Open(const std::string& path) const override { + return open_status_; + } + + private: + Status open_status_; +}; + +class BlobFormatWriterTestBase : public ::testing::Test { public: void SetUp() override { - blob_as_descriptor_ = GetParam(); pool_ = GetDefaultPool(); dir_ = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(dir_); @@ -52,24 +66,11 @@ class BlobFormatWriterTest : public ::testing::Test, public ::testing::WithParam ASSERT_OK(output_stream_->Close()); } - Result> PrepareBlobArray( - const std::shared_ptr& blob) const { - arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), - {std::make_shared()}); - auto blob_builder = - static_cast(struct_builder.field_builder(0)); - PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Append()); - if (blob_as_descriptor_) { - PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->Append( - blob->ToDescriptor(pool_)->data(), blob->ToDescriptor(pool_)->size())); - } else { - PAIMON_ASSIGN_OR_RAISE(auto blob_data, blob->ToData(file_system_, pool_)); - PAIMON_RETURN_NOT_OK_FROM_ARROW( - blob_builder->Append(blob_data->data(), blob_data->size())); - } - std::shared_ptr array; - PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Finish(&array)); - return array; + /// Create a writer on output_stream_ with both write-null options disabled. + Result> CreateDefaultWriter() const { + return BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, file_system_, pool_); } Status AddBatchOnce(const std::shared_ptr& format_writer, @@ -79,8 +80,30 @@ class BlobFormatWriterTest : public ::testing::Test, public ::testing::WithParam return format_writer->AddBatch(c_array.get()); } - private: - bool blob_as_descriptor_; + Result> PrepareDescriptorArray( + const std::shared_ptr& blob) const { + return paimon::test::TestHelper::MakeBlobDescriptorArray(struct_type_, blob, pool_); + } + + Result> ReadBackAsData() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); + PAIMON_RETURN_NOT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(chunked_array->chunks())); + return arrow::internal::checked_pointer_cast(concat_array); + } + + protected: std::shared_ptr pool_; std::unique_ptr dir_; std::shared_ptr output_stream_; @@ -88,13 +111,45 @@ class BlobFormatWriterTest : public ::testing::Test, public ::testing::WithParam std::shared_ptr struct_type_; }; +class BlobFormatWriterTest : public BlobFormatWriterTestBase, + public ::testing::WithParamInterface { + public: + void SetUp() override { + blob_as_descriptor_ = GetParam(); + BlobFormatWriterTestBase::SetUp(); + } + + Result> PrepareBlobArray( + const std::shared_ptr& blob) const { + if (blob_as_descriptor_) { + return PrepareDescriptorArray(blob); + } + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Append()); + PAIMON_ASSIGN_OR_RAISE(PAIMON_UNIQUE_PTR blob_data, + blob->ToData(file_system_, pool_)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->Append(blob_data->data(), blob_data->size())); + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Finish(&array)); + return array; + } + + private: + bool blob_as_descriptor_; +}; + +/// The write-null tests always feed descriptor bytes, so they do not depend on the +/// blob_as_descriptor_ parameter and run once on the non-parameterized fixture. +using BlobFormatWriterWriteNullTest = BlobFormatWriterTestBase; + INSTANTIATE_TEST_SUITE_P(BlobAsDescriptor, BlobFormatWriterTest, ::testing::Values(false, true)); TEST_P(BlobFormatWriterTest, TestSimple) { // write - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); std::vector> expected_blobs; std::string file1 = paimon::test::GetDataDir() + "/avro/data/avro_with_null"; @@ -154,37 +209,42 @@ TEST_P(BlobFormatWriterTest, TestSimple) { TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) { // Test with nullptr output stream - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(nullptr, struct_type_, file_system_, pool_), - "blob format writer create failed. out is nullptr"); + ASSERT_NOK_WITH_MSG( + BlobFormatWriter::Create(nullptr, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, file_system_, pool_), + "blob format writer create failed. out is nullptr"); // Test with nullptr data type - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, nullptr, file_system_, pool_), - "blob format writer create failed. data_type is nullptr"); + ASSERT_NOK_WITH_MSG( + BlobFormatWriter::Create(output_stream_, nullptr, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, file_system_, pool_), + "blob format writer create failed. data_type is nullptr"); // Test with nullptr memory pool ASSERT_NOK_WITH_MSG( - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, nullptr), + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, file_system_, nullptr), "blob format writer create failed. pool is nullptr"); // Test with invalid field count (more than 1 field) auto multi_field_type = arrow::struct_( {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())}); - ASSERT_NOK_WITH_MSG( - BlobFormatWriter::Create(output_stream_, multi_field_type, file_system_, pool_), - "blob data type field number 2 is not 1"); + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( + output_stream_, multi_field_type, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, file_system_, pool_), + "blob data type field number 2 is not 1"); // Test with non-blob field (missing blob metadata) auto non_blob_field = arrow::field("regular_col", arrow::binary()); auto non_blob_type = arrow::struct_({non_blob_field}); - ASSERT_NOK_WITH_MSG( - BlobFormatWriter::Create(output_stream_, non_blob_type, file_system_, pool_), - "field regular_col: binary is not BLOB"); + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( + output_stream_, non_blob_type, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, file_system_, pool_), + "field regular_col: binary is not BLOB"); } TEST_P(BlobFormatWriterTest, TestInvalidCase) { - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Test nullptr batch ASSERT_NOK_WITH_MSG(writer->AddBatch(nullptr), @@ -201,9 +261,7 @@ TEST_P(BlobFormatWriterTest, TestInvalidCase) { } TEST_P(BlobFormatWriterTest, TestAddBatchWithInvalidBatchLength) { - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Test batch with wrong length (not 1) arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), @@ -229,9 +287,7 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithInvalidBatchLength) { } TEST_P(BlobFormatWriterTest, TestReachTargetSize) { - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Initially should not reach target size ASSERT_OK_AND_ASSIGN(bool reached, writer->ReachTargetSize(true, 1000)); @@ -254,9 +310,7 @@ TEST_P(BlobFormatWriterTest, TestReachTargetSize) { } TEST_P(BlobFormatWriterTest, TestGetWriterMetrics) { - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); auto metrics = writer->GetWriterMetrics(); ASSERT_TRUE(metrics); @@ -264,9 +318,7 @@ TEST_P(BlobFormatWriterTest, TestGetWriterMetrics) { TEST_P(BlobFormatWriterTest, TestEmptyWriter) { // Test creating a writer and finishing without adding any data - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -285,9 +337,7 @@ TEST_P(BlobFormatWriterTest, TestEmptyWriter) { } TEST_P(BlobFormatWriterTest, TestLargeBlob) { - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Create a temporary large file for testing std::string large_file_path = dir_->Str() + "/large_test_file.bin"; @@ -340,9 +390,7 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { } TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Write one row with child-level null blob arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), @@ -388,18 +436,159 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { ASSERT_TRUE(struct_builder2.Finish(&null_struct_array).ok()); auto null_c_array = std::make_unique(); ASSERT_TRUE(arrow::ExportArray(*null_struct_array, null_c_array.get()).ok()); - ASSERT_OK_AND_ASSIGN( - std::shared_ptr writer2, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer2, CreateDefaultWriter()); ASSERT_NOK_WITH_MSG(writer2->AddBatch(null_c_array.get()), "BlobFormatWriter does not support struct-level null."); ArrowArrayRelease(null_c_array.get()); } -TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnMissingFile) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); + ASSERT_OK(AddBatchOnce(writer, missing_array)); + + // A fetch failure is not converted to NULL by write_null_on_missing_file alone (aligned + // with Java); the rejected row leaves the writer usable. + std::string file = paimon::test::GetDataDir() + "/xxhash.data"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, + Blob::FromPath(file, /*offset=*/1 << 20, /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto bad_offset_array, PrepareDescriptorArray(bad_offset_blob)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, bad_offset_array), "exceed total length"); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, Blob::FromPath(file)); + ASSERT_OK_AND_ASSIGN(auto array, PrepareDescriptorArray(blob)); + ASSERT_OK(AddBatchOnce(writer, array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 2); + ASSERT_TRUE(result_struct->field(0)->IsNull(0)); + ASSERT_FALSE(result_struct->field(0)->IsNull(1)); + auto binary_array = + arrow::internal::checked_pointer_cast(result_struct->field(0)); + ASSERT_OK_AND_ASSIGN(auto expected_data, blob->ToData(file_system_, pool_)); + ASSERT_EQ(binary_array->GetView(1), + std::string_view(expected_data->data(), expected_data->size())); +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnFetchFailure) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr writer, + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + + std::string file = paimon::test::GetDataDir() + "/xxhash.data"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, + Blob::FromPath(file, /*offset=*/1 << 20, /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto bad_offset_array, PrepareDescriptorArray(bad_offset_blob)); + ASSERT_OK(AddBatchOnce(writer, bad_offset_array)); + + // A missing file is not converted to NULL by write_null_on_fetch_failure alone (aligned + // with Java); the rejected row leaves the writer usable. + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, missing_array), "not exists"); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 1); + ASSERT_TRUE(result_struct->field(0)->IsNull(0)); +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, - BlobFormatWriter::Create(output_stream_, struct_type_, file_system_, pool_)); + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + + // Row 0: missing file -> NULL. + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); + ASSERT_OK(AddBatchOnce(writer, missing_array)); + + // Row 1: fetch failure (offset beyond EOF) -> NULL. + std::string file = paimon::test::GetDataDir() + "/xxhash.data"; + ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, + Blob::FromPath(file, /*offset=*/1 << 20, /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto bad_offset_array, PrepareDescriptorArray(bad_offset_blob)); + ASSERT_OK(AddBatchOnce(writer, bad_offset_array)); + + // Row 2: valid blob. + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, Blob::FromPath(file)); + ASSERT_OK_AND_ASSIGN(auto array, PrepareDescriptorArray(blob)); + ASSERT_OK(AddBatchOnce(writer, array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 3); + ASSERT_TRUE(result_struct->field(0)->IsNull(0)); + ASSERT_TRUE(result_struct->field(0)->IsNull(1)); + ASSERT_FALSE(result_struct->field(0)->IsNull(2)); + auto binary_array = + arrow::internal::checked_pointer_cast(result_struct->field(0)); + ASSERT_OK_AND_ASSIGN(auto expected_data, blob->ToData(file_system_, pool_)); + ASSERT_EQ(binary_array->GetView(2), + std::string_view(expected_data->data(), expected_data->size())); +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByStatusCode) { + // The missing-file vs fetch-failure split keys on the open status code (NotExist <-> missing + // file), independent of the file system implementation. + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, + Blob::FromPath(dir_->Str() + "/any_file", /*offset=*/0, /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto array, PrepareDescriptorArray(blob)); + auto not_exist_fs = std::make_shared(Status::NotExist("mock not exist")); + auto io_error_fs = std::make_shared(Status::IOError("mock io error")); + + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, not_exist_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + } + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "mock io error"); + } + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + } + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, not_exist_fs, pool_)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "mock not exist"); + } +} + +TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); // Create a zero-length file std::string zero_file_path = dir_->Str() + "/zero_length_file.bin"; diff --git a/src/paimon/format/blob/blob_writer_builder.h b/src/paimon/format/blob/blob_writer_builder.h index 2b594e72..dfd09c00 100644 --- a/src/paimon/format/blob/blob_writer_builder.h +++ b/src/paimon/format/blob/blob_writer_builder.h @@ -26,6 +26,8 @@ #include #include "arrow/api.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/defs.h" #include "paimon/format/blob/blob_format_writer.h" #include "paimon/format/format_writer.h" #include "paimon/format/writer_builder.h" @@ -67,7 +69,14 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { if (fs_ == nullptr) { return Status::Invalid("File system is nullptr. Please call WithFileSystem() first."); } - return BlobFormatWriter::Create(out, data_type_, fs_, pool_); + PAIMON_ASSIGN_OR_RAISE(bool write_null_on_missing_file, + OptionsUtils::GetValueFromMap( + options_, Options::BLOB_WRITE_NULL_ON_MISSING_FILE, false)); + PAIMON_ASSIGN_OR_RAISE(bool write_null_on_fetch_failure, + OptionsUtils::GetValueFromMap( + options_, Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, false)); + return BlobFormatWriter::Create(out, data_type_, write_null_on_missing_file, + write_null_on_fetch_failure, fs_, pool_); } private: diff --git a/src/paimon/format/blob/blob_writer_builder_test.cpp b/src/paimon/format/blob/blob_writer_builder_test.cpp index 6adbca72..5b522970 100644 --- a/src/paimon/format/blob/blob_writer_builder_test.cpp +++ b/src/paimon/format/blob/blob_writer_builder_test.cpp @@ -19,11 +19,15 @@ #include "paimon/format/blob/blob_writer_builder.h" #include "arrow/api.h" +#include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/data/blob.h" +#include "paimon/defs.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" +#include "paimon/testing/utils/test_helper.h" #include "paimon/testing/utils/testharness.h" namespace paimon::blob::test { @@ -35,7 +39,7 @@ class BlobWriterBuilderTest : public ::testing::Test { file_system_ = std::make_shared(); ASSERT_OK_AND_ASSIGN(output_stream_, file_system_->Create(dir_->Str() + "/file.blob", /*overwrite=*/true)); - struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", false)}); + struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)}); } void TearDown() override {} @@ -55,4 +59,88 @@ TEST_F(BlobWriterBuilderTest, TestSimple) { ASSERT_OK(builder.Build(output_stream_, "none")); } +TEST_F(BlobWriterBuilderTest, TestWriteNullOptions) { + auto prepare_batch = [&](const std::shared_ptr& blob, ArrowArray* c_array) -> Status { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr array, + paimon::test::TestHelper::MakeBlobDescriptorArray(struct_type_, blob, + GetDefaultPool())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array)); + return Status::OK(); + }; + // Each writer gets its own output file so that a Finish() in one scenario cannot leak + // bytes into the next. + int32_t file_index = 0; + auto build_writer = [&](const std::map& options) + -> Result> { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr out, + file_system_->Create(dir_->Str() + "/file" + std::to_string(file_index++) + ".blob", + /*overwrite=*/true)); + BlobWriterBuilder builder(struct_type_, options); + builder.WithFileSystem(file_system_); + return builder.Build(out, "none"); + }; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + std::string data_file = dir_->Str() + "/data_file.bin"; + ASSERT_OK_AND_ASSIGN(auto data_file_stream, + file_system_->Create(data_file, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(int64_t written, data_file_stream->Write("blob data", 9)); + ASSERT_EQ(written, 9); + ASSERT_OK(data_file_stream->Flush()); + ASSERT_OK(data_file_stream->Close()); + // The offset beyond the end of the 9-byte file makes the descriptor a fetch failure. + ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, + Blob::FromPath(data_file, /*offset=*/100, /*length=*/10)); + + // Both options default to false. + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, build_writer({})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(missing_blob, &c_array)); + ASSERT_NOK_WITH_MSG(writer->AddBatch(&c_array), "not exists"); + } + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, build_writer({})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(bad_offset_blob, &c_array)); + ASSERT_NOK_WITH_MSG(writer->AddBatch(&c_array), "exceed total length"); + } + + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + build_writer({{Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"}})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(missing_blob, &c_array)); + ASSERT_OK(writer->AddBatch(&c_array)); + ASSERT_OK(writer->Finish()); + } + + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + build_writer({{Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "true"}})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(bad_offset_blob, &c_array)); + ASSERT_OK(writer->AddBatch(&c_array)); + ASSERT_OK(writer->Finish()); + } + + // An explicit "false" exercises value parsing, not just the absent-key default. + { + ASSERT_OK_AND_ASSIGN(std::unique_ptr writer, + build_writer({{Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "false"}, + {Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "false"}})); + ArrowArray c_array; + ASSERT_OK(prepare_batch(missing_blob, &c_array)); + ASSERT_NOK_WITH_MSG(writer->AddBatch(&c_array), "not exists"); + } + + { + ASSERT_NOK_WITH_MSG(build_writer({{Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "invalid"}}), + "convert key blob-write-null-on-fetch-failure"); + } +} + } // namespace paimon::blob::test diff --git a/src/paimon/fs/jindo/jindo_utils.h b/src/paimon/fs/jindo/jindo_utils.h index 9dfcc9d4..ca3d3a69 100644 --- a/src/paimon/fs/jindo/jindo_utils.h +++ b/src/paimon/fs/jindo/jindo_utils.h @@ -19,15 +19,21 @@ #pragma once #include "JdoStatus.hpp" // NOLINT(build/include_subdir) +#include "jdo_error.h" // NOLINT(build/include_subdir) #include "paimon/status.h" namespace paimon::jindo { -#define PAIMON_RETURN_NOT_OK_FROM_JINDO(JINDO_STATUS) \ - do { \ - auto __s = (JINDO_STATUS); \ - if (PAIMON_UNLIKELY(!(__s).ok())) { \ - return Status::IOError(__s.errMsg()); \ - } \ +/// Maps a jindo file-not-found error to Status::NotExist so that callers can distinguish +/// a missing file from other IO errors, consistent with LocalFileSystem. +#define PAIMON_RETURN_NOT_OK_FROM_JINDO(JINDO_STATUS) \ + do { \ + auto __s = (JINDO_STATUS); \ + if (PAIMON_UNLIKELY(!(__s).ok())) { \ + if ((__s).getErrCode() == JDO_FILE_NOT_FOUND_ERROR) { \ + return Status::NotExist(__s.errMsg()); \ + } \ + return Status::IOError(__s.errMsg()); \ + } \ } while (false) } // namespace paimon::jindo diff --git a/src/paimon/fs/jindo/jindo_utils_test.cpp b/src/paimon/fs/jindo/jindo_utils_test.cpp new file mode 100644 index 00000000..635c081d --- /dev/null +++ b/src/paimon/fs/jindo/jindo_utils_test.cpp @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/fs/jindo/jindo_utils.h" + +#include "gtest/gtest.h" +#include "jdo_error.h" // NOLINT(build/include_subdir) +#include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::jindo::test { + +namespace { + +Status Convert(const JdoStatus& jdo_status) { + PAIMON_RETURN_NOT_OK_FROM_JINDO(jdo_status); + return Status::OK(); +} + +} // namespace + +TEST(JindoUtilsTest, TestMacroMapsStatusByErrorCode) { + ASSERT_OK(Convert(JdoStatus())); + + Status not_found = Convert(JdoStatus(JDO_FILE_NOT_FOUND_ERROR, "file not found")); + ASSERT_TRUE(not_found.IsNotExist()); + ASSERT_NOK_WITH_MSG(not_found, "file not found"); + + Status other = Convert(JdoStatus(JDO_FILE_NOT_FOUND_ERROR + 1, "some other error")); + ASSERT_TRUE(other.IsIOError()); + ASSERT_NOK_WITH_MSG(other, "some other error"); +} + +} // namespace paimon::jindo::test diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index fb77985c..4c1ce663 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -24,6 +24,7 @@ #include #include +#include "arrow/api.h" #include "arrow/c/bridge.h" #include "arrow/ipc/api.h" #include "paimon/api.h" @@ -177,6 +178,27 @@ class TestHelper { return result_plan->Splits(); } + /// Builds a one-row struct array holding the serialized descriptor of the blob. + static Result> MakeBlobDescriptorArray( + const std::shared_ptr& struct_type, const std::shared_ptr& blob, + const std::shared_ptr& pool) { + if (struct_type->num_fields() != 1 || + struct_type->field(0)->type()->id() != arrow::Type::LARGE_BINARY) { + return Status::Invalid("struct_type must have a single large binary field"); + } + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Append()); + PAIMON_UNIQUE_PTR descriptor = blob->ToDescriptor(pool); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + blob_builder->Append(descriptor->data(), descriptor->size())); + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Finish(&array)); + return array; + } + static Result CheckBlobsEqual(const std::vector>& result_blobs, const std::vector>& expected_blobs, const std::shared_ptr& fs) { diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index a814192b..5f878849 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -372,6 +373,50 @@ class BlobTableInteTest : public testing::Test, public ::testing::WithParamInter }); } + /// Delete the file referenced by the serialized BlobDescriptor at `row` of blob field + /// `field_name`, so that a subsequent table write observes a missing file. + Status DeleteDescriptorTarget(const std::shared_ptr& desc_array, + const std::string& field_name, int64_t row) const { + const auto& blob_col = arrow::internal::checked_cast( + *desc_array->GetFieldByName(field_name)); + std::string_view descriptor_bytes = blob_col.GetView(row); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr descriptor, + BlobDescriptor::Deserialize(descriptor_bytes.data(), descriptor_bytes.size())); + LocalFileSystem fs; + return fs.Delete(descriptor->Uri(), /*recursive=*/false); + } + + /// Rewrite the serialized BlobDescriptor at `row` of blob field `field_name` to reference + /// an offset beyond the end of the target file, so that a subsequent table write observes + /// a fetch failure that is not a missing file. `row` counts non-null rows. + Result> CorruptDescriptorOffset( + const std::shared_ptr& desc_array, const std::string& field_name, + int64_t row) const { + int64_t current_row = 0; + return TransformBlobFields( + desc_array, {field_name}, + [&](const std::string_view& descriptor_bytes, + arrow::LargeBinaryBuilder* builder) -> Status { + if (current_row++ != row) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + builder->Append(descriptor_bytes.data(), descriptor_bytes.size())); + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr descriptor, + BlobDescriptor::Deserialize(descriptor_bytes.data(), descriptor_bytes.size())); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr corrupted, + BlobDescriptor::Create(descriptor->Version(), descriptor->Uri(), + /*offset=*/1 << 20, descriptor->Length())); + auto corrupted_bytes = corrupted->Serialize(pool_); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + builder->Append(corrupted_bytes->data(), corrupted_bytes->size())); + return Status::OK(); + }); + } + struct BlobDescriptorPathRewrite { std::string table_path; std::vector table_relative_blob_dirs; @@ -560,6 +605,200 @@ TEST_P(BlobTableInteTest, TestAppendTableWriteWithBlobAsDescriptorFalse) { ASSERT_OK(ScanAndRead(table_path, schema->field_names(), write_array)); } +TEST_P(BlobTableInteTest, TestWriteNullOnMissingFile) { + // blob-write-null-on-missing-file=true: a descriptor whose file is gone at write time + // becomes a NULL blob element, while the row itself is still written and merged with + // the data file on read. + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::BLOB_WRITE_NULL_ON_MISSING_FILE, "true"}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + std::string raw_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, "blob_data_1"], + ["str_2", 2, "blob_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + + // Remove the file behind row 1's descriptor so the write observes a missing file. + ASSERT_OK(DeleteDescriptorTarget(desc_array, "blob", /*row=*/1)); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Both the main data file and the .blob file keep the full row count; the read path + // merges them back into rows where row 1's blob is NULL. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/2, /*expected_row_counts=*/{3, 3}, + /*expected_min_seqs=*/{1, 1}, /*expected_max_seqs=*/{1, 1}, + /*expected_first_row_ids=*/{0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "f1"}, std::vector{"blob"}}); + + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"blob"})); + + std::string expected_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, null], + ["str_2", 2, "blob_data_2"] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestMissingFileFailsWriteWhenWriteNullDisabled) { + // Without blob-write-null-on-missing-file, a missing descriptor file fails the write. + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + std::string raw_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, "blob_data_1"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + ASSERT_OK(DeleteDescriptorTarget(desc_array, "blob", /*row=*/1)); + + auto schema = arrow::schema(fields); + ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, schema->field_names(), {desc_array}), + "not exists"); +} + +TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailure) { + // blob-write-null-on-fetch-failure=true: a descriptor whose data cannot be fetched for a + // reason other than a missing file (here: offset beyond the end of the file) becomes a + // NULL blob element, while the row itself is still written and merged with the data file + // on read. + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "true"}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + std::string raw_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, "blob_data_1"], + ["str_2", 2, "blob_data_2"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + + // Point row 1's descriptor at an offset beyond the end of its file so the write observes + // a fetch failure that is not a missing file. + ASSERT_OK_AND_ASSIGN(desc_array, CorruptDescriptorOffset(desc_array, "blob", /*row=*/1)); + + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + // Both the main data file and the .blob file keep the full row count; the read path + // merges them back into rows where row 1's blob is NULL. + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/2, /*expected_row_counts=*/{3, 3}, + /*expected_min_seqs=*/{1, 1}, /*expected_max_seqs=*/{1, 1}, + /*expected_first_row_ids=*/{0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "f1"}, std::vector{"blob"}}); + + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"blob"})); + + std::string expected_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, null], + ["str_2", 2, "blob_data_2"] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailureKeepsMissingFileFailing) { + // blob-write-null-on-fetch-failure only converts non-NotExist open failures; a missing + // descriptor file still fails the write when only this option is enabled. + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32()), + BlobUtils::ToArrowField("blob", true)}; + + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::TARGET_FILE_SIZE, "700"}, + {Options::BUCKET, "-1"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::BLOB_AS_DESCRIPTOR, "true"}, + {Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, "true"}, + {Options::FILE_SYSTEM, "local"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + std::string raw_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, "blob_data_1"] + ])"; + auto raw_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto desc_array, ConvertRawBlobToDescriptor(raw_array, {"blob"})); + ASSERT_OK(DeleteDescriptorTarget(desc_array, "blob", /*row=*/1)); + + auto schema = arrow::schema(fields); + ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, schema->field_names(), {desc_array}), + "not exists"); +} + TEST_P(BlobTableInteTest, TestBasic) { CreateTable(); std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); From 968a3859ebcbf3e246db09add0af965e5bcc00d1 Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Fri, 17 Jul 2026 15:09:23 +0800 Subject: [PATCH 100/138] fix(commit): fix changelogRecordCount nullopt and add ut --- .../catalog/commit_table_request_test.cpp | 1 - src/paimon/core/manifest/manifest_entry.cpp | 16 + src/paimon/core/manifest/manifest_entry.h | 3 + .../manifest_entry_serializer_test.cpp | 28 +- .../operation/commit/conflict_detection.cpp | 209 ++++++-- .../operation/commit/conflict_detection.h | 35 +- .../commit/conflict_detection_test.cpp | 497 +++++++++++++++++- .../commit/row_tracking_commit_utils_test.cpp | 122 +++++ .../core/operation/file_store_commit_impl.cpp | 12 +- .../core/operation/file_store_commit_impl.h | 2 - .../operation/file_store_commit_impl_test.cpp | 6 +- src/paimon/core/snapshot.cpp | 8 +- src/paimon/core/snapshot_test.cpp | 16 +- src/paimon/core/tag/tag_test.cpp | 8 +- test/inte/data_evolution_table_test.cpp | 2 +- 15 files changed, 875 insertions(+), 90 deletions(-) diff --git a/src/paimon/core/catalog/commit_table_request_test.cpp b/src/paimon/core/catalog/commit_table_request_test.cpp index 7de6935a..deafee36 100644 --- a/src/paimon/core/catalog/commit_table_request_test.cpp +++ b/src/paimon/core/catalog/commit_table_request_test.cpp @@ -58,7 +58,6 @@ TEST(CommitTableRequestTest, TestSimple) { "baseManifestListSize": 291, "deltaManifestList": "manifest-list-3879e56f-2f27-49ae-a2f3-3dcbb8eb0beb-1", "deltaManifestListSize": 1342, - "changelogManifestList": null, "commitUser": "commit_user_1", "commitIdentifier": 9223372036854775807, "commitKind": "APPEND", diff --git a/src/paimon/core/manifest/manifest_entry.cpp b/src/paimon/core/manifest/manifest_entry.cpp index c19e32af..c0fbac0c 100644 --- a/src/paimon/core/manifest/manifest_entry.cpp +++ b/src/paimon/core/manifest/manifest_entry.cpp @@ -34,6 +34,22 @@ const std::shared_ptr& ManifestEntry::DataType() { return data_type; } +int64_t ManifestEntry::RecordCount(const std::vector& entries) { + int64_t record_count = 0; + for (const auto& entry : entries) { + record_count += entry.File()->row_count; + } + return record_count; +} + +std::optional ManifestEntry::NullableRecordCount( + const std::vector& entries) { + if (entries.empty()) { + return std::nullopt; + } + return RecordCount(entries); +} + int64_t ManifestEntry::RecordCountAdd(const std::vector& entries) { int64_t record_count = 0; for (const auto& entry : entries) { diff --git a/src/paimon/core/manifest/manifest_entry.h b/src/paimon/core/manifest/manifest_entry.h index 85e387ae..e39fa51c 100644 --- a/src/paimon/core/manifest/manifest_entry.h +++ b/src/paimon/core/manifest/manifest_entry.h @@ -39,6 +39,9 @@ namespace paimon { class ManifestEntry : public FileEntry { public: static const std::shared_ptr& DataType(); + static int64_t RecordCount(const std::vector& manifest_entries); + static std::optional NullableRecordCount( + const std::vector& manifest_entries); static int64_t RecordCountAdd(const std::vector& manifest_entries); static int64_t RecordCountDelete(const std::vector& manifest_entries); diff --git a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp index 3fa23466..2aa2db52 100644 --- a/src/paimon/core/manifest/manifest_entry_serializer_test.cpp +++ b/src/paimon/core/manifest/manifest_entry_serializer_test.cpp @@ -34,12 +34,12 @@ namespace paimon::test { class ManifestEntrySerializerTest : public testing::Test { private: - std::shared_ptr GetDataFileMeta() { + std::shared_ptr GetDataFileMeta(int64_t row_count) { return std::make_shared( - "some_file_name", 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), - SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/16, - /*max_seq_no=*/32, - /*schema_id=*/1, /*level=*/2, /*extra_files=*/std::vector>(), + "some_file_name", 1024, row_count, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + /*min_seq_no=*/16, /*max_seq_no=*/32, /*schema_id=*/1, /*level=*/2, + /*extra_files=*/std::vector>(), /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/3, /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, /*value_stats_cols=*/std::nullopt, /*external_path=*/std::optional(), @@ -49,9 +49,9 @@ class ManifestEntrySerializerTest : public testing::Test { TEST_F(ManifestEntrySerializerTest, TestToFromRow) { auto pool = GetDefaultPool(); std::vector entries = { - ManifestEntry(FileKind::Add(), BinaryRow::EmptyRow(), 0, 2, GetDataFileMeta()), + ManifestEntry(FileKind::Add(), BinaryRow::EmptyRow(), 0, 2, GetDataFileMeta(8)), ManifestEntry(FileKind::Add(), BinaryRowGenerator::GenerateRow({10}, pool.get()), 1, 2, - GetDataFileMeta())}; + GetDataFileMeta(8))}; ManifestEntrySerializer serializer(pool); for (const auto& entry : entries) { ASSERT_OK_AND_ASSIGN(auto row, serializer.ToRow(entry)); @@ -60,4 +60,18 @@ TEST_F(ManifestEntrySerializerTest, TestToFromRow) { ASSERT_EQ(entry.ToString(), result_entry.ToString()); } } + +TEST_F(ManifestEntrySerializerTest, TestNullableRecordCount) { + std::vector empty_entries; + ASSERT_FALSE(ManifestEntry::NullableRecordCount(empty_entries).has_value()); + + std::vector entries = { + ManifestEntry(FileKind::Add(), BinaryRow::EmptyRow(), 0, 2, GetDataFileMeta(3)), + ManifestEntry(FileKind::Delete(), BinaryRow::EmptyRow(), 0, 2, GetDataFileMeta(8))}; + ASSERT_EQ(11, ManifestEntry::RecordCount(entries)); + + std::optional record_count = ManifestEntry::NullableRecordCount(entries); + ASSERT_TRUE(record_count.has_value()); + ASSERT_EQ(11, record_count.value()); +} } // namespace paimon::test diff --git a/src/paimon/core/operation/commit/conflict_detection.cpp b/src/paimon/core/operation/commit/conflict_detection.cpp index 7ad14464..883cfb4f 100644 --- a/src/paimon/core/operation/commit/conflict_detection.cpp +++ b/src/paimon/core/operation/commit/conflict_detection.cpp @@ -32,6 +32,7 @@ #include "paimon/common/data/binary_row.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/range_helper.h" #include "paimon/core/deletionvectors/deletion_vectors_index_file.h" @@ -47,6 +48,8 @@ #include "paimon/core/operation/commit/row_id_column_conflict_checker.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/bucket_mode.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/utils/range.h" #include "paimon/utils/row_range_index.h" @@ -85,13 +88,18 @@ ConflictDetection::ConflictDetection(std::shared_ptr table_schema, std::shared_ptr snapshot_manager, std::shared_ptr manifest_list, std::shared_ptr manifest_file, - std::shared_ptr commit_scanner) + std::shared_ptr commit_scanner, + const std::string& commit_user, const std::string& table_name, + const std::shared_ptr& path_factory) : table_schema_(std::move(table_schema)), options_(options), snapshot_manager_(std::move(snapshot_manager)), manifest_list_(std::move(manifest_list)), manifest_file_(std::move(manifest_file)), - commit_scanner_(std::move(commit_scanner)) {} + commit_scanner_(std::move(commit_scanner)), + path_factory_(path_factory), + commit_user_(commit_user), + table_name_(table_name) {} void ConflictDetection::SetRowIdCheckFromSnapshot( const std::optional& row_id_check_from_snapshot) { @@ -109,6 +117,7 @@ Status ConflictDetection::CheckConflicts( const std::optional>& row_id_column_conflict_checker, const Snapshot::CommitKind& commit_kind) const { + std::string base_commit_user = latest_snapshot.CommitUser(); if (options_.DeletionVectorsEnabled() && ResolveBucketMode(options_.GetBucket(), table_schema_) == BucketMode::BUCKET_UNAWARE) { return Status::NotImplemented( @@ -117,18 +126,30 @@ Status ConflictDetection::CheckConflicts( std::vector all_entries = base_entries; all_entries.insert(all_entries.end(), delta_entries.begin(), delta_entries.end()); - PAIMON_RETURN_NOT_OK(CheckBucketKeepSame(all_entries, commit_kind)); + PAIMON_RETURN_NOT_OK(CheckBucketKeepSame(all_entries, commit_kind, base_commit_user, + base_entries, delta_entries)); // check the delta, it is important not to delete and add the same file. Since scan // relies on map for deduplication, this may result in the loss of this file std::vector merged_delta_entries; - PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(delta_entries, &merged_delta_entries)); + if (Status status = FileEntry::MergeEntries(delta_entries, &merged_delta_entries); + !status.ok()) { + return Status::Invalid( + BuildConflictMessage("File deletion conflicts detected! Give up committing.", + base_commit_user, base_entries, delta_entries, status.ToString())); + } std::vector merged_entries; // merge manifest entries and also check if the files we want to delete are still there - PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(all_entries, &merged_entries)); - PAIMON_RETURN_NOT_OK(CheckDeleteInEntries(merged_entries)); - PAIMON_RETURN_NOT_OK(CheckKeyRange(merged_entries)); + if (Status status = FileEntry::MergeEntries(all_entries, &merged_entries); !status.ok()) { + return Status::Invalid( + BuildConflictMessage("File deletion conflicts detected! Give up committing.", + base_commit_user, base_entries, delta_entries, status.ToString())); + } + PAIMON_RETURN_NOT_OK( + CheckDeleteInEntries(merged_entries, base_commit_user, base_entries, delta_entries)); + PAIMON_RETURN_NOT_OK( + CheckKeyRange(merged_entries, base_commit_user, base_entries, delta_entries)); if (commit_kind != Snapshot::CommitKind::Compact()) { PAIMON_RETURN_NOT_OK( CheckRowIdExistence(base_entries, delta_entries, latest_snapshot.NextRowId())); @@ -158,8 +179,10 @@ bool ConflictDetection::ShouldBeOverwriteCommit( return false; } -Status ConflictDetection::CheckBucketKeepSame(const std::vector& all_entries, - const Snapshot::CommitKind& commit_kind) const { +Status ConflictDetection::CheckBucketKeepSame( + const std::vector& all_entries, const Snapshot::CommitKind& commit_kind, + const std::string& base_commit_user, const std::vector& base_entries, + const std::vector& delta_entries) const { if (commit_kind == Snapshot::CommitKind::Overwrite()) { return Status::OK(); } @@ -180,7 +203,8 @@ Status ConflictDetection::CheckBucketKeepSame(const std::vector& continue; } - return BucketNumMismatch(entry.Partition(), entry.TotalBuckets(), iter->second); + return TotalBucketsChanged(entry.Partition(), entry.TotalBuckets(), iter->second, + base_commit_user, base_entries, delta_entries); } MarkBucketCheckedPartitions(total_buckets); @@ -223,10 +247,91 @@ Status ConflictDetection::CheckSameBucketByTotalBuckets( Status ConflictDetection::BucketNumMismatch(const BinaryRow& partition, int32_t num_buckets, int32_t previous_num_buckets) const { + std::string part_info; + if (table_schema_->PartitionKeys().empty()) { + part_info = "table"; + } else { + PAIMON_ASSIGN_OR_RAISE(std::string partition_string, + path_factory_->GetPartitionString(partition)); + part_info = fmt::format("partition {{{}}}", partition_string); + } return Status::Invalid(fmt::format( + "Try to write {} with a new bucket num {}, but the previous bucket num is {}. Please " + "switch to batch mode, and perform INSERT OVERWRITE to rescale current data layout first.", + part_info, num_buckets, previous_num_buckets)); +} + +Status ConflictDetection::TotalBucketsChanged( + const BinaryRow& partition, int32_t num_buckets, int32_t previous_num_buckets, + const std::string& base_commit_user, const std::vector& base_entries, + const std::vector& delta_entries) const { + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(table_schema_->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema_->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::string partition_string, + BinaryRowPartitionComputer::PartToSimpleString(partition_schema, partition, "-", 200, + /*legacy_partition_name_enabled=*/false)); + std::string message = fmt::format( "Total buckets of partition {} changed from {} to {} without overwrite. Give up " "committing.", - partition.ToString(), previous_num_buckets, num_buckets)); + partition_string, previous_num_buckets, num_buckets); + return Status::Invalid( + BuildConflictMessage(message, base_commit_user, base_entries, delta_entries)); +} + +std::string ConflictDetection::BuildConflictMessage(const std::string& message, + const std::string& base_commit_user, + const std::vector& base_entries, + const std::vector& delta_entries, + const std::string& cause) const { + static constexpr const char* kPossibleCauses = + "Don't panic!\n" + "Conflicts during commits are normal and this failure is intended to resolve the " + "conflicts.\n" + "Conflicts are mainly caused by the following scenarios:\n" + "1. Multiple jobs are writing into the same partition at the same time, or you use " + "STATEMENT SET to execute multiple INSERT statements into the same Paimon table.\n" + " You'll probably see different base commit user and current commit user below.\n" + " You can use dedicated compaction job to support multiple writing.\n" + "2. You're recovering from an old savepoint, or you're creating multiple jobs from a " + "savepoint.\n" + " The job will fail continuously in this scenario to protect metadata from corruption.\n" + " You can either recover from the latest savepoint, or you can revert the table to the " + "snapshot corresponding to the old savepoint."; + + constexpr size_t kMaxEntry = 50; + auto join_entries = [](const std::vector& entries, + size_t max_entry) -> std::string { + std::string joined; + size_t limit = std::min(entries.size(), max_entry); + for (size_t i = 0; i < limit; ++i) { + if (i > 0) { + joined += "\n"; + } + joined += entries[i].ToString(); + } + return joined; + }; + + std::string commit_user_string = fmt::format( + "Base commit user is: {}; Current commit user is: {}", base_commit_user, commit_user_); + std::string base_entries_string = "Base entries are:\n" + join_entries(base_entries, kMaxEntry); + std::string changes_string = "Changes are:\n" + join_entries(delta_entries, kMaxEntry); + + std::string result = fmt::format("{}\n\n{}\n\n{}\n\n{}\n\n{}", message, kPossibleCauses, + commit_user_string, base_entries_string, changes_string); + if (base_entries.size() > kMaxEntry || delta_entries.size() > kMaxEntry) { + result += + "\n\nThe entry list above are not fully displayed, please refer to logs for more " + "information."; + } + if (!cause.empty()) { + result += "\n\nCaused by: " + cause; + } + return result; } void ConflictDetection::MarkBucketCheckedPartitions( @@ -244,18 +349,27 @@ void ConflictDetection::MarkBucketCheckedPartitions( } Status ConflictDetection::CheckDeleteInEntries( - const std::vector& merged_entries) const { + const std::vector& merged_entries, const std::string& base_commit_user, + const std::vector& base_entries, + const std::vector& delta_entries) const { for (const auto& entry : merged_entries) { if (entry.Kind() == FileKind::Delete()) { - return Status::Invalid(fmt::format( - "Trying to delete file {} which is not previously added.", entry.FileName())); + std::string message = fmt::format( + "File deletion conflicts detected! Give up committing. Trying to delete file {} " + "for table {} which is not previously added.", + entry.FileName(), table_name_); + return Status::Invalid( + BuildConflictMessage(message, base_commit_user, base_entries, delta_entries)); } } return Status::OK(); } -Status ConflictDetection::CheckKeyRange(const std::vector& merged_entries) const { +Status ConflictDetection::CheckKeyRange(const std::vector& merged_entries, + const std::string& base_commit_user, + const std::vector& base_entries, + const std::vector& delta_entries) const { if (table_schema_->PrimaryKeys().empty()) { return Status::OK(); } @@ -290,9 +404,18 @@ Status ConflictDetection::CheckKeyRange(const std::vector& merged const ManifestEntry& a = entries[i]; const ManifestEntry& b = entries[i + 1]; if (key_comparator->CompareTo(a.MaxKey(), b.MinKey()) >= 0) { - return Status::Invalid(fmt::format( - "LSM conflicts detected! Give up committing. Conflict files are {} and {}.", - a.FileName(), b.FileName())); + PAIMON_ASSIGN_OR_RAISE(std::string a_partition_string, + path_factory_->GetPartitionString(a.Partition())); + PAIMON_ASSIGN_OR_RAISE(std::string b_partition_string, + path_factory_->GetPartitionString(b.Partition())); + std::string message = fmt::format( + "LSM conflicts detected! Give up committing. Conflict files are:\n" + "{}, bucket {}, level {}, file {}\n" + "{}, bucket {}, level {}, file {}", + a_partition_string, a.Bucket(), a.Level(), a.FileName(), b_partition_string, + b.Bucket(), b.Level(), b.FileName()); + return Status::Invalid( + BuildConflictMessage(message, base_commit_user, base_entries, delta_entries)); } } } @@ -349,7 +472,9 @@ Status ConflictDetection::CheckRowIdExistence(const std::vector& if (!exists) { return Status::Invalid(fmt::format( "Row ID existence conflict: file '{}' references firstRowId={}, rowCount={} in " - "bucket {}, but no matching file exists in the current snapshot.", + "bucket {}, but no matching file exists in the current snapshot. The referenced " + "file may have been rewritten by a concurrent compaction or removed by an " + "overwrite.", entry.FileName(), entry.File()->first_row_id.value(), entry.File()->row_count, entry.Bucket())); } @@ -413,9 +538,17 @@ Status ConflictDetection::CheckDataFileRowIdRangeConflicts( PAIMON_ASSIGN_OR_RAISE(bool all_data_ranges_same, range_helper.AreAllRangesSame(data_file_group)); if (!all_data_ranges_same) { - return Status::Invalid( - "For Data Evolution table, multiple MERGE INTO/COMPACT operations have " - "encountered row-id range conflicts."); + std::string data_files_str; + for (size_t i = 0; i < data_file_group.size(); ++i) { + if (i > 0) { + data_files_str += ", "; + } + data_files_str += data_file_group[i].ToString(); + } + return Status::Invalid(fmt::format( + "For Data Evolution table, multiple 'MERGE INTO' and 'COMPACT' operations have " + "encountered conflicts, data files: [{}]", + data_files_str)); } } @@ -454,10 +587,24 @@ Status ConflictDetection::CheckDedicatedFileRowIdRangeConflicts( std::string conflict_reason = intersecting_ranges.size() > 1 ? "spans multiple data file ranges" : "is not covered by one data file range"; + std::string intersecting_files_str; + bool first = true; + for (const ManifestEntry& data_file : data_files) { + int64_t data_from = data_file.File()->first_row_id.value(); + int64_t data_to = data_from + data_file.File()->row_count - 1; + if (data_from <= dedicated_range.to && dedicated_range.from <= data_to) { + if (!first) { + intersecting_files_str += ", "; + } + intersecting_files_str += data_file.ToString(); + first = false; + } + } return Status::Invalid(fmt::format( - "For Data Evolution table, multiple MERGE INTO/COMPACT operations have " - "encountered row-id range conflicts, dedicated file '{}' range {} {}.", - dedicated_file.FileName(), dedicated_range.ToString(), conflict_reason)); + "For Data Evolution table, multiple 'MERGE INTO' and 'COMPACT' operations have " + "encountered conflicts, dedicated file {} {} {}: [{}]", + dedicated_file.ToString(), dedicated_range.ToString(), conflict_reason, + intersecting_files_str)); } } @@ -528,9 +675,9 @@ Status ConflictDetection::CheckForRowIdFromSnapshot( row_id_column_conflict_checker.value()->ConflictsWith(history_entry.File())); if (conflicts) { return Status::Invalid( - "For Data Evolution table, multiple MERGE INTO operations have " - "encountered conflicts while checking row-id history from " - "snapshot."); + "For Data Evolution table, multiple 'MERGE INTO' operations have " + "encountered conflicts, updating the same file, which can render some " + "updates ineffective."); } } } @@ -585,7 +732,8 @@ Status ConflictDetection::CheckGlobalIndexRowIdExistence( if (group_iter == range_index_by_group.end()) { return Status::Invalid(fmt::format( "Global index row ID existence conflict: index file '{}' references row range {}, " - "but this range is not fully covered by current data files.", + "but this range is not fully covered by current data files. The referenced row " + "IDs may have been reassigned or removed by a concurrent commit.", index_entry.index_file->FileName(), Range(index_entry.index_file->GetGlobalIndexMeta().value().row_range_start, index_entry.index_file->GetGlobalIndexMeta().value().row_range_end) @@ -602,7 +750,8 @@ Status ConflictDetection::CheckGlobalIndexRowIdExistence( if (!covered) { return Status::Invalid(fmt::format( "Global index row ID existence conflict: index file '{}' references row range {}, " - "but this range is not fully covered by current data files.", + "but this range is not fully covered by current data files. The referenced row " + "IDs may have been reassigned or removed by a concurrent commit.", index_entry.index_file->FileName(), index_range.ToString())); } } diff --git a/src/paimon/core/operation/commit/conflict_detection.h b/src/paimon/core/operation/commit/conflict_detection.h index 803da2cb..de2793e2 100644 --- a/src/paimon/core/operation/commit/conflict_detection.h +++ b/src/paimon/core/operation/commit/conflict_detection.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -38,6 +39,7 @@ namespace paimon { class ManifestEntry; struct IndexManifestEntry; class CommitScanner; +class FileStorePathFactory; class ManifestFile; class ManifestList; class RowIdColumnConflictChecker; @@ -51,7 +53,9 @@ class ConflictDetection { std::shared_ptr snapshot_manager, std::shared_ptr manifest_list, std::shared_ptr manifest_file, - std::shared_ptr commit_scanner); + std::shared_ptr commit_scanner, const std::string& commit_user, + const std::string& table_name, + const std::shared_ptr& path_factory); Status CheckConflicts(const Snapshot& latest_snapshot, const std::vector& base_entries, @@ -78,17 +82,37 @@ class ConflictDetection { private: Status CheckBucketKeepSame(const std::vector& all_entries, - const Snapshot::CommitKind& commit_kind) const; + const Snapshot::CommitKind& commit_kind, + const std::string& base_commit_user, + const std::vector& base_entries, + const std::vector& delta_entries) const; Status BucketNumMismatch(const BinaryRow& partition, int32_t num_buckets, int32_t previous_num_buckets) const; + Status TotalBucketsChanged(const BinaryRow& partition, int32_t num_buckets, + int32_t previous_num_buckets, const std::string& base_commit_user, + const std::vector& base_entries, + const std::vector& delta_entries) const; + + std::string BuildConflictMessage(const std::string& message, + const std::string& base_commit_user, + const std::vector& base_entries, + const std::vector& delta_entries, + const std::string& cause = "") const; + void MarkBucketCheckedPartitions( const std::unordered_map& total_buckets) const; - Status CheckDeleteInEntries(const std::vector& merged_entries) const; + Status CheckDeleteInEntries(const std::vector& merged_entries, + const std::string& base_commit_user, + const std::vector& base_entries, + const std::vector& delta_entries) const; - Status CheckKeyRange(const std::vector& merged_entries) const; + Status CheckKeyRange(const std::vector& merged_entries, + const std::string& base_commit_user, + const std::vector& base_entries, + const std::vector& delta_entries) const; Status CheckRowIdExistence(const std::vector& base_entries, const std::vector& delta_entries, @@ -124,6 +148,9 @@ class ConflictDetection { std::shared_ptr manifest_list_; std::shared_ptr manifest_file_; std::shared_ptr commit_scanner_; + std::shared_ptr path_factory_; + std::string commit_user_; + std::string table_name_; mutable LinkedHashMap same_bucket_checked_partitions_; }; diff --git a/src/paimon/core/operation/commit/conflict_detection_test.cpp b/src/paimon/core/operation/commit/conflict_detection_test.cpp index 4da37304..28e4fed1 100644 --- a/src/paimon/core/operation/commit/conflict_detection_test.cpp +++ b/src/paimon/core/operation/commit/conflict_detection_test.cpp @@ -28,6 +28,7 @@ #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/utils/path_util.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" #include "paimon/core/index/global_index_meta.h" #include "paimon/core/index/index_file_meta.h" #include "paimon/core/io/data_file_meta.h" @@ -36,6 +37,7 @@ #include "paimon/core/manifest/manifest_entry.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/stats/simple_stats.h" +#include "paimon/core/utils/file_store_path_factory.h" #include "paimon/data/timestamp.h" #include "paimon/defs.h" #include "paimon/memory/bytes.h" @@ -68,6 +70,29 @@ Snapshot MakeSnapshot(const Snapshot::CommitKind& commit_kind) { /*next_row_id=*/std::nullopt); } +Snapshot MakeSnapshotWithNextRowId(const Snapshot::CommitKind& commit_kind, + const std::optional& next_row_id) { + return Snapshot( + /*id=*/1, + /*schema_id=*/1, + /*base_manifest_list=*/"base-manifest-list", + /*base_manifest_list_size=*/std::nullopt, + /*delta_manifest_list=*/"delta-manifest-list", + /*delta_manifest_list_size=*/std::nullopt, + /*changelog_manifest_list=*/std::nullopt, + /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, + /*commit_user=*/"test-user", + /*commit_identifier=*/1, commit_kind, + /*time_millis=*/0, + /*total_record_count=*/0, + /*delta_record_count=*/0, + /*changelog_record_count=*/std::nullopt, + /*watermark=*/std::nullopt, + /*statistics=*/std::nullopt, + /*properties=*/std::nullopt, next_row_id); +} + Status CheckConflicts(const ConflictDetection& detection, const std::vector& base_entries, const std::vector& delta_entries, @@ -87,6 +112,17 @@ Status CheckConflicts(const ConflictDetection& detection, /*row_id_column_conflict_checker=*/std::nullopt, commit_kind); } +Status CheckConflictsWithNextRowId(const ConflictDetection& detection, + const std::vector& base_entries, + const std::vector& delta_entries, + const std::optional& next_row_id, + const Snapshot::CommitKind& commit_kind) { + return detection.CheckConflicts(MakeSnapshotWithNextRowId(commit_kind, next_row_id), + base_entries, delta_entries, + /*delta_index_entries=*/{}, + /*row_id_column_conflict_checker=*/std::nullopt, commit_kind); +} + } // namespace class ConflictDetectionTest : public testing::Test { @@ -97,6 +133,19 @@ class ConflictDetectionTest : public testing::Test { } protected: + std::shared_ptr CreatePathFactory( + const std::vector& partition_keys) const { + EXPECT_OK_AND_ASSIGN( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + /*root=*/"/tmp/conflict_detection_test", arrow::schema(fields_), partition_keys, + /*default_part_value=*/"__DEFAULT_PARTITION__", /*identifier=*/"orc", + /*data_file_prefix=*/"data-", /*legacy_partition_name_enabled=*/false, + /*external_paths=*/{}, /*global_index_external_path=*/std::nullopt, + /*index_file_in_data_file_dir=*/false, GetDefaultPool())); + return path_factory; + } + ManifestEntry CreateManifestEntry(const std::string& file_name, const FileKind& kind) const { int32_t arity = 1; BinaryRow row(arity); @@ -153,12 +202,28 @@ class ConflictDetectionTest : public testing::Test { const BinaryRow& partition, int32_t bucket, int64_t row_range_start, int64_t row_range_end) const { + return CreateGlobalIndexEntry(file_name, partition, bucket, FileKind::Add(), + row_range_start, row_range_end); + } + + IndexManifestEntry CreateGlobalIndexEntry(const std::string& file_name, + const BinaryRow& partition, int32_t bucket, + const FileKind& kind, int64_t row_range_start, + int64_t row_range_end) const { GlobalIndexMeta global_index_meta(row_range_start, row_range_end, /*index_field_id=*/1, /*extra_field_ids=*/std::nullopt, std::make_shared("meta", GetDefaultPool().get())); auto index_file_meta = std::make_shared( "HASH", file_name, /*file_size=*/100, /*row_count=*/5, /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index_meta); + return IndexManifestEntry(kind, partition, bucket, index_file_meta); + } + + IndexManifestEntry CreateDvIndexEntry(const std::string& file_name, const BinaryRow& partition, + int32_t bucket) const { + auto index_file_meta = std::make_shared( + DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, file_name, /*file_size=*/100, + /*row_count=*/1, /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt); return IndexManifestEntry(FileKind::Add(), partition, bucket, index_file_meta); } @@ -179,7 +244,10 @@ TEST_F(ConflictDetectionTest, TestFileDeletionConflicts) { TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); { std::vector base_entries; @@ -220,7 +288,10 @@ TEST_F(ConflictDetectionTest, TestGlobalIndexRowIdExistenceConflicts) { /*primary_keys=*/{}, /*options=*/{})); ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); const BinaryRow partition = CreateIntRow(10); std::vector base_entries; @@ -253,7 +324,10 @@ TEST_F(ConflictDetectionTest, TestDedicatedStorageRowIdRangeConflicts) { /*primary_keys=*/{}, /*options=*/{})); ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); const BinaryRow partition = CreateIntRow(10); std::vector base_entries; @@ -268,7 +342,7 @@ TEST_F(ConflictDetectionTest, TestDedicatedStorageRowIdRangeConflicts) { /*row_count=*/10)); ASSERT_NOK_WITH_MSG(CheckConflicts(detection, base_entries, out_of_range_dedicated_entries, Snapshot::CommitKind::Compact()), - "row-id range conflicts"); + "is not covered by one data file range"); std::vector contained_dedicated_entries; contained_dedicated_entries.push_back(CreateManifestEntryWithFirstRowId( @@ -311,7 +385,10 @@ TEST_F(ConflictDetectionTest, TestBucketKeepSame) { const BinaryRow partition = CreateIntRow(10); { - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); std::vector base_entries; base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), DataFileMeta::EmptyMinKey(), @@ -326,7 +403,10 @@ TEST_F(ConflictDetectionTest, TestBucketKeepSame) { ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); } { - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); std::vector base_entries; base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), DataFileMeta::EmptyMinKey(), @@ -343,7 +423,10 @@ TEST_F(ConflictDetectionTest, TestBucketKeepSame) { "Total buckets of partition"); } { - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); std::vector base_entries; base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), DataFileMeta::EmptyMinKey(), @@ -358,7 +441,10 @@ TEST_F(ConflictDetectionTest, TestBucketKeepSame) { ASSERT_OK(CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append())); } { - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); std::vector base_entries; base_entries.push_back(CreateManifestEntry("base", partition, FileKind::Add(), DataFileMeta::EmptyMinKey(), @@ -381,7 +467,10 @@ TEST_F(ConflictDetectionTest, TestBucketKeepSameHelpers) { TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); const BinaryRow partition = CreateIntRow(10); std::vector changes; @@ -408,11 +497,13 @@ TEST_F(ConflictDetectionTest, TestBucketKeepSameHelpers) { ASSERT_OK(detection.CollectUncheckedBucketPartitions(changes, &cached_total_buckets)); ASSERT_TRUE(cached_total_buckets.empty()); - ConflictDetection mismatch_detection(table_schema, core_options, nullptr, nullptr, nullptr, - nullptr); + ConflictDetection mismatch_detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + CreatePathFactory({"f1"})); ASSERT_NOK_WITH_MSG( mismatch_detection.CheckSameBucketByTotalBuckets(expected_total_buckets, {{partition, 2}}), - "Total buckets of partition"); + "new bucket num"); } TEST_F(ConflictDetectionTest, TestCollectUncheckedBucketPartitionsMismatch) { @@ -421,7 +512,10 @@ TEST_F(ConflictDetectionTest, TestCollectUncheckedBucketPartitionsMismatch) { TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + CreatePathFactory({"f1"})); const BinaryRow partition = CreateIntRow(10); std::vector changes; @@ -435,8 +529,11 @@ TEST_F(ConflictDetectionTest, TestCollectUncheckedBucketPartitionsMismatch) { /*bucket=*/1, /*total_buckets=*/4)); std::unordered_map total_buckets; + // Verify the partition value is rendered via FileStorePathFactory::GetPartitionString + // (i.e. "partition {f1=10...}"), not dropped. Before the fmt escaping fix this printed a + // literal "partition {}" with the partition string silently discarded. ASSERT_NOK_WITH_MSG(detection.CollectUncheckedBucketPartitions(changes, &total_buckets), - "Total buckets of partition"); + "partition {f1=10"); } TEST_F(ConflictDetectionTest, TestBucketKeepSameCacheEviction) { @@ -445,7 +542,10 @@ TEST_F(ConflictDetectionTest, TestBucketKeepSameCacheEviction) { TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, /*options=*/{})); ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); const int32_t total_buckets = 4; for (int32_t value = 0; value <= 1000; ++value) { @@ -480,7 +580,10 @@ TEST_F(ConflictDetectionTest, TestDeletionVectorsNotSupportedWithBucketUnawareMo ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({{Options::BUCKET, "0"}, {Options::DELETION_VECTORS_ENABLED, "true"}})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); ASSERT_NOK_WITH_MSG(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{}, Snapshot::CommitKind::Append()), @@ -496,7 +599,10 @@ TEST_F(ConflictDetectionTest, ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({{Options::BUCKET, "-1"}, {Options::DELETION_VECTORS_ENABLED, "true"}})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); ASSERT_NOK_WITH_MSG(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{}, Snapshot::CommitKind::Append()), @@ -514,7 +620,10 @@ TEST_F(ConflictDetectionTest, TestDeletionVectorsAllowedWithResolvedDynamicBucke ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({{Options::BUCKET, "-1"}, {Options::DELETION_VECTORS_ENABLED, "true"}})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); ASSERT_OK(CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{}, Snapshot::CommitKind::Append())); @@ -529,7 +638,10 @@ TEST_F(ConflictDetectionTest, TestCheckLsmKeyRangeConflict) { TableSchema::Create(/*schema_id=*/0, arrow::schema(fields), /*partition_keys=*/{"f1"}, /*primary_keys=*/{"f1", "f0"}, {{Options::BUCKET, "4"}})); ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({{Options::BUCKET, "4"}})); - ConflictDetection detection(table_schema, core_options, nullptr, nullptr, nullptr, nullptr); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + CreatePathFactory({"f1"})); const BinaryRow partition = CreateIntRow(10); { @@ -587,4 +699,351 @@ TEST_F(ConflictDetectionTest, TestCheckLsmKeyRangeConflict) { } } +TEST_F(ConflictDetectionTest, TestShouldBeOverwriteCommit) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); + + std::vector add_only_entries; + add_only_entries.push_back(CreateManifestEntry("f1", FileKind::Add())); + add_only_entries.push_back(CreateManifestEntry("f2", FileKind::Add())); + ASSERT_FALSE(detection.ShouldBeOverwriteCommit(add_only_entries, /*append_index_files=*/{})); + + ASSERT_FALSE(detection.ShouldBeOverwriteCommit(/*append_table_files=*/{}, + /*append_index_files=*/{})); + + std::vector delete_entries; + delete_entries.push_back(CreateManifestEntry("f1", FileKind::Delete())); + delete_entries.push_back(CreateManifestEntry("f2", FileKind::Add())); + ASSERT_TRUE(detection.ShouldBeOverwriteCommit(delete_entries, /*append_index_files=*/{})); + + const BinaryRow partition = CreateIntRow(10); + std::vector dv_index_files; + dv_index_files.push_back(CreateDvIndexEntry("dv1", partition, /*bucket=*/0)); + ASSERT_TRUE(detection.ShouldBeOverwriteCommit(/*append_table_files=*/{}, dv_index_files)); +} + +TEST_F(ConflictDetectionTest, TestCheckRowIdExistenceNormalFiles) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); + const BinaryRow partition = CreateIntRow(10); + + // No conflict: delta references the same row-id range as an existing data file. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100)); + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100)); + ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/100, Snapshot::CommitKind::Append())); + } + + // Base data file removed: no matching range remains. + { + std::vector base_entries; + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100)); + ASSERT_NOK_WITH_MSG( + CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/100, Snapshot::CommitKind::Append()), + "Row ID existence conflict"); + } + + // Base data file rewritten with a different range. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f2", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/200)); + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100)); + ASSERT_NOK_WITH_MSG( + CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/200, Snapshot::CommitKind::Append()), + "Row ID existence conflict"); + } + + // Normal file must match exactly one data range, not span adjacent files. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/2)); + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f2", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/2, /*row_count=*/2)); + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/4)); + ASSERT_NOK_WITH_MSG( + CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/4, Snapshot::CommitKind::Append()), + "Row ID existence conflict"); + } + + // Newly appended files (firstRowId >= nextRowId) are skipped. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100)); + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100)); + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "new1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/100, + /*row_count=*/50)); + ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/100, Snapshot::CommitKind::Append())); + } + + // Files without a pre-assigned first row id are skipped. + { + std::vector base_entries; + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntry("f1", partition, FileKind::Add())); + ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/100, Snapshot::CommitKind::Append())); + } + + // A null nextRowId disables row-id existence checking. + { + std::vector base_entries; + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/100)); + ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/std::nullopt, + Snapshot::CommitKind::Append())); + } +} + +TEST_F(ConflictDetectionTest, TestCheckRowIdExistenceDedicatedFiles) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); + const BinaryRow partition = CreateIntRow(10); + + // Dedicated file contained within a single data range is allowed. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/4)); + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/2)); + ASSERT_OK(CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/4, Snapshot::CommitKind::Append())); + } + + // Dedicated file spanning adjacent data files is rejected. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/2)); + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f2", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/2, /*row_count=*/2)); + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/4)); + ASSERT_NOK_WITH_MSG( + CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/4, Snapshot::CommitKind::Append()), + "Row ID existence conflict"); + } + + // Dedicated file whose range is not covered by one data file is rejected. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/2)); + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/3)); + ASSERT_NOK_WITH_MSG( + CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/3, Snapshot::CommitKind::Append()), + "Row ID existence conflict"); + } + + // Base dedicated files are ignored when building existing data ranges. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "old.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/2)); + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/2)); + ASSERT_NOK_WITH_MSG( + CheckConflictsWithNextRowId(detection, base_entries, delta_entries, + /*next_row_id=*/2, Snapshot::CommitKind::Append()), + "Row ID existence conflict"); + } +} + +TEST_F(ConflictDetectionTest, TestGlobalIndexRowIdExistenceByPartitionAndBucket) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); + + const BinaryRow partition0 = CreateIntRow(0); + const BinaryRow partition1 = CreateIntRow(1); + + // Index in partition0 but data lives in partition1 -> conflict. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId("f1", partition1, FileKind::Add(), + /*bucket=*/0, /*first_row_id=*/0, + /*row_count=*/150)); + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, /*delta_entries=*/{}, + {CreateGlobalIndexEntry("idx", partition0, /*bucket=*/0, + /*row_range_start=*/0, /*row_range_end=*/149)}, + Snapshot::CommitKind::Append()), + "Global index row ID existence conflict"); + } + + // Index in bucket 0 but data lives in bucket 1 -> conflict. + { + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId("f1", partition0, FileKind::Add(), + /*bucket=*/1, /*first_row_id=*/0, + /*row_count=*/150)); + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, /*delta_entries=*/{}, + {CreateGlobalIndexEntry("idx", partition0, /*bucket=*/0, + /*row_range_start=*/0, /*row_range_end=*/149)}, + Snapshot::CommitKind::Append()), + "Global index row ID existence conflict"); + } +} + +TEST_F(ConflictDetectionTest, TestGlobalIndexRowIdExistenceSkipsDeleteIndexEntry) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); + const BinaryRow partition = CreateIntRow(10); + + ASSERT_OK( + CheckConflicts(detection, /*base_entries=*/{}, /*delta_entries=*/{}, + {CreateGlobalIndexEntry("idx", partition, /*bucket=*/0, FileKind::Delete(), + /*row_range_start=*/0, /*row_range_end=*/149)}, + Snapshot::CommitKind::Append())); +} + +TEST_F(ConflictDetectionTest, TestCheckRowIdRangeConflictsAllowsAdjacentDataFiles) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); + const BinaryRow partition = CreateIntRow(10); + + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/2)); + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f2", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/2, /*row_count=*/2)); + ASSERT_OK(CheckConflicts(detection, base_entries, /*delta_entries=*/{}, + Snapshot::CommitKind::Compact())); +} + +TEST_F(ConflictDetectionTest, TestCheckRowIdRangeConflictsAllowsDedicatedFileCoveredByOneDataFile) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, + CoreOptions::FromMap({{Options::DATA_EVOLUTION_ENABLED, "true"}})); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); + const BinaryRow partition = CreateIntRow(10); + + std::vector base_entries; + base_entries.push_back(CreateManifestEntryWithFirstRowId( + "f1", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/0, /*row_count=*/4)); + std::vector delta_entries; + delta_entries.push_back(CreateManifestEntryWithFirstRowId( + "p1.blob", partition, FileKind::Add(), /*bucket=*/0, /*first_row_id=*/1, /*row_count=*/2)); + ASSERT_OK( + CheckConflicts(detection, base_entries, delta_entries, Snapshot::CommitKind::Compact())); +} + +TEST_F(ConflictDetectionTest, TestConflictMessageTruncatesLargeEntryList) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields_), /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, /*options=*/{})); + ASSERT_OK_AND_ASSIGN(CoreOptions core_options, CoreOptions::FromMap({})); + ConflictDetection detection(table_schema, core_options, /*snapshot_manager=*/nullptr, + /*manifest_list=*/nullptr, /*manifest_file=*/nullptr, + /*commit_scanner=*/nullptr, "test_user", "test_table", + /*path_factory=*/nullptr); + + // kMaxEntry in conflict_detection.cpp is 50. Exceed it so the conflict message appends the + // "not fully displayed" truncation hint. Each lone DELETE (no matching base add) is a + // deletion conflict, so the delta entry list drives the message. + constexpr int kEntryCount = 60; + std::vector base_entries; + std::vector changes; + changes.reserve(kEntryCount); + for (int i = 0; i < kEntryCount; ++i) { + changes.push_back(CreateManifestEntry("delete-" + std::to_string(i), FileKind::Delete())); + } + + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "which is not previously added"); + ASSERT_NOK_WITH_MSG( + CheckConflicts(detection, base_entries, changes, Snapshot::CommitKind::Append()), + "not fully displayed"); +} + } // namespace paimon::test diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp index 55be4c67..02e79fc5 100644 --- a/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp +++ b/src/paimon/core/operation/commit/row_tracking_commit_utils_test.cpp @@ -26,6 +26,7 @@ #include "gtest/gtest.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/table/special_fields.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_kind.h" #include "paimon/core/manifest/file_source.h" @@ -64,6 +65,26 @@ class RowTrackingCommitUtilsTest : public testing::Test { return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1, file_meta); } + + ManifestEntry CreateEntryWithFirstRowId( + const std::string& file_name, int64_t row_count, int64_t min_seq_number, + int64_t max_seq_number, const std::optional& file_source, + const std::optional>& write_cols, + const std::optional& first_row_id) const { + auto file_meta = std::make_shared( + file_name, /*file_size=*/row_count, row_count, DataFileMeta::EmptyMinKey(), + DataFileMeta::EmptyMaxKey(), SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), + min_seq_number, max_seq_number, + /*schema_id=*/1, /*level=*/0, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, file_source, + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, first_row_id, write_cols); + return ManifestEntry(FileKind::Add(), CreateIntRow(1), /*bucket=*/0, /*total_buckets=*/1, + file_meta); + } }; TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingStampsSequence) { @@ -216,4 +237,105 @@ TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingReassignsOnRetryWithAdva EXPECT_EQ(std::nullopt, input[0].File()->first_row_id); } +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingEmptyInput) { + std::vector input; + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/42, input)); + EXPECT_TRUE(assigned.assigned_entries.empty()); + EXPECT_EQ(42, assigned.next_row_id_start); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingSkipsFilesWithRowIdColumn) { + std::vector input; + input.push_back( + CreateEntry("has-row-id-column", /*row_count=*/10, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{std::string(SpecialFields::RowId().Name())})); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input)); + + ASSERT_EQ(1u, assigned.assigned_entries.size()); + // Sequence numbers are still stamped for a new file. + EXPECT_EQ(100, assigned.assigned_entries[0].File()->min_sequence_number); + EXPECT_EQ(100, assigned.assigned_entries[0].File()->max_sequence_number); + // But a file already carrying the row-id column must not be assigned a first row id. + EXPECT_EQ(std::nullopt, assigned.assigned_entries[0].File()->first_row_id); + EXPECT_EQ(0, assigned.next_row_id_start); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingKeepsExistingFirstRowId) { + std::vector input; + input.push_back(CreateEntryWithFirstRowId("already-assigned", /*row_count=*/10, + /*min_seq_number=*/0, /*max_seq_number=*/0, + FileSource::Append(), std::vector{"f0"}, + /*first_row_id=*/500)); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input)); + + ASSERT_EQ(1u, assigned.assigned_entries.size()); + // The existing first row id is preserved and start is not advanced. + ASSERT_TRUE(assigned.assigned_entries[0].File()->first_row_id.has_value()); + EXPECT_EQ(500, assigned.assigned_entries[0].File()->first_row_id.value()); + EXPECT_EQ(0, assigned.next_row_id_start); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingCompactFileKeepsNoFirstRowId) { + std::vector input; + input.push_back(CreateEntry("compact-file", /*row_count=*/6, /*min_seq_number=*/3, + /*max_seq_number=*/5, FileSource::Compact(), + std::vector{"f0"})); + + ASSERT_OK_AND_ASSIGN(RowTrackingCommitUtils::RowTrackingAssigned assigned, + RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/7, input)); + + ASSERT_EQ(1u, assigned.assigned_entries.size()); + // Pure compact file keeps its original sequence numbers and gets no first row id. + EXPECT_EQ(3, assigned.assigned_entries[0].File()->min_sequence_number); + EXPECT_EQ(5, assigned.assigned_entries[0].File()->max_sequence_number); + EXPECT_EQ(std::nullopt, assigned.assigned_entries[0].File()->first_row_id); + EXPECT_EQ(7, assigned.next_row_id_start); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingBlobBeforeNormalFileFails) { + std::vector input; + input.push_back(CreateEntry("blob-a.blob", /*row_count=*/3, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"blob_a"})); + + ASSERT_NOK_WITH_MSG(RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input), + "blobStart"); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingVectorStoreBeforeNormalFileFails) { + std::vector input; + input.push_back(CreateEntry("vector-1.vector.data", /*row_count=*/4, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"vec"})); + + ASSERT_NOK_WITH_MSG(RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input), + "vectorStoreStart"); +} + +TEST_F(RowTrackingCommitUtilsTest, TestAssignRowTrackingBlobWithoutWriteColsFails) { + std::vector input; + input.push_back(CreateEntry("normal-file", /*row_count=*/10, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{"f0"})); + input.push_back(CreateEntry("blob-a.blob", /*row_count=*/3, /*min_seq_number=*/0, + /*max_seq_number=*/0, FileSource::Append(), + std::vector{})); + + ASSERT_NOK_WITH_MSG(RowTrackingCommitUtils::AssignRowTracking( + /*new_snapshot_id=*/100, /*first_row_id_start=*/0, input), + "does not have write_cols"); +} + } // namespace paimon::test diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index 76a34d5a..ea5c5e38 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp @@ -171,7 +171,7 @@ FileStoreCommitImpl::FileStoreCommitImpl( table_schema, schema, options, executor, pool, partition_computer_.get(), std::move(scan_supplier))), conflict_detection_(table_schema, options, snapshot_manager_, manifest_list, manifest_file, - commit_scanner_), + commit_scanner_, commit_user, table_name_, path_factory), manifest_file_(manifest_file), manifest_list_(manifest_list), index_manifest_file_(index_manifest_file), @@ -1042,7 +1042,8 @@ Result FileStoreCommitImpl::TryCommitOnce( std::optional statistics = latest_snapshot ? latest_snapshot.value().Statistics() : std::nullopt; - int64_t changelog_record_count = RowCounts(changelog_entries); + std::optional changelog_record_count = + ManifestEntry::NullableRecordCount(changelog_entries); int64_t schema_id = 0; PAIMON_ASSIGN_OR_RAISE(std::optional> table_schema, schema_manager_->Latest()); @@ -1197,11 +1198,4 @@ void FileStoreCommitImpl::ReportCommit(const ManifestEntryChanges& changes, int6 CommitMetrics::ReportCommit(metrics_, commit_stats); } -int64_t FileStoreCommitImpl::RowCounts(const std::vector& files) { - return std::accumulate(files.begin(), files.end(), 0L, - [](int64_t row_count, const ManifestEntry& entry) { - return row_count + entry.File()->row_count; - }); -} - } // namespace paimon diff --git a/src/paimon/core/operation/file_store_commit_impl.h b/src/paimon/core/operation/file_store_commit_impl.h index 2ef62bbf..f1b40f80 100644 --- a/src/paimon/core/operation/file_store_commit_impl.h +++ b/src/paimon/core/operation/file_store_commit_impl.h @@ -225,8 +225,6 @@ class FileStoreCommitImpl : public FileStoreCommit { Status CheckFilesExistence( const std::vector>& committables) const; - static int64_t RowCounts(const std::vector& files); - private: std::shared_ptr memory_pool_; std::shared_ptr executor_; diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index f16da448..f1f4ad8b 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -396,9 +396,9 @@ TEST_F(FileStoreCommitImplTest, TestRESTCatalogCommit) { /*changelog_manifest_list_size=*/std::nullopt, /*index_manifest=*/std::nullopt, /*commit_user=*/"commit_user_1", /*commit_identifier=*/9223372036854775807, /*commit_kind=*/Snapshot::CommitKind::Append(), /*time_millis=*/1758097357597, - /*total_record_count=*/5, - /*delta_record_count=*/5, /*changelog_record_count=*/0, /*watermark=*/std::nullopt, - /*statistics=*/std::nullopt, /*properties=*/std::nullopt, /*next_row_id=*/0); + /*total_record_count=*/5, /*delta_record_count=*/5, /*changelog_record_count=*/std::nullopt, + /*watermark=*/std::nullopt, /*statistics=*/std::nullopt, /*properties=*/std::nullopt, + /*next_row_id=*/0); std::vector expected_partition_statistics = { PartitionStatistics(/*spec=*/{{"f1", "20"}}, /*record_count=*/1, /*file_size_in_bytes=*/541, /*file_count=*/1, diff --git a/src/paimon/core/snapshot.cpp b/src/paimon/core/snapshot.cpp index e87f708b..d3081c91 100644 --- a/src/paimon/core/snapshot.cpp +++ b/src/paimon/core/snapshot.cpp @@ -199,9 +199,11 @@ rapidjson::Value Snapshot::ToJson(rapidjson::Document::AllocatorType* allocator) RapidJsonUtil::SerializeValue(delta_manifest_list_size_, allocator).Move(), *allocator); } - obj.AddMember(rapidjson::StringRef(FIELD_CHANGELOG_MANIFEST_LIST), - RapidJsonUtil::SerializeValue(changelog_manifest_list_, allocator).Move(), - *allocator); + if (changelog_manifest_list_) { + obj.AddMember(rapidjson::StringRef(FIELD_CHANGELOG_MANIFEST_LIST), + RapidJsonUtil::SerializeValue(changelog_manifest_list_, allocator).Move(), + *allocator); + } if (changelog_manifest_list_size_) { obj.AddMember( rapidjson::StringRef(FIELD_CHANGELOG_MANIFEST_LIST_SIZE), diff --git a/src/paimon/core/snapshot_test.cpp b/src/paimon/core/snapshot_test.cpp index d786e803..606e3200 100644 --- a/src/paimon/core/snapshot_test.cpp +++ b/src/paimon/core/snapshot_test.cpp @@ -35,9 +35,14 @@ class SnapshotTest : public testing::Test { replaced_str = StringUtils::Replace(replaced_str, "\t", ""); replaced_str = StringUtils::Replace(replaced_str, "\n", ""); // logOffsets was removed from snapshot json; normalize legacy fixtures. - replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{},", ""); - replaced_str = StringUtils::Replace(replaced_str, ",\"logOffsets\":{}", ""); - replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{}", ""); + replaced_str = StringUtils::Replace(replaced_str, R"("logOffsets":{},)", ""); + replaced_str = StringUtils::Replace(replaced_str, R"(,"logOffsets":{})", ""); + replaced_str = StringUtils::Replace(replaced_str, R"("logOffsets":{})", ""); + // changelogManifestList is @JsonInclude(NON_NULL) in Java and omitted on serialization; + // strip the stale null field from checked-in Java fixtures before comparing. + replaced_str = StringUtils::Replace(replaced_str, R"("changelogManifestList":null,)", ""); + replaced_str = StringUtils::Replace(replaced_str, R"(,"changelogManifestList":null)", ""); + replaced_str = StringUtils::Replace(replaced_str, R"("changelogManifestList":null)", ""); return replaced_str; } }; @@ -114,7 +119,6 @@ TEST_F(SnapshotTest, TestJsonizable) { "baseManifestListSize" : 20, "deltaManifestList" : "manifest-list-d96fcc30-99e8-4f45-962b-a1157c56f378-1", "deltaManifestListSize" : 50, - "changelogManifestList" : null, "commitUser" : "0e4d92f7-53b0-40d6-a7c0-102bf3801e6a", "commitIdentifier" : 9223372036854775807, "commitKind" : "OVERWRITE", @@ -192,7 +196,6 @@ TEST_F(SnapshotTest, TestSerializeAndDeserialize) { "baseManifestListSize" : 100, "deltaManifestList" : "delta-manifest-list-2", "deltaManifestListSize" : 200, - "changelogManifestList" : null, "commitUser" : "commit-usr-3", "commitIdentifier" : 12, "commitKind" : "APPEND", @@ -218,7 +221,6 @@ TEST_F(SnapshotTest, TestSerializeAndDeserialize) { "baseManifestListSize" : 100, "deltaManifestList" : "delta-manifest-list-2", "deltaManifestListSize" : 200, - "changelogManifestList" : null, "commitUser" : "commit-usr-3", "commitIdentifier" : 12, "commitKind" : "APPEND", @@ -273,7 +275,6 @@ TEST_F(SnapshotTest, TestCommitKindAnalyzeSerializeAndDeserialize) { "baseManifestListSize" : 100, "deltaManifestList" : "delta-manifest-analyze", "deltaManifestListSize" : 200, - "changelogManifestList" : null, "commitUser" : "analyze-user", "commitIdentifier" : 42, "commitKind" : "ANALYZE", @@ -368,7 +369,6 @@ TEST_F(SnapshotTest, TestChangelogManifestListSerialization) { "schemaId" : 0, "baseManifestList" : "base-manifest-list", "deltaManifestList" : "delta-manifest-list", - "changelogManifestList" : null, "commitUser" : "user-02", "commitIdentifier" : 200, "commitKind" : "COMPACT", diff --git a/src/paimon/core/tag/tag_test.cpp b/src/paimon/core/tag/tag_test.cpp index b3423a2c..c94d823b 100644 --- a/src/paimon/core/tag/tag_test.cpp +++ b/src/paimon/core/tag/tag_test.cpp @@ -38,6 +38,11 @@ class TagTest : public testing::Test { replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{},", ""); replaced_str = StringUtils::Replace(replaced_str, ",\"logOffsets\":{}", ""); replaced_str = StringUtils::Replace(replaced_str, "\"logOffsets\":{}", ""); + // changelogManifestList is @JsonInclude(NON_NULL) in Java and omitted on serialization; + // strip the stale null field from checked-in Java fixtures before comparing. + replaced_str = StringUtils::Replace(replaced_str, "\"changelogManifestList\":null,", ""); + replaced_str = StringUtils::Replace(replaced_str, ",\"changelogManifestList\":null", ""); + replaced_str = StringUtils::Replace(replaced_str, "\"changelogManifestList\":null", ""); if (serialized) { replaced_str = StringUtils::Replace(replaced_str, ".0", ".000000000"); } @@ -123,7 +128,6 @@ TEST_F(TagTest, TestJsonizable) { "baseManifestListSize" : 20, "deltaManifestList" : "manifest-list-d96fcc30-99e8-4f45-962b-a1157c56f378-1", "deltaManifestListSize" : 50, - "changelogManifestList" : null, "commitUser" : "0e4d92f7-53b0-40d6-a7c0-102bf3801e6a", "commitIdentifier" : 9223372036854775807, "commitKind" : "OVERWRITE", @@ -189,7 +193,6 @@ TEST_F(TagTest, TestSerializeAndDeserialize) { "baseManifestListSize" : 100, "deltaManifestList" : "delta-manifest-list-2", "deltaManifestListSize" : 200, - "changelogManifestList" : null, "commitUser" : "commit-usr-3", "commitIdentifier" : 12, "commitKind" : "APPEND", @@ -216,7 +219,6 @@ TEST_F(TagTest, TestSerializeAndDeserialize) { "baseManifestListSize" : 100, "deltaManifestList" : "delta-manifest-list-2", "deltaManifestListSize" : 200, - "changelogManifestList" : null, "commitUser" : "commit-usr-3", "commitIdentifier" : 12, "commitKind" : "APPEND", diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp index a4fa11a3..69276442 100644 --- a/test/inte/data_evolution_table_test.cpp +++ b/test/inte/data_evolution_table_test.cpp @@ -395,7 +395,7 @@ TEST_P(DataEvolutionTableTest, TestCommitConflictOnOverlappedRowIdAndWriteColumn ASSERT_NOK_WITH_MSG( CommitWithRowIdCheckFromSnapshot(table_path, commit_msgs_2, /*row_id_check_from_snapshot=*/1), - "multiple MERGE INTO operations have encountered conflicts while checking row-id history"); + "multiple 'MERGE INTO' operations have encountered conflicts, updating the same file"); } TEST_P(DataEvolutionTableTest, TestMultipleAppends) { From 637f4c53e0ca3f68e61d6a168f430b2dd0b6dc74 Mon Sep 17 00:00:00 2001 From: Zouxxyy Date: Mon, 20 Jul 2026 11:02:36 +0800 Subject: [PATCH 101/138] fix(fs): create parent directories before opening Jindo writers --- src/paimon/CMakeLists.txt | 3 +- src/paimon/fs/jindo/jindo_file_system.cpp | 13 ++ src/paimon/fs/jindo/jindo_file_system.h | 3 + .../fs/jindo/jindo_file_system_unit_test.cpp | 139 ++++++++++++++++++ 4 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 src/paimon/fs/jindo/jindo_file_system_unit_test.cpp diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 96187c8e..abb151e9 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -798,7 +798,8 @@ if(PAIMON_BUILD_TESTS) # OSS access and stay disabled. set(FS_TEST_JINDO_SOURCES) if(PAIMON_ENABLE_JINDO) - list(APPEND FS_TEST_JINDO_SOURCES fs/jindo/jindo_utils_test.cpp) + list(APPEND FS_TEST_JINDO_SOURCES fs/jindo/jindo_file_system_unit_test.cpp + fs/jindo/jindo_utils_test.cpp) endif() add_paimon_test(fs_test diff --git a/src/paimon/fs/jindo/jindo_file_system.cpp b/src/paimon/fs/jindo/jindo_file_system.cpp index aaa6fa90..35675457 100644 --- a/src/paimon/fs/jindo/jindo_file_system.cpp +++ b/src/paimon/fs/jindo/jindo_file_system.cpp @@ -28,6 +28,7 @@ #include "fmt/format.h" #include "jdo_error.h" // NOLINT(build/include_subdir) #include "paimon/common/utils/math.h" +#include "paimon/common/utils/path_util.h" #include "paimon/fs/jindo/jindo_file_status.h" #include "paimon/fs/jindo/jindo_utils.h" @@ -70,6 +71,18 @@ Result> JindoFileSystem::Create(const std::string& return Status::Invalid( fmt::format("do not allow overwrite, but the file {} already exists", path)); } + const std::string parent_path = PathUtil::GetParentDirPath(path); + if (!parent_path.empty()) { + PAIMON_ASSIGN_OR_RAISE(Path parent, PathUtil::ToPath(parent_path)); + // Do not issue mkdir for scheme-only or authority-only URI parents. + if (!parent.path.empty()) { + PAIMON_RETURN_NOT_OK(Mkdirs(parent_path)); + } + } + return OpenWriter(path); +} + +Result> JindoFileSystem::OpenWriter(const std::string& path) const { std::unique_ptr writer; PAIMON_RETURN_NOT_OK_FROM_JINDO(impl_->GetFileSystem()->openWriter(path, &writer)); return std::make_unique(impl_, std::move(writer)); diff --git a/src/paimon/fs/jindo/jindo_file_system.h b/src/paimon/fs/jindo/jindo_file_system.h index f3dccc04..7a430a3d 100644 --- a/src/paimon/fs/jindo/jindo_file_system.h +++ b/src/paimon/fs/jindo/jindo_file_system.h @@ -59,6 +59,9 @@ class JindoFileSystem : public FileSystem { Result Exists(const std::string& path) const override; + protected: + virtual Result> OpenWriter(const std::string& path) const; + private: std::shared_ptr impl_; }; diff --git a/src/paimon/fs/jindo/jindo_file_system_unit_test.cpp b/src/paimon/fs/jindo/jindo_file_system_unit_test.cpp new file mode 100644 index 00000000..11a4c76e --- /dev/null +++ b/src/paimon/fs/jindo/jindo_file_system_unit_test.cpp @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/fs/jindo/jindo_file_system.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class TestOutputStream : public OutputStream { + public: + Result GetPos() const override { + return 0; + } + + Result Write(const char*, int64_t size) override { + return size; + } + + Status Flush() override { + return Status::OK(); + } + + Status Close() override { + return Status::OK(); + } + + Result GetUri() const override { + return std::string(); + } +}; + +class TestJindoFileSystem : public jindo::JindoFileSystem { + public: + TestJindoFileSystem() : JindoFileSystem(std::make_unique()) {} + + Result Exists(const std::string&) const override { + return false; + } + + Status Mkdirs(const std::string& path) const override { + parent_path_ = path; + calls_.push_back("mkdirs"); + return mkdirs_status_; + } + + void SetMkdirsStatus(Status status) { + mkdirs_status_ = std::move(status); + } + + const std::string& GetParentPath() const { + return parent_path_; + } + + bool IsWriterOpened() const { + return writer_opened_; + } + + const std::vector& GetCalls() const { + return calls_; + } + + protected: + Result> OpenWriter(const std::string&) const override { + calls_.push_back("open_writer"); + writer_opened_ = true; + std::unique_ptr output = std::make_unique(); + return output; + } + + private: + mutable std::string parent_path_; + mutable bool writer_opened_ = false; + mutable std::vector calls_; + Status mkdirs_status_ = Status::OK(); +}; + +TEST(JindoFileSystemUnitTest, CreateMakesParentDirectoryBeforeOpeningWriter) { + TestJindoFileSystem fs; + const std::string path = "oss://bucket/table/bucket-24/data.orc"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr output, + fs.Create(path, /*overwrite=*/false)); + ASSERT_TRUE(output); + ASSERT_EQ(fs.GetParentPath(), "oss://bucket/table/bucket-24"); + ASSERT_TRUE(fs.IsWriterOpened()); + ASSERT_EQ(fs.GetCalls().size(), 2); + ASSERT_EQ(fs.GetCalls()[0], "mkdirs"); + ASSERT_EQ(fs.GetCalls()[1], "open_writer"); +} + +TEST(JindoFileSystemUnitTest, CreateSkipsObjectStoreRootParent) { + TestJindoFileSystem fs; + const std::string path = "oss://bucket/data.orc"; + + ASSERT_OK_AND_ASSIGN(std::unique_ptr output, + fs.Create(path, /*overwrite=*/false)); + ASSERT_TRUE(output); + ASSERT_TRUE(fs.GetParentPath().empty()); + ASSERT_TRUE(fs.IsWriterOpened()); + ASSERT_EQ(fs.GetCalls().size(), 1); + ASSERT_EQ(fs.GetCalls()[0], "open_writer"); +} + +TEST(JindoFileSystemUnitTest, CreateReturnsParentDirectoryFailure) { + TestJindoFileSystem fs; + const std::string path = "oss://bucket/table/bucket-24/data.orc"; + fs.SetMkdirsStatus(Status::IOError("failed to create parent directory")); + + ASSERT_NOK_WITH_MSG(fs.Create(path, /*overwrite=*/false), "failed to create parent directory"); + ASSERT_EQ(fs.GetParentPath(), "oss://bucket/table/bucket-24"); + ASSERT_FALSE(fs.IsWriterOpened()); + ASSERT_EQ(fs.GetCalls().size(), 1); + ASSERT_EQ(fs.GetCalls()[0], "mkdirs"); +} + +} // namespace paimon::test From 7f9b180437bd4dbb51c56765d002ead1175fab7b Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:58:17 +0800 Subject: [PATCH 102/138] fix(jindo): make async reads concurrency-safe --- CMakeLists.txt | 7 ++ cmake_modules/DefineOptions.cmake | 3 + include/paimon/fs/file_system.h | 1 + src/paimon/CMakeLists.txt | 27 ++++--- src/paimon/common/fs/file_system_test.cpp | 4 +- .../key_value_file_store_write_test.cpp | 1 - src/paimon/fs/jindo/jindo_file_system.cpp | 50 +++++++++--- src/paimon/fs/jindo/jindo_file_system.h | 2 - .../fs/jindo/jindo_file_system_test.cpp | 78 +++++++++++++++++++ src/paimon/testing/utils/test_helper.h | 17 ++-- src/paimon/testing/utils/testharness.cpp | 2 + test/inte/CMakeLists.txt | 4 + test/inte/pk_compaction_inte_test.cpp | 3 - test/inte/write_and_read_inte_test.cpp | 4 +- test/inte/write_inte_test.cpp | 49 ++++-------- 15 files changed, 179 insertions(+), 73 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b91fefb..cb6ee880 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,8 @@ option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON) option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON) option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON) option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF) +option(PAIMON_ENABLE_NETWORK_TESTS + "Whether to enable tests that access real remote services over the network" OFF) option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF) option(PAIMON_ENABLE_TANTIVY "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF) @@ -73,6 +75,11 @@ endif() if(PAIMON_ENABLE_JINDO) add_definitions(-DPAIMON_ENABLE_JINDO) endif() +if(PAIMON_ENABLE_NETWORK_TESTS) + if(NOT PAIMON_BUILD_TESTS) + message(FATAL_ERROR "PAIMON_ENABLE_NETWORK_TESTS requires PAIMON_BUILD_TESTS=ON") + endif() +endif() if(PAIMON_USE_CXX11_ABI) add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1) else() diff --git a/cmake_modules/DefineOptions.cmake b/cmake_modules/DefineOptions.cmake index 291ce1db..4aacc33d 100644 --- a/cmake_modules/DefineOptions.cmake +++ b/cmake_modules/DefineOptions.cmake @@ -107,6 +107,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") define_option(PAIMON_BUILD_TESTS "Build the Paimon googletest unit tests" OFF) + define_option(PAIMON_ENABLE_NETWORK_TESTS + "Enable tests that access real remote services over the network" OFF) + define_option(PAIMON_BUILD_BENCHMARKS "Build the Paimon Google Benchmark performance benchmarks" OFF) diff --git a/include/paimon/fs/file_system.h b/include/paimon/fs/file_system.h index 0a89a1e6..bee3ecc8 100644 --- a/include/paimon/fs/file_system.h +++ b/include/paimon/fs/file_system.h @@ -100,6 +100,7 @@ class PAIMON_EXPORT InputStream : public Stream { /// @param callback The callback function to be invoked upon completion of the read operation. /// The callback will receive a Status object indicating the success or failure /// of the read operation. + /// @note The caller must keep the input stream and buffer alive until the callback is invoked. virtual void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) = 0; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index abb151e9..5c0c7f44 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -793,13 +793,20 @@ if(PAIMON_BUILD_TESTS) ${TEST_STATIC_LINK_LIBS} ${GTEST_LINK_TOOLCHAIN}) - # jindo_utils_test only checks the in-memory status conversion and does not need an - # OSS cluster, so it runs whenever jindo is built. The other jindo tests need real - # OSS access and stay disabled. - set(FS_TEST_JINDO_SOURCES) + set(PAIMON_JINDO_FS_TEST_SOURCES) + set(PAIMON_JINDO_FS_TEST_LINK_LIBS) if(PAIMON_ENABLE_JINDO) - list(APPEND FS_TEST_JINDO_SOURCES fs/jindo/jindo_file_system_unit_test.cpp + # These unit tests do not access remote OSS. + list(APPEND PAIMON_JINDO_FS_TEST_SOURCES fs/jindo/jindo_file_system_unit_test.cpp fs/jindo/jindo_utils_test.cpp) + if(PAIMON_ENABLE_NETWORK_TESTS) + # Factory and integration tests access real OSS. + list(APPEND PAIMON_JINDO_FS_TEST_SOURCES + fs/jindo/jindo_file_system_factory_test.cpp + fs/jindo/jindo_file_system_test.cpp) + endif() + list(APPEND PAIMON_JINDO_FS_TEST_LINK_LIBS + ${PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS}) endif() add_paimon_test(fs_test @@ -807,16 +814,18 @@ if(PAIMON_BUILD_TESTS) common/fs/file_system_test.cpp common/fs/resolving_file_system_test.cpp fs/local/local_file_test.cpp - ${FS_TEST_JINDO_SOURCES} - # fs/jindo/jindo_file_system_factory_test.cpp - # fs/jindo/jindo_file_system_test.cpp + ${PAIMON_JINDO_FS_TEST_SOURCES} STATIC_LINK_LIBS paimon_shared ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS} - ${PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS} + ${PAIMON_JINDO_FS_TEST_LINK_LIBS} test_utils_static ${GTEST_LINK_TOOLCHAIN} EXTRA_INCLUDES ${JINDOSDK_INCLUDE_DIR}) + if(PAIMON_ENABLE_NETWORK_TESTS) + target_compile_definitions(paimon-fs-test PRIVATE PAIMON_ENABLE_NETWORK_TESTS) + endif() + endif() diff --git a/src/paimon/common/fs/file_system_test.cpp b/src/paimon/common/fs/file_system_test.cpp index 7a80904f..c40a7678 100644 --- a/src/paimon/common/fs/file_system_test.cpp +++ b/src/paimon/common/fs/file_system_test.cpp @@ -1477,7 +1477,9 @@ TEST_P(FileSystemTest, TestAtomicStoreAlreadyExist) { std::vector GetTestValuesForFileSystemTest() { std::vector values; values.emplace_back("local"); - // values.emplace_back("jindo"); +#if defined(PAIMON_ENABLE_NETWORK_TESTS) && defined(PAIMON_ENABLE_JINDO) + values.emplace_back("jindo"); +#endif return values; } diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp index 9448a1fe..705220be 100644 --- a/src/paimon/core/operation/key_value_file_store_write_test.cpp +++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp @@ -322,7 +322,6 @@ TEST_F(KeyValueFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNextW {"fields.tags.map.shared-shredding.column-placement-policy", "plain"}, {"write-only", "true"}, {"bucket", "1"}, - {"enable-pk-commit-in-inte-test", ""}, }; auto logical_schema = arrow::schema({ arrow::field("id", arrow::int32(), /*nullable=*/false), diff --git a/src/paimon/fs/jindo/jindo_file_system.cpp b/src/paimon/fs/jindo/jindo_file_system.cpp index 35675457..0bfb7c8c 100644 --- a/src/paimon/fs/jindo/jindo_file_system.cpp +++ b/src/paimon/fs/jindo/jindo_file_system.cpp @@ -18,7 +18,9 @@ #include "paimon/fs/jindo/jindo_file_system.h" +#include #include +#include #include #include "JdoFileInfo.hpp" // NOLINT(build/include_subdir) @@ -55,6 +57,29 @@ class JindoFileSystemImpl { std::unique_ptr fs_; }; +namespace { + +class AsyncReadState { + public: + explicit AsyncReadState(std::function&& callback) + : callback_(std::move(callback)) {} + + void Complete(JdoStatus status) { + if (completed_.exchange(true)) { + return; + } + callback_(status.ok() ? Status::OK() : Status::IOError(status.errMsg())); + } + + std::string_view result; + + private: + std::atomic completed_{false}; + std::function callback_; +}; + +} // namespace + JindoFileSystem::JindoFileSystem(std::unique_ptr&& fs) : impl_(std::make_shared(std::move(fs))) {} @@ -230,15 +255,17 @@ Result JindoInputStream::Length() const { Result JindoInputStream::Read(char* buffer, int64_t size) { PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length")); - PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->read(size, &result_, buffer)); - return result_.length(); + std::string_view result; + PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->read(size, &result, buffer)); + return result.length(); } Result JindoInputStream::Read(char* buffer, int64_t size, int64_t offset) { PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length")); PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(offset, "read offset")); - PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->pread(offset, size, &result_, buffer)); - return result_.length(); + std::string_view result; + PAIMON_RETURN_NOT_OK_FROM_JINDO(reader_->pread(offset, size, &result, buffer)); + return result.length(); } void JindoInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, @@ -253,12 +280,17 @@ void JindoInputStream::ReadAsync(char* buffer, int64_t size, int64_t offset, callback(validate_status); return; } - auto outer_callback = [=](JdoStatus status) { - callback(status.ok() ? Status::OK() : Status::IOError(status.errMsg())); - }; - auto task = reader_->preadAsync(offset, size, &result_, buffer, outer_callback); + std::shared_ptr state = std::make_shared(std::move(callback)); + auto task = reader_->preadAsync(offset, size, &state->result, buffer, + [state](JdoStatus status) { state->Complete(status); }); assert(task); - [[maybe_unused]] auto perform_status = task->perform(); + + auto perform_status = task->perform(); + if (!perform_status.ok()) { + state->Complete(perform_status); + [[maybe_unused]] auto status = task->cancel(); + return; + } } Status JindoInputStream::Close() { diff --git a/src/paimon/fs/jindo/jindo_file_system.h b/src/paimon/fs/jindo/jindo_file_system.h index 7a430a3d..0ba8990b 100644 --- a/src/paimon/fs/jindo/jindo_file_system.h +++ b/src/paimon/fs/jindo/jindo_file_system.h @@ -22,7 +22,6 @@ #include #include #include -#include #include #include "JdoFileSystem.hpp" // NOLINT(build/include_subdir) @@ -85,7 +84,6 @@ class JindoInputStream : public InputStream { // the Jindo Reader. std::shared_ptr fs_; std::unique_ptr reader_; - std::string_view result_; }; class JindoOutputStream : public OutputStream { diff --git a/src/paimon/fs/jindo/jindo_file_system_test.cpp b/src/paimon/fs/jindo/jindo_file_system_test.cpp index 437e4ba4..efaf2a72 100644 --- a/src/paimon/fs/jindo/jindo_file_system_test.cpp +++ b/src/paimon/fs/jindo/jindo_file_system_test.cpp @@ -16,10 +16,19 @@ * limitations under the License. */ +#include +#include +#include +#include +#include +#include + #include "gtest/gtest.h" #include "paimon/fs/jindo/jindo_file_system_factory.h" #include "paimon/testing/utils/testharness.h" + namespace paimon::jindo::test { + // This test shows inconsistent behavior with the local file system in some abnormal scenarios. class JindoFileSystemTest : public ::testing::Test { public: @@ -126,4 +135,73 @@ TEST_F(JindoFileSystemTest, TestSeek) { ASSERT_OK(in_stream->Close()); } +TEST(JindoFileSystemPaginationTest, TestListDirAcrossOssPageBoundary) { + constexpr int32_t kFileCount = 1234; + const std::string test_dir = "oss://paimon-unittest/test_data/jindo_listdir_truncated_1234/"; + std::map options = paimon::test::GetJindoTestOptions(); + + auto fs_factory = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fs, fs_factory->Create(test_dir, options)); + + std::vector> file_statuses; + ASSERT_OK(fs->ListDir(test_dir, &file_statuses)); + ASSERT_EQ(file_statuses.size(), kFileCount); + + std::unordered_set actual_paths; + for (const std::unique_ptr& file_status : file_statuses) { + ASSERT_TRUE(actual_paths.insert(file_status->GetPath()).second) + << "duplicate path: " << file_status->GetPath(); + } + for (int32_t i = 0; i < kFileCount; ++i) { + std::string index = std::to_string(i); + index.insert(/*pos=*/0, /*count=*/4 - index.size(), /*ch=*/'0'); + ASSERT_NE(actual_paths.find(test_dir + "file-" + index + ".txt"), actual_paths.end()); + } +} + +TEST(JindoFileSystemAsyncReadTest, TestConcurrentReadAsyncAndReadFromOss) { + constexpr int32_t kConcurrentReads = 64; + constexpr int64_t kAsyncReadSize = 7; + const std::string file_path = "oss://paimon-unittest/test_data/jindo_read_async_128mb.bin"; + std::map options = paimon::test::GetJindoTestOptions(); + auto fs_factory = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::unique_ptr fs, fs_factory->Create(file_path, options)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr input_stream, fs->Open(file_path)); + + std::vector> async_buffers(kConcurrentReads, + std::vector(kAsyncReadSize)); + std::vector> promises(kConcurrentReads); + std::vector> futures; + futures.reserve(kConcurrentReads); + for (std::promise& promise : promises) { + futures.push_back(promise.get_future()); + } + + std::vector sync_read_statuses; + std::vector sync_read_sizes; + sync_read_statuses.reserve(kConcurrentReads); + sync_read_sizes.reserve(kConcurrentReads); + for (int32_t i = 0; i < kConcurrentReads; ++i) { + input_stream->ReadAsync( + async_buffers[i].data(), async_buffers[i].size(), /*offset=*/0, + [&promises, i](Status status) { promises[i].set_value(std::move(status)); }); + + char sync_buffer = 0; + Result sync_read_result = + input_stream->Read(&sync_buffer, /*size=*/1, /*offset=*/i); + sync_read_statuses.push_back(sync_read_result.status()); + sync_read_sizes.push_back(sync_read_result.ok() ? sync_read_result.value() : -1); + } + + for (int32_t i = 0; i < kConcurrentReads; ++i) { + ASSERT_EQ(futures[i].wait_for(std::chrono::seconds(60)), std::future_status::ready) + << "async read=" << i; + ASSERT_OK(futures[i].get()); + } + for (int32_t i = 0; i < kConcurrentReads; ++i) { + ASSERT_OK(sync_read_statuses[i]) << "sync read=" << i; + ASSERT_EQ(sync_read_sizes[i], 1) << "sync read=" << i; + } +} + } // namespace paimon::jindo::test diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index 4c1ce663..5489d15f 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -61,19 +61,16 @@ class TestHelper { const std::vector& primary_keys, const std::map& options, bool is_streaming_mode, bool ignore_if_exists = false, const std::string& temp_directory = "") { - // only for test && only check the key - auto new_options = options; - new_options["enable-object-store-catalog-in-inte-test"] = ""; - PAIMON_ASSIGN_OR_RAISE(auto catalog, Catalog::Create(root_path, new_options)); - PAIMON_RETURN_NOT_OK(catalog->CreateDatabase("foo", new_options, ignore_if_exists)); + PAIMON_ASSIGN_OR_RAISE(auto catalog, Catalog::Create(root_path, options)); + PAIMON_RETURN_NOT_OK(catalog->CreateDatabase("foo", options, ignore_if_exists)); ::ArrowSchema c_schema; ScopeGuard guard([schema = &c_schema]() { ArrowSchemaRelease(schema); }); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); PAIMON_RETURN_NOT_OK(catalog->CreateTable(Identifier("foo", "bar"), &c_schema, - partition_keys, primary_keys, new_options, + partition_keys, primary_keys, options, ignore_if_exists)); std::string table_path = PathUtil::JoinPath(root_path, "foo.db/bar"); - return Create(table_path, new_options, is_streaming_mode, temp_directory); + return Create(table_path, options, is_streaming_mode, temp_directory); } static Result> Create( @@ -98,14 +95,10 @@ class TestHelper { .Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr write, FileStoreWrite::Create(std::move(write_context))); - std::map new_options = options; - // only for test && only check the key - new_options["enable-pk-commit-in-inte-test"] = ""; - new_options["enable-object-store-commit-in-inte-test"] = ""; CommitContextBuilder commit_context_builder(table_path, commit_user); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr commit_context, - commit_context_builder.SetOptions(new_options).IgnoreEmptyCommit(false).Finish()); + commit_context_builder.SetOptions(options).IgnoreEmptyCommit(false).Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit, FileStoreCommit::Create(std::move(commit_context))); return std::unique_ptr(new TestHelper(std::move(file_system), std::move(write), diff --git a/src/paimon/testing/utils/testharness.cpp b/src/paimon/testing/utils/testharness.cpp index 9c61ac05..d018ab08 100644 --- a/src/paimon/testing/utils/testharness.cpp +++ b/src/paimon/testing/utils/testharness.cpp @@ -107,6 +107,8 @@ std::map GetJindoTestOptions() { {"fs.oss.bucket.paimon-unittest.accessKeyId", access_key_id}, {"fs.oss.bucket.paimon-unittest.accessKeySecret", access_key_secret}, {"fs.oss.user", "paimon"}, + {"enable-object-store-catalog-in-inte-test", ""}, + {"enable-object-store-commit-in-inte-test", ""}, }; return options; } diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt index 27960afe..051b1030 100644 --- a/test/inte/CMakeLists.txt +++ b/test/inte/CMakeLists.txt @@ -42,6 +42,10 @@ if(PAIMON_BUILD_TESTS) ${TEST_STATIC_LINK_LIBS} test_utils_static ${GTEST_LINK_TOOLCHAIN}) + if(PAIMON_ENABLE_NETWORK_TESTS) + target_compile_definitions(paimon-write-and-read-inte-test + PRIVATE PAIMON_ENABLE_NETWORK_TESTS) + endif() add_paimon_test(clean_inte_test STATIC_LINK_LIBS diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp index aa90d626..0d7a5c2e 100644 --- a/test/inte/pk_compaction_inte_test.cpp +++ b/test/inte/pk_compaction_inte_test.cpp @@ -185,9 +185,6 @@ class PkCompactionInteTest : public ::testing::Test, Status Commit(const std::string& table_path, const std::vector>& commit_msgs) const { CommitContextBuilder commit_builder(table_path, "commit_user_1"); - std::map commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - commit_builder.SetOptions(commit_options); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_store_commit, diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index 28f55148..a58f25ce 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -1098,7 +1098,9 @@ TEST_P(WriteAndReadInteTest, TestCharVarcharBinaryVarbinaryTypes) { std::vector> GetTestValuesForWriteAndReadInteTest() { std::vector> values = {{"parquet", "local"}}; - // values.emplace_back("parquet", "jindo"); +#if defined(PAIMON_ENABLE_NETWORK_TESTS) && defined(PAIMON_ENABLE_JINDO) + values.emplace_back("parquet", "jindo"); +#endif #ifdef PAIMON_ENABLE_ORC values.emplace_back("orc", "local"); #endif diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index f58e105d..a5e43bd1 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -345,11 +345,9 @@ class WriteInteTest : public testing::Test, public ::testing::WithParamInterface Status CommitMessages(const std::string& table_path, const std::vector>& commit_messages, - const std::map& commit_options, bool ignore_empty_commit = true, int64_t commit_identifier = BATCH_WRITE_COMMIT_IDENTIFIER) const { CommitContextBuilder commit_builder(table_path, "commit_user_1"); - commit_builder.SetOptions(commit_options); commit_builder.IgnoreEmptyCommit(ignore_empty_commit); PAIMON_ASSIGN_OR_RAISE(auto commit_context, commit_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(auto file_store_commit, @@ -2535,10 +2533,7 @@ TEST_P(WriteInteTest, TestWriteWithFieldId) { ASSERT_OK(file_store_write->Close()); // commit - std::map commit_options = { - {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK(CommitMessages(table_path, commit_messages, commit_options, - /*ignore_empty_commit=*/false)); + ASSERT_OK(CommitMessages(table_path, commit_messages, /*ignore_empty_commit=*/false)); // check data file has field id meta std::vector> status_list; @@ -2620,10 +2615,8 @@ TEST_P(WriteInteTest, TestAppendTableWriteAndReadWithExternalPath) { ASSERT_EQ(results.size(), 1); auto commit_msg_impl = std::dynamic_pointer_cast(results[0]); auto meta = commit_msg_impl->data_increment_.new_files_[0]; - std::map commit_options = { - {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK(CommitMessages(root_path, results, commit_options, - /*ignore_empty_commit=*/false, /*commit_identifier=*/1)); + ASSERT_OK(CommitMessages(root_path, results, /*ignore_empty_commit=*/false, + /*commit_identifier=*/1)); // check external path ASSERT_OK_AND_ASSIGN(bool file_exist, file_system_->Exists(meta->external_path.value())); @@ -2941,10 +2934,8 @@ TEST_P(WriteInteTest, TestWriteAndReadWithSpecialPartitionValue) { ASSERT_EQ(results.size(), 3); auto commit_msg_impl = std::dynamic_pointer_cast(results[0]); auto meta = commit_msg_impl->data_increment_.new_files_[0]; - std::map commit_options = { - {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK(CommitMessages(root_path, results, commit_options, - /*ignore_empty_commit=*/false, /*commit_identifier=*/1)); + ASSERT_OK(CommitMessages(root_path, results, /*ignore_empty_commit=*/false, + /*commit_identifier=*/1)); arrow::FieldVector fields_with_row_kind = fields; fields_with_row_kind.insert(fields_with_row_kind.begin(), @@ -3126,10 +3117,8 @@ TEST_P(WriteInteTest, TestWriteWithNestedSchema) { ASSERT_OK_AND_ASSIGN(std::vector> results, file_store_write->PrepareCommit()); ASSERT_EQ(results.size(), 1); - std::map commit_options = { - {Options::MANIFEST_TARGET_FILE_SIZE, "8mb"}, {Options::FILE_SYSTEM, "local"}}; - ASSERT_OK(CommitMessages(root_path, results, commit_options, - /*ignore_empty_commit=*/false, /*commit_identifier=*/1)); + ASSERT_OK(CommitMessages(root_path, results, /*ignore_empty_commit=*/false, + /*commit_identifier=*/1)); // check read result ScanContextBuilder scan_context_builder(table_path); @@ -4065,9 +4054,7 @@ TEST_P(WriteInteTest, TestPkSpillableDiskQuotaExhaustedFallsBackToFlush) { file_store_write->PrepareCommit(/*wait_compaction=*/false, /*commit_identifier=*/0)); - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(table_path, commit_messages, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages)); ASSERT_OK(file_store_write->Close()); std::string expected = R"([ @@ -4137,9 +4124,7 @@ TEST_P(WriteInteTest, TestPkSpillableGlobalMemoryPreemptionDataCorrectness) { ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir)); - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(table_path, commit_messages, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages)); ASSERT_OK(file_store_write->Close()); // Scan and verify both partitions @@ -4303,9 +4288,7 @@ TEST_P(WriteInteTest, TestPkSpillableIntermediateMergeWithTempFileTracking) { /*commit_identifier=*/0)); ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir)); - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(table_path, commit_messages, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages)); ASSERT_OK(file_store_write->Close()); // Scan: Alice deduped to f1=3, Bob f1=2 @@ -4402,9 +4385,7 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { /*commit_identifier=*/0)); // Spill files should be cleaned after PrepareCommit ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir)); - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(table_path, commit_messages_1, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages_1)); // Round 2: Bucket 0 writes Charlie + Bob(overwrite), Bucket 1 writes Frank + Eve(overwrite) auto r2_b0_batch1 = @@ -4435,7 +4416,7 @@ TEST_P(WriteInteTest, TestPkSpillableMultiBucketMultiRoundDataCorrectness) { /*commit_identifier=*/1)); // Spill files should be cleaned after PrepareCommit ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir)); - ASSERT_OK(CommitMessages(table_path, commit_messages_2, pk_commit_options)); + ASSERT_OK(CommitMessages(table_path, commit_messages_2)); ASSERT_OK(file_store_write->Close()); // Scan and verify per (partition, bucket) @@ -4579,10 +4560,8 @@ TEST_P(WriteInteTest, TestPkSpillableWithIOException) { io_hook->Clear(); // Commit both rounds - std::map pk_commit_options = { - {"enable-pk-commit-in-inte-test", ""}, {"enable-object-store-commit-in-inte-test", ""}}; - ASSERT_OK(CommitMessages(root_path, results_1.value(), pk_commit_options)); - ASSERT_OK(CommitMessages(root_path, results_2.value(), pk_commit_options)); + ASSERT_OK(CommitMessages(root_path, results_1.value())); + ASSERT_OK(CommitMessages(root_path, results_2.value())); ASSERT_OK(file_store_write->Close()); // Scan and verify final state after spill: From d84744a12dcb787ae2f43e5894107708e5ff71ea Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:43:07 +0800 Subject: [PATCH 103/138] fix: enable TSAN and resolve detected data races --- .github/workflows/gcc8_test.yaml | 2 +- build_support/tsan-suppressions.txt | 6 + ci/scripts/build_paimon.sh | 108 ++++++++++++++-- cmake_modules/BuildUtils.cmake | 6 +- cmake_modules/ThirdpartyToolchain.cmake | 122 +++++------------- cmake_modules/san-config.cmake | 18 +++ .../operation/metrics/compaction_metrics.h | 24 ++-- .../metrics/compaction_metrics_test.cpp | 35 +++++ .../testing/utils/counting_cache_test_utils.h | 24 +++- test/inte/read_inte_test.cpp | 20 +-- 10 files changed, 235 insertions(+), 130 deletions(-) diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml index 01ed551c..692f4585 100644 --- a/.github/workflows/gcc8_test.yaml +++ b/.github/workflows/gcc8_test.yaml @@ -69,7 +69,7 @@ jobs: env: CC: gcc-8 CXX: g++-8 - run: ci/scripts/build_paimon.sh $(pwd) + run: ci/scripts/build_paimon.sh --source_dir "$(pwd)" - name: Show ccache statistics if: always() run: ccache -s diff --git a/build_support/tsan-suppressions.txt b/build_support/tsan-suppressions.txt index 13a83393..4c7aef1b 100644 --- a/build_support/tsan-suppressions.txt +++ b/build_support/tsan-suppressions.txt @@ -14,3 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +# Prebuilt shared libraries are not TSAN-instrumented. Suppress reports from the whole library. +race:liblance_lib_rc.so +thread:liblance_lib_rc.so +race:liblumina.so +race:libjindosdk_c.so.6 diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index 75b00369..32350438 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -17,11 +17,92 @@ set -eux -source_dir=${1} -enable_sanitizer=${2:-false} -check_clang_tidy=${3:-false} -build_type=${4:-Debug} -install_smoke=${5:-false} +usage() { + echo "Usage: $0 --source_dir [--enable_asan] [--enable_ubsan] [--enable_tsan] [--check_clang_tidy] [--build_type ] [--lint_git_target_commit ] [--install_smoke]" +} + +source_dir="" +enable_asan="false" +enable_ubsan="false" +enable_tsan="false" +check_clang_tidy="false" +build_type="Debug" +lint_git_target_commit="origin/main" +install_smoke="false" + +while [[ $# -gt 0 ]]; do + case "$1" in + --source_dir) + if [[ $# -lt 2 ]]; then + echo "Missing value for --source_dir" >&2 + usage >&2 + exit 1 + fi + source_dir=$2 + shift 2 + ;; + --enable_asan) + enable_asan="true" + shift + ;; + --enable_ubsan) + enable_ubsan="true" + shift + ;; + --enable_tsan) + enable_tsan="true" + shift + ;; + --check_clang_tidy) + check_clang_tidy="true" + shift + ;; + --build_type) + if [[ $# -lt 2 ]]; then + echo "Missing value for --build_type" >&2 + usage >&2 + exit 1 + fi + build_type=$2 + shift 2 + ;; + --lint_git_target_commit) + if [[ $# -lt 2 ]]; then + echo "Missing value for --lint_git_target_commit" >&2 + usage >&2 + exit 1 + fi + lint_git_target_commit=$2 + shift 2 + ;; + --install_smoke) + install_smoke="true" + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -z "${source_dir}" ]]; then + echo "--source_dir is required" >&2 + usage >&2 + exit 1 +fi + +if [[ "${enable_asan}" == "true" && "${enable_tsan}" == "true" ]]; then + echo "ASAN and TSAN cannot be enabled together" >&2 + usage >&2 + exit 1 +fi + build_dir="${source_dir}/build" if [[ -n "${PAIMON_BUILD_JOBS:-}" ]]; then @@ -52,6 +133,9 @@ if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then ENABLE_LUMINA="OFF" ENABLE_TANTIVY="OFF" # tantivy-fts (Rust FFI) is not built on the gcc-8 image. fi +if [[ "${enable_tsan}" == "true" ]]; then + ENABLE_TANTIVY="OFF" # Tantivy's Rust library is not TSAN-instrumented. +fi CMAKE_ARGS=( "-G Ninja" @@ -61,13 +145,17 @@ CMAKE_ARGS=( "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}" "-DPAIMON_ENABLE_LUCENE=ON" "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}" + "-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}" ) -if [[ "${enable_sanitizer}" == "true" ]]; then - CMAKE_ARGS+=( - "-DPAIMON_USE_ASAN=ON" - "-DPAIMON_USE_UBSAN=ON" - ) +if [[ "${enable_asan}" == "true" ]]; then + CMAKE_ARGS+=("-DPAIMON_USE_ASAN=ON") +fi +if [[ "${enable_ubsan}" == "true" ]]; then + CMAKE_ARGS+=("-DPAIMON_USE_UBSAN=ON") +fi +if [[ "${enable_tsan}" == "true" ]]; then + CMAKE_ARGS+=("-DPAIMON_USE_TSAN=ON") fi cmake "${CMAKE_ARGS[@]}" "${source_dir}" diff --git a/cmake_modules/BuildUtils.cmake b/cmake_modules/BuildUtils.cmake index bf93c77f..c0a9e14f 100644 --- a/cmake_modules/BuildUtils.cmake +++ b/cmake_modules/BuildUtils.cmake @@ -186,11 +186,13 @@ function(add_paimon_lib LIB_NAME) if(NOT APPLE) set(SHARED_LINK_OPTIONS -Wl,--exclude-libs,ALL -Wl,-Bsymbolic -Wl,--gc-sections) - # -z defs (--no-undefined) rejects the __asan_*/__ubsan_* symbols that + # -z defs (--no-undefined) rejects the __asan_*/__tsan_*/__ubsan_* symbols that # sanitizer-instrumented shared libraries legitimately leave undefined # (they are resolved at load time from the executable's sanitizer # runtime). Only enforce it for non-sanitizer builds. - if(NOT PAIMON_USE_ASAN AND NOT PAIMON_USE_UBSAN) + if(NOT PAIMON_USE_ASAN + AND NOT PAIMON_USE_TSAN + AND NOT PAIMON_USE_UBSAN) list(APPEND SHARED_LINK_OPTIONS -Wl,-z,defs) endif() target_link_options(${LIB_NAME}_shared PRIVATE ${SHARED_LINK_OPTIONS}) diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index e56d6257..cb3d825b 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -23,6 +23,8 @@ set(THIRDPARTY_LOG_OPTIONS LOG_INSTALL 1 LOG_DOWNLOAD + 1 + LOG_OUTPUT_ON_FAILURE 1) set(THIRDPARTY_CONFIGURE_COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}") if(CMAKE_GENERATOR_TOOLSET) @@ -376,7 +378,7 @@ if(NOT MSVC_TOOLCHAIN) # Set -fPIC on all external projects string(APPEND EP_CXX_FLAGS " -fPIC -Wno-error -Wno-sign-compare -Wno-ignored-attributes") - string(APPEND EP_C_FLAGS " -fPIC") + string(APPEND EP_C_FLAGS " -fPIC -Wno-error") endif() if(PAIMON_USE_CXX11_ABI) @@ -769,19 +771,12 @@ macro(build_lucene) set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install") - set(LUCENE_CMAKE_CXX_FLAGS "-pthread") - if(PAIMON_USE_CXX11_ABI) - string(APPEND LUCENE_CMAKE_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=1") - else() - string(APPEND LUCENE_CMAKE_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=0") - endif() - set(LUCENE_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DLUCENE_BUILD_SHARED=OFF" "-DENABLE_TEST=OFF" - "-DCMAKE_C_FLAGS=-pthread" - "-DCMAKE_CXX_FLAGS=${LUCENE_CMAKE_CXX_FLAGS}" + "-DCMAKE_C_FLAGS=${EP_C_FLAGS} -pthread" + "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -pthread" "-DCMAKE_EXE_LINKER_FLAGS=-pthread" "-DBoost_NO_BOOST_CMAKE=ON" "-DBoost_NO_SYSTEM_PATHS=ON" @@ -849,6 +844,8 @@ macro(build_jieba) externalproject_add(limonp_ep URL ${LIMONP_SOURCE_URL} URL_HASH "SHA256=${PAIMON_LIMONP_BUILD_SHA256_CHECKSUM}" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "") message(STATUS "Building jieba from source") @@ -925,17 +922,9 @@ macro(build_fmt) "${FMT_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${FMT_STATIC_LIB_NAME}${FMT_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(FMT_LIBRARIES ${FMT_STATIC_LIB}) - set(FMT_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(FMT_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" FMT_CMAKE_CXX_FLAGS ${FMT_CMAKE_CXX_FLAGS}) - set(FMT_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${FMT_PREFIX} - "-DCMAKE_CXX_FLAGS=${FMT_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${FMT_CMAKE_C_FLAGS}" - -DFMT_TEST=OFF - -DFMT_DOC=OFF) + set(FMT_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${FMT_PREFIX} + -DFMT_TEST=OFF -DFMT_DOC=OFF) set(FMT_CONFIGURE CMAKE_ARGS ${FMT_CMAKE_ARGS}) externalproject_add(fmt_ep URL ${FMT_SOURCE_URL} @@ -971,12 +960,16 @@ macro(build_boost) ${BOOST_LIBRARY_DIR}/libboost_chrono.a ${BOOST_LIBRARY_DIR}/libboost_iostreams.a) - set(BOOST_CXX_FLAGS "-fPIC") - if(PAIMON_USE_CXX11_ABI) - string(APPEND BOOST_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=1") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(BOOST_TOOLSET clang) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(BOOST_TOOLSET gcc) else() - string(APPEND BOOST_CXX_FLAGS " -D_GLIBCXX_USE_CXX11_ABI=0") + message(FATAL_ERROR "Unsupported compiler for Boost: ${CMAKE_CXX_COMPILER_ID}") endif() + set(BOOST_USER_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/boost-user-config.jam") + file(WRITE ${BOOST_USER_CONFIG} + "using ${BOOST_TOOLSET} : : \"${CMAKE_CXX_COMPILER}\" ;\n") externalproject_add(boost_ep URL ${BOOST_SOURCE_URL} @@ -990,14 +983,17 @@ macro(build_boost) -sZLIB_INCLUDE=${ZLIB_INCLUDE_DIR} -sZLIB_LIBRARY_PATH=${ZLIB_PREFIX}/lib runtime-link=shared threading=multi variant=release - cxxflags=${BOOST_CXX_FLAGS} install + --user-config=${BOOST_USER_CONFIG} + toolset=${BOOST_TOOLSET} cxxflags=${EP_CXX_FLAGS} + linkflags=${EP_CXX_FLAGS} install INSTALL_COMMAND bash -c "mkdir -p ${BOOST_INSTALL}/include/boost && cp -r ${BOOST_PREFIX}/src/boost_ep/libs/*/include/boost/* ${BOOST_INSTALL}/include/boost && cp -r ${BOOST_PREFIX}/src/boost_ep/libs/*/*/include/boost/* ${BOOST_INSTALL}/include/boost" DEPENDS zlib BUILD_BYPRODUCTS ${BOOST_BYPRODUCTS} LOG_DOWNLOAD ON LOG_CONFIGURE ON - LOG_BUILD ON) + LOG_BUILD ON + LOG_OUTPUT_ON_FAILURE ON) include_directories(SYSTEM ${BOOST_INCLUDE_DIR}) @@ -1141,17 +1137,9 @@ macro(build_zstd) "${ZSTD_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ZSTD_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(ZSTD_LIBRARIES ${ZSTD_STATIC_LIB}) - set(ZSTD_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(ZSTD_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" ZSTD_CMAKE_CXX_FLAGS ${ZSTD_CMAKE_CXX_FLAGS}) - set(ZSTD_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX} - "-DCMAKE_CXX_FLAGS=${ZSTD_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ZSTD_CMAKE_C_FLAGS}" - -DZSTD_BUILD_SHARED=OFF - -DZSTD_BUILD_PROGRAMS=OFF) + set(ZSTD_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX} + -DZSTD_BUILD_SHARED=OFF -DZSTD_BUILD_PROGRAMS=OFF) set(ZSTD_CONFIGURE SOURCE_SUBDIR "build/cmake" CMAKE_ARGS ${ZSTD_CMAKE_ARGS}) externalproject_add(zstd_ep @@ -1291,14 +1279,8 @@ macro(build_jindosdk_nextarch) get_target_property(JINDOSDK_C_LIBRARY_LOCATION jindosdk::c_sdk IMPORTED_LOCATION) get_filename_component(JINDOSDK_C_DIR_ROOT "${JINDOSDK_C_INCLUDE_DIR}" DIRECTORY) - # Compile flags for jindosdk-nextarch - set(JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") - set(JINDOSDK_NEXTARCH_CMAKE_C_FLAGS "${EP_C_FLAGS}") set(JINDOSDK_NEXTARCH_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${JINDOSDK_NEXTARCH_PREFIX}" - "-DCMAKE_CXX_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${JINDOSDK_NEXTARCH_CMAKE_C_FLAGS}" + ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${JINDOSDK_NEXTARCH_PREFIX}" -DJINDOSDK_ROOT=${JINDOSDK_C_DIR_ROOT} -DJINDOSDK_LIBRARY_NAME=${JINDOSDK_C_DYNAMIC_LIB_NAME}) @@ -1429,14 +1411,9 @@ macro(build_avro) get_target_property(AVRO_FMT_INCLUDE_DIR fmt INTERFACE_INCLUDE_DIRECTORIES) get_filename_component(AVRO_FMT_ROOT "${AVRO_FMT_INCLUDE_DIR}" DIRECTORY) - set(AVRO_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(AVRO_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - set(AVRO_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${AVRO_PREFIX}" - "-DCMAKE_CXX_FLAGS=${AVRO_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${AVRO_CMAKE_C_FLAGS}" "-DAVRO_BUILD_TESTS=OFF" "-DAVRO_BUILD_EXECUTABLES=OFF" "-DZLIB_ROOT=${AVRO_ZLIB_ROOT}" @@ -1493,13 +1470,6 @@ macro(build_orc) "-DCMAKE_MODULE_LINKER_FLAGS=-Wl,-rpath=${ORC_RPATH}") endif() - string(REPLACE "-Werror" "" EP_CXX_FLAGS ${EP_CXX_FLAGS}) - - set(ORC_CMAKE_CXX_FLAGS - "${EP_CXX_FLAGS} -fPIC -Wno-error ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") - set(ORC_CMAKE_C_FLAGS - "${EP_C_FLAGS} -fPIC -Wno-error ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") - set(ORC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/orc_ep-prefix") set(ORC_INCLUDE_DIR "${ORC_PREFIX}/include") set(ORC_SOURCE_DIR "${ORC_PREFIX}/cpp") @@ -1507,16 +1477,9 @@ macro(build_orc) set(ORC_STATIC_LIB "${ORC_PREFIX}/lib/liborc.a") - message("ORC_STATIC_LIB IS ${ORC_STATIC_LIB}") - message("ORC_CMAKE_CXX_FLAGS ${ORC_CMAKE_CXX_FLAGS}") - message("ORC_CMAKE_C_FLAGS ${ORC_CMAKE_C_FLAGS}") - set(ORC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ORC_PREFIX}" - "-DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ORC_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ORC_CMAKE_CXX_FLAGS}" ${ORC_LINKER_FLAGS} "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}" "-DLZ4_HOME=${ORC_LZ4_ROOT}" @@ -1582,9 +1545,7 @@ macro(build_arrow) get_target_property(ARROW_RE2_INCLUDE_DIR re2::re2 INTERFACE_INCLUDE_DIRECTORIES) get_filename_component(ARROW_RE2_ROOT "${ARROW_RE2_INCLUDE_DIR}" DIRECTORY) - set(ARROW_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(ARROW_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" ARROW_CMAKE_CXX_FLAGS ${ARROW_CMAKE_CXX_FLAGS}) + set(ARROW_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") # Fix for thrift Mutex.h missing #include (GCC 15 strictness) # Use -include to force include cstdint for all C++ files string(APPEND ARROW_CMAKE_CXX_FLAGS " -include cstdint") @@ -1619,8 +1580,6 @@ macro(build_arrow) ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}" "-DCMAKE_CXX_FLAGS=${ARROW_CMAKE_CXX_FLAGS}" - "-DCMAKE_C_FLAGS=${ARROW_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${ARROW_CMAKE_CXX_FLAGS}" -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_DEPENDENCY_USE_SHARED=OFF -DARROW_BUILD_SHARED=OFF @@ -1741,9 +1700,6 @@ endmacro(build_arrow) macro(build_gtest) message(STATUS "Building gtest from source") - set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" GTEST_CMAKE_CXX_FLAGS ${GTEST_CMAKE_CXX_FLAGS}) - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-install") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") @@ -1764,10 +1720,7 @@ macro(build_gtest) "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main.a") endif() set(GTEST_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" - "-DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${GTEST_CMAKE_CXX_FLAGS}" + ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" "-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${_GTEST_RUNTIME_DIR}" "-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_${CMAKE_BUILD_TYPE}=${_GTEST_RUNTIME_DIR}") @@ -1806,9 +1759,8 @@ endmacro() macro(build_tbb) message(STATUS "Building Tbb from source") - set(TBB_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -Wno-error") - set(TBB_CMAKE_C_FLAGS "${EP_C_FLAGS} -Wno-error") - string(REPLACE "-Werror" "" TBB_CMAKE_CXX_FLAGS ${TBB_CMAKE_CXX_FLAGS}) + set(TBB_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") + set(TBB_CMAKE_C_FLAGS "${EP_C_FLAGS}") string(REPLACE "-Wdocumentation" "" TBB_CMAKE_CXX_FLAGS ${TBB_CMAKE_CXX_FLAGS}) string(REPLACE "-Wdocumentation" "" TBB_CMAKE_C_FLAGS ${TBB_CMAKE_C_FLAGS}) @@ -1831,7 +1783,6 @@ macro(build_tbb) "-DCMAKE_INSTALL_PREFIX=${TBB_PREFIX}" "-DCMAKE_CXX_FLAGS=${TBB_CMAKE_CXX_FLAGS}" "-DCMAKE_C_FLAGS=${TBB_CMAKE_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${TBB_CMAKE_CXX_FLAGS}" -DTBB_TEST=OFF) externalproject_add(tbb_ep @@ -1903,20 +1854,9 @@ macro(build_glog) set(GLOG_LIB_SUFFIX "") endif() set(GLOG_STATIC_LIB "${GLOG_PREFIX}/lib/libglog${GLOG_LIB_SUFFIX}.a") - set(GLOG_CMAKE_CXX_FLAGS " -Wno-error ${EP_CXX_FLAGS}") - set(GLOG_CMAKE_C_FLAGS " -Wno-error ${EP_C_FLAGS}") - if(CMAKE_THREAD_LIBS_INIT) - string(APPEND GLOG_CMAKE_CXX_FLAGS " ${CMAKE_THREAD_LIBS_INIT}") - string(APPEND GLOG_CMAKE_C_FLAGS " ${CMAKE_THREAD_LIBS_INIT}") - endif() - set(GLOG_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${GLOG_PREFIX} - -DWITH_GFLAGS=OFF - -DWITH_GTEST=OFF - -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS} - -DCMAKE_C_FLAGS=${GLOG_CMAKE_C_FLAGS}) + set(GLOG_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${GLOG_PREFIX} + -DWITH_GFLAGS=OFF -DWITH_GTEST=OFF) if(NOT LIBUNWIND_LIBRARY) list(APPEND GLOG_CMAKE_ARGS -DWITH_UNWIND=none) endif() diff --git a/cmake_modules/san-config.cmake b/cmake_modules/san-config.cmake index 7b985e5b..a2ab16ae 100644 --- a/cmake_modules/san-config.cmake +++ b/cmake_modules/san-config.cmake @@ -16,6 +16,11 @@ add_library(paimon_sanitizer_flags INTERFACE) +if(PAIMON_USE_ASAN AND PAIMON_USE_TSAN) + message(FATAL_ERROR "Address Sanitizer and Thread Sanitizer cannot be enabled together" + ) +endif() + if(PAIMON_USE_ASAN) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") target_compile_options(paimon_sanitizer_flags INTERFACE -fsanitize=address @@ -27,6 +32,19 @@ if(PAIMON_USE_ASAN) endif() endif() +if(PAIMON_USE_TSAN) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + # Bundled dependencies are linked statically into Paimon. Instrument them too so TSAN can + # observe their synchronization primitives and does not report false races at the boundary. + string(APPEND CMAKE_C_FLAGS " -fsanitize=thread -fno-omit-frame-pointer") + string(APPEND CMAKE_CXX_FLAGS " -fsanitize=thread -fno-omit-frame-pointer") + target_link_options(paimon_sanitizer_flags INTERFACE -fsanitize=thread) + message(STATUS "Thread Sanitizer enabled") + else() + message(WARNING "Thread Sanitizer is only supported for GCC and Clang compilers") + endif() +endif() + if(PAIMON_USE_UBSAN) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") target_compile_options(paimon_sanitizer_flags diff --git a/src/paimon/core/operation/metrics/compaction_metrics.h b/src/paimon/core/operation/metrics/compaction_metrics.h index aa48a5e4..24a7169f 100644 --- a/src/paimon/core/operation/metrics/compaction_metrics.h +++ b/src/paimon/core/operation/metrics/compaction_metrics.h @@ -52,16 +52,16 @@ class CompactionMetrics { : metrics_(metrics), partition_(partition), bucket_(bucket) {} void ReportLevel0FileCount(int64_t count) { - level0_file_count_ = count; + level0_file_count_.store(count, std::memory_order_relaxed); } void ReportCompactionInputSize(int64_t bytes) { - compaction_input_size_ = bytes; + compaction_input_size_.store(bytes, std::memory_order_relaxed); } void ReportCompactionOutputSize(int64_t bytes) { - compaction_output_size_ = bytes; + compaction_output_size_.store(bytes, std::memory_order_relaxed); } void ReportTotalFileSize(int64_t bytes) { - total_file_size_ = bytes; + total_file_size_.store(bytes, std::memory_order_relaxed); } void ReportCompactionTime(int64_t time) { metrics_->ReportCompactionTime(time); @@ -84,19 +84,19 @@ class CompactionMetrics { } int64_t Level0FileCount() const { - return level0_file_count_; + return level0_file_count_.load(std::memory_order_relaxed); } int64_t CompactionInputSize() const { - return compaction_input_size_; + return compaction_input_size_.load(std::memory_order_relaxed); } int64_t CompactionOutputSize() const { - return compaction_output_size_; + return compaction_output_size_.load(std::memory_order_relaxed); } int64_t TotalFileSize() const { - return total_file_size_; + return total_file_size_.load(std::memory_order_relaxed); } private: @@ -105,10 +105,10 @@ class CompactionMetrics { int32_t bucket_; // Data fields for metrics. - int64_t level0_file_count_ = 0; - int64_t compaction_input_size_ = 0; - int64_t compaction_output_size_ = 0; - int64_t total_file_size_ = 0; + std::atomic level0_file_count_ = {0}; + std::atomic compaction_input_size_ = {0}; + std::atomic compaction_output_size_ = {0}; + std::atomic total_file_size_ = {0}; }; std::shared_ptr CreateReporter(const BinaryRow& partition, int32_t bucket) { diff --git a/src/paimon/core/operation/metrics/compaction_metrics_test.cpp b/src/paimon/core/operation/metrics/compaction_metrics_test.cpp index 56919dd9..b8f27424 100644 --- a/src/paimon/core/operation/metrics/compaction_metrics_test.cpp +++ b/src/paimon/core/operation/metrics/compaction_metrics_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/operation/metrics/compaction_metrics.h" #include +#include #include "gtest/gtest.h" #include "paimon/testing/utils/testharness.h" @@ -149,4 +150,38 @@ TEST(CompactionMetricsTest, TestCompactionTimeWindow) { EXPECT_DOUBLE_EQ(60.5, avg_time); } +TEST(CompactionMetricsTest, TestConcurrentReporterUpdateAndMetricsSnapshot) { + CompactionMetrics metrics; + std::shared_ptr reporter = + metrics.CreateReporter(BinaryRow::EmptyRow(), 0); + + constexpr int64_t kLastValue = 9999; + std::thread writer([&reporter]() { + for (int64_t value = 0; value <= kLastValue; ++value) { + reporter->ReportLevel0FileCount(value); + reporter->ReportCompactionInputSize(value); + reporter->ReportCompactionOutputSize(value); + reporter->ReportTotalFileSize(value); + } + }); + for (int64_t i = 0; i <= kLastValue; ++i) { + metrics.GetMetrics(); + } + writer.join(); + + std::shared_ptr snapshot = metrics.GetMetrics(); + ASSERT_OK_AND_ASSIGN(double level0_file_count, + snapshot->GetGauge(CompactionMetrics::MAX_LEVEL0_FILE_COUNT)); + ASSERT_OK_AND_ASSIGN(double compaction_input_size, + snapshot->GetGauge(CompactionMetrics::MAX_COMPACTION_INPUT_SIZE)); + ASSERT_OK_AND_ASSIGN(double compaction_output_size, + snapshot->GetGauge(CompactionMetrics::MAX_COMPACTION_OUTPUT_SIZE)); + ASSERT_OK_AND_ASSIGN(double total_file_size, + snapshot->GetGauge(CompactionMetrics::MAX_TOTAL_FILE_SIZE)); + ASSERT_DOUBLE_EQ(static_cast(kLastValue), level0_file_count); + ASSERT_DOUBLE_EQ(static_cast(kLastValue), compaction_input_size); + ASSERT_DOUBLE_EQ(static_cast(kLastValue), compaction_output_size); + ASSERT_DOUBLE_EQ(static_cast(kLastValue), total_file_size); +} + } // namespace paimon::test diff --git a/src/paimon/testing/utils/counting_cache_test_utils.h b/src/paimon/testing/utils/counting_cache_test_utils.h index 7bd806e9..bc22fee3 100644 --- a/src/paimon/testing/utils/counting_cache_test_utils.h +++ b/src/paimon/testing/utils/counting_cache_test_utils.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "paimon/cache/cache.h" @@ -48,16 +49,23 @@ class CountingRoutingCache : public Cache { const std::shared_ptr& key, std::function>(const std::shared_ptr&)> supplier) override { - ++get_count_; - last_kind_ = key->GetKind(); - ++get_count_by_kind_[key->GetKind()]; + CacheKind kind = key->GetKind(); + { + std::lock_guard lock(count_mutex_); + ++get_count_; + last_kind_ = kind; + ++get_count_by_kind_[kind]; + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr cache, GetCache(key)); return cache->Get( key, [this, supplier = std::move(supplier)](const std::shared_ptr& supplier_key) -> Result> { - ++supplier_call_count_; - ++supplier_call_count_by_kind_[supplier_key->GetKind()]; + { + std::lock_guard lock(count_mutex_); + ++supplier_call_count_; + ++supplier_call_count_by_kind_[supplier_key->GetKind()]; + } return supplier(supplier_key); }); } @@ -90,22 +98,27 @@ class CountingRoutingCache : public Cache { } int64_t GetCount() const { + std::lock_guard lock(count_mutex_); return get_count_; } int64_t GetCount(CacheKind kind) const { + std::lock_guard lock(count_mutex_); return GetCount(get_count_by_kind_, kind); } int64_t SupplierCallCount() const { + std::lock_guard lock(count_mutex_); return supplier_call_count_; } int64_t SupplierCallCount(CacheKind kind) const { + std::lock_guard lock(count_mutex_); return GetCount(supplier_call_count_by_kind_, kind); } CacheKind LastKind() const { + std::lock_guard lock(count_mutex_); return last_kind_; } @@ -132,6 +145,7 @@ class CountingRoutingCache : public Cache { int64_t get_count_ = 0; int64_t supplier_call_count_ = 0; CacheKind last_kind_ = CacheKind::DEFAULT; + mutable std::mutex count_mutex_; }; } // namespace paimon::test diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 1fd010ae..d6757150 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -16,6 +16,7 @@ * limitations under the License. */ +#include #include #include #include @@ -3582,7 +3583,8 @@ TEST_P(ReadInteTest, TestReadWithAppendPtBranch) { TEST_P(ReadInteTest, TestSpecificFs) { class CountableInputStream : public InputStream { public: - CountableInputStream(const std::shared_ptr& input, size_t* io_count) + CountableInputStream(const std::shared_ptr& input, + std::atomic* io_count) : input_(input), io_count_(io_count) {} ~CountableInputStream() override = default; @@ -3593,16 +3595,16 @@ TEST_P(ReadInteTest, TestSpecificFs) { return input_->GetPos(); } Result Read(char* buffer, int64_t size) override { - (*io_count_)++; + io_count_->fetch_add(1, std::memory_order_relaxed); return input_->Read(buffer, size); } Result Read(char* buffer, int64_t size, int64_t offset) override { - (*io_count_)++; + io_count_->fetch_add(1, std::memory_order_relaxed); return input_->Read(buffer, size, offset); } void ReadAsync(char* buffer, int64_t size, int64_t offset, std::function&& callback) override { - (*io_count_)++; + io_count_->fetch_add(1, std::memory_order_relaxed); return input_->ReadAsync(buffer, size, offset, std::move(callback)); } @@ -3617,12 +3619,12 @@ TEST_P(ReadInteTest, TestSpecificFs) { } std::shared_ptr input_; - size_t* io_count_; + std::atomic* io_count_; }; class CountableFileSystem : public FileSystem { public: - CountableFileSystem(const std::shared_ptr& fs, size_t* io_count) + CountableFileSystem(const std::shared_ptr& fs, std::atomic* io_count) : fs_(fs), io_count_(io_count) {} ~CountableFileSystem() override = default; @@ -3660,10 +3662,10 @@ TEST_P(ReadInteTest, TestSpecificFs) { } std::shared_ptr fs_; - size_t* io_count_; + std::atomic* io_count_; }; - size_t io_count = 0; + std::atomic io_count = {0}; auto param = GetParam(); std::string path = paimon::test::GetDataDir() + "/" + param.file_format + "/append_09.db/append_09"; @@ -3716,7 +3718,7 @@ TEST_P(ReadInteTest, TestSpecificFs) { &expected_array); ASSERT_TRUE(array_status.ok()); ASSERT_TRUE(result_array->Equals(expected_array)); - ASSERT_GT(io_count, 0); + ASSERT_GT(io_count.load(std::memory_order_relaxed), 0); } } // namespace paimon::test From d84e29a72aa061e2a67f1d5ffde21b0f2a92308a Mon Sep 17 00:00:00 2001 From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:26:56 +0800 Subject: [PATCH 104/138] feat(parquet): support configurable bitmap row-range refining strategies (coalesce and trim) --- .../page_filtered_row_group_reader_test.cpp | 227 ++++++++++++++++-- .../parquet/parquet_file_batch_reader.cpp | 124 +++++++++- .../parquet/parquet_file_batch_reader.h | 20 +- .../parquet_file_batch_reader_test.cpp | 5 +- .../format/parquet/parquet_format_defs.h | 19 ++ 5 files changed, 356 insertions(+), 39 deletions(-) diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 35cffc12..75bf7522 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -139,15 +139,12 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { const std::shared_ptr& predicate, const RoaringBitmap32& bitmap, std::shared_ptr* out, - int32_t batch_size = 1024, - bool enable_page_level_filter = true) { + const std::map options = {}, + int32_t batch_size = 1024) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = std::make_shared(in, arrow_pool_, length); - std::map options; - options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = - enable_page_level_filter ? "true" : "false"; ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size, nullptr, arrow_pool_)); @@ -1293,15 +1290,16 @@ TEST_F(PageFilteredRowGroupReaderTest, BitmapAllPagesSomeRowGroups) { /// Test: bitmap hits partial pages of a row group (no predicate). /// /// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups. -/// Bitmap: {30..59} hits pages 3-5 of RG0 (rows 30-59), RG1 excluded. -/// Expected: 30 rows (30-59). -TEST_F(PageFilteredRowGroupReaderTest, BitmapPartialPagesSingleRowGroup) { +/// Bitmap: {90..109} hits page 9 of RG0 (rows 90-99), and page 0 of RG1 (rows 100-109). +/// Expected: 20 rows (90-109). +TEST_F(PageFilteredRowGroupReaderTest, BitmapPartialPagesAcrossRowGroups) { std::string file_name = dir_->Str() + "/bitmap_partial_pages_rg.parquet"; auto data = MakeSequentialIntData(200); WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100); RoaringBitmap32 bitmap; - bitmap.AddRange(90, 110); // hits pages 3-5 of RG0 + // {90..109} hits page 9 of RG0 (rows 90-99), and page 0 of RG1 (rows 100-109). + bitmap.AddRange(90, 110); auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); std::shared_ptr result; @@ -1367,8 +1365,10 @@ TEST_F(PageFilteredRowGroupReaderTest, BitmapWithPageFilteredOptionDisabled) { auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); std::shared_ptr result; + std::map options; + options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "false"; ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, - 1024, false); + options); ASSERT_TRUE(result); ASSERT_EQ(100, result->length()); @@ -1490,42 +1490,217 @@ TEST_F(PageFilteredRowGroupReaderTest, BitmapMixedWithPredicate) { } } -/// Test: read parquet with scattered bitmap +/// Test: coalesce strategy with default hole_size_limit (32). /// /// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups. -/// Bitmap: [20,30), [35, 40), [125, 126), [130, 131), [150, 200) -/// To test if unneeded row at the start and end of pages are filtered out. -/// Expected: 76 rows ([20, 40) + [125, 131) + [150, 200). -TEST_F(PageFilteredRowGroupReaderTest, ScatteredBitmapTest) { +/// Bitmap: [0,10), [45, 50), [60, 70). +/// - [0,10) and [45,50) are both in RG0 page 0 ([0,49]); gap = 35 > 32, NOT merged. +/// - [45,50) and [60,70) straddle RG0 page 0 ([0,49]) and page 1 ([50,99]); gap = 10 <= 32, +/// merged across the page boundary, so rows 50-59 are read even though not in the bitmap. +/// Expected: 35 rows ([0,10) + [45, 70)). +TEST_F(PageFilteredRowGroupReaderTest, BitmapCoalesceTest) { std::string file_name = dir_->Str() + "/scattered_bitmap.parquet"; auto data = MakeSequentialIntData(200); WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); RoaringBitmap32 bitmap; - bitmap.AddRange(20, 30); - bitmap.AddRange(35, 40); - bitmap.Add(125); - bitmap.Add(130); - bitmap.AddRange(150, 200); + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); std::shared_ptr result; ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result); ASSERT_TRUE(result); - ASSERT_EQ(76, result->length()); + ASSERT_EQ(35, result->length()); auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); auto struct_arr = std::dynamic_pointer_cast(flat); ASSERT_TRUE(struct_arr); auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); - for (int32_t i = 0; i < 20; ++i) { - ASSERT_EQ(20 + i, val_arr->Value(i)); + // [0, 9] — not merged (gap to next range = 35 > 32) + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(0 + i, val_arr->Value(i)); + } + // [45, 69] — merged across page boundary (gap = 10 <= 32) + for (int32_t i = 0; i < 25; ++i) { + ASSERT_EQ(45 + i, val_arr->Value(10 + i)); + } +} + +/// Test: coalesce strategy with hole_size_limit=5 (instead of default 32). +/// +/// Same bitmap as BitmapCoalesceTest: [0,10), [45, 50), [60, 70). +/// Both gaps (35 and 10) exceed 5, so no ranges are merged. +/// Expected: 25 rows (exact bitmap selection, no holes filled). +/// Compare with BitmapCoalesceTest which gets 35 rows (gap of 10 is filled with default limit=32). +TEST_F(PageFilteredRowGroupReaderTest, BitmapCoalesceSmallHoleSizeTest) { + std::string file_name = dir_->Str() + "/coalesce_small_hole.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); + + std::map options; + options[PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT] = "5"; + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, + options); + ASSERT_TRUE(result); + ASSERT_EQ(25, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + ASSERT_TRUE(val_arr); + // [0, 9] — not merged (gap = 35 > 5) + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(0 + i, val_arr->Value(i)); + } + // [45, 49] — not merged (gap = 10 > 5) + for (int32_t i = 0; i < 5; ++i) { + ASSERT_EQ(45 + i, val_arr->Value(10 + i)); + } + // [60, 69] — not merged + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(60 + i, val_arr->Value(15 + i)); + } +} + +/// Test: trim strategy for bitmap row filtering. +/// +/// Same bitmap as BitmapCoalesceTest: [0,10), [45, 50), [60, 70). +/// Trim produces one range per page (trimmed to first/last selected row in that page): +/// - RG0 page 0 ([0,49]): first selected = 0, last selected = 49 → [0, 49] (50 rows, includes +/// the 35-row gap 10-44 that coalesce keeps as a hole because gap = 35 > 32). +/// - RG0 page 1 ([50,99]): first selected = 60, last selected = 69 → [60, 69] (10 rows; rows +/// 50-59 are NOT read, unlike coalesce which merges across the page boundary). +/// Expected: 60 rows ([0, 50) + [60, 70)). +/// Compare with BitmapCoalesceTest which gets 35 rows. +TEST_F(PageFilteredRowGroupReaderTest, BitmapTrimStrategyTest) { + std::string file_name = dir_->Str() + "/trim_strategy.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); + + std::map options; + options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "trim"; + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, + options); + ASSERT_TRUE(result); + ASSERT_EQ(60, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto val_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + ASSERT_TRUE(val_arr); + // RG0 page 0 trimmed to [0, 49] — includes gap 10-44 that coalesce skips + for (int32_t i = 0; i < 50; ++i) { + ASSERT_EQ(0 + i, val_arr->Value(i)); } - for (int32_t i = 0; i < 6; ++i) { - ASSERT_EQ(125 + i, val_arr->Value(20 + i)); + // RG0 page 1 trimmed to [60, 69] — rows 50-59 not read (unlike coalesce) + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(60 + i, val_arr->Value(50 + i)); } +} + +/// Test: invalid strategy value returns Status::Invalid. +/// +/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups. +/// Bitmap: [0,10), [45, 50), [60, 70) — same as BitmapCoalesceTest. +/// Strategy: "invalid" (not one of "coalesce", "trim", "none"). +/// Expected: SetReadSchema returns Status::Invalid. +TEST_F(PageFilteredRowGroupReaderTest, BitmapInvalidStrategyTest) { + std::string file_name = dir_->Str() + "/invalid_strategy.parquet"; + auto data = MakeSequentialIntData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); + + std::map options; + options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "invalid"; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); + ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); + auto in_stream = std::make_shared(in, arrow_pool_, length); + + ASSERT_OK_AND_ASSIGN( + auto batch_reader, + ParquetFileBatchReader::Create(std::move(in_stream), options, 1024, nullptr, arrow_pool_)); + + auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); + + auto status = batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, bitmap); + ASSERT_FALSE(status.ok()); + ASSERT_TRUE(status.IsInvalid()); +} + +/// Test: trim strategy with multiple columns — intersection of per-column trimmed ranges. +/// +/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups. +/// Two columns: a[i] = i, b[i] = i * 10. +/// Bitmap: [0,10), [45, 50), [60, 70) — same as BitmapTrimStrategyTest. +/// Both columns share the same page boundaries, so their trimmed ranges are identical. +/// The intersection is the same as either column alone: +/// - RG0 page 0 ([0,49]): trimmed to [0, 49] (50 rows) +/// - RG0 page 1 ([50,99]): trimmed to [60, 69] (10 rows) +/// Expected: 60 rows. Both columns must remain aligned after trimming. +TEST_F(PageFilteredRowGroupReaderTest, BitmapTrimMultiColumnTest) { + std::string file_name = dir_->Str() + "/trim_multi_col.parquet"; + auto data = MakeTwoColumnData(200); + WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100); + + RoaringBitmap32 bitmap; + bitmap.AddRange(0, 10); + bitmap.AddRange(45, 50); + bitmap.AddRange(60, 70); + + std::map options; + options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "trim"; + + auto read_schema = + arrow::schema({arrow::field("a", arrow::int32()), arrow::field("b", arrow::int32())}); + std::shared_ptr result; + ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result, + options); + ASSERT_TRUE(result); + ASSERT_EQ(60, result->length()); + + auto flat = arrow::Concatenate(result->chunks()).ValueOrDie(); + auto struct_arr = std::dynamic_pointer_cast(flat); + ASSERT_TRUE(struct_arr); + auto a_arr = std::dynamic_pointer_cast(struct_arr->field(0)); + auto b_arr = std::dynamic_pointer_cast(struct_arr->field(1)); + ASSERT_TRUE(a_arr); + ASSERT_TRUE(b_arr); + // RG0 page 0 trimmed to [0, 49] for (int32_t i = 0; i < 50; ++i) { - ASSERT_EQ(150 + i, val_arr->Value(26 + i)); + ASSERT_EQ(i, a_arr->Value(i)); + ASSERT_EQ(i * 10, b_arr->Value(i)); + } + // RG0 page 1 trimmed to [60, 69] + for (int32_t i = 0; i < 10; ++i) { + ASSERT_EQ(60 + i, a_arr->Value(50 + i)); + ASSERT_EQ((60 + i) * 10, b_arr->Value(50 + i)); } } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index bbff55c1..8c44572f 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -179,9 +179,27 @@ Status ParquetFileBatchReader::SetReadSchema( // workaround: page index filter does not support nested fields for now, skip page index // bitmap pushdown if there is any nested field in the schema if (!has_nested_field && enable_page_index_filter) { - PAIMON_ASSIGN_OR_RAISE(target_row_groups, - FilterPagesByBitmap(selection_bitmap.value(), - target_row_groups, column_indices)); + // To decide which strategy to use, "trim" or "coalesce". "Coalesce" By default. + PAIMON_ASSIGN_OR_RAISE( + std::string strategy, + OptionsUtils::GetValueFromMap( + options_, PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY, + DEFAULT_PARQUET_READ_BITMAP_STRATEGY)); + if (strategy == "trim") { + PAIMON_ASSIGN_OR_RAISE( + target_row_groups, + RefineRowRangesByTrimming(selection_bitmap.value(), target_row_groups, + column_indices)); + } else if (strategy == "coalesce") { + PAIMON_ASSIGN_OR_RAISE( + target_row_groups, + RefineRowRangesByCoalescing(selection_bitmap.value(), target_row_groups)); + } else { + return Status::Invalid( + fmt::format("Invalid row range refining strategy :{}, valid strategies " + "are: trim, coalesce", + strategy)); + } } } // Apply page-level filtering after bitmap pruning so we don't read page index @@ -297,7 +315,96 @@ Result ParquetFileBatchReader::FilterRowGroupsByBitmap( return target_row_groups; } -Result ParquetFileBatchReader::FilterPagesByBitmap( +RowRanges ParquetFileBatchReader::CoalesceNearbyRanges(const RowRanges& input, + uint64_t hole_size_limit) { + if (input.IsEmpty()) { + return RowRanges(); + } + + const auto& ranges = input.GetRanges(); + RowRanges result; + int64_t merge_start = ranges.front().from; + int64_t merge_end = ranges.front().to; + + for (size_t i = 1; i < ranges.size(); ++i) { + // Gap between [merge_start, merge_end] and [ranges[i].from, ranges[i].to] + int64_t gap = ranges[i].from - merge_end - 1; + if (static_cast(gap) > hole_size_limit) { + result.Add(RowRanges::Range(merge_start, merge_end)); + merge_start = ranges[i].from; + } + merge_end = ranges[i].to; + } + result.Add(RowRanges::Range(merge_start, merge_end)); + return result; +} + +RowRanges ParquetFileBatchReader::BitmapToContiguousRanges(const RoaringBitmap32& bitmap, + uint64_t start_row, uint64_t end_row) { + RowRanges ranges; + if (bitmap.IsEmpty() || start_row >= end_row) { + return ranges; + } + + auto it = bitmap.EqualOrLarger(static_cast(start_row)); + const auto end = bitmap.End(); + if (it == end || static_cast(*it) >= end_row) { + return ranges; + } + + auto run_start = static_cast(*it); + auto prev = run_start; + + for (++it; it != end; ++it) { + auto current = static_cast(*it); + if (current >= static_cast(end_row)) { + break; + } + if (current != prev + 1) { + ranges.Add(RowRanges::Range(run_start - start_row, prev - start_row)); + run_start = current; + } + prev = current; + } + ranges.Add(RowRanges::Range(run_start - start_row, prev - start_row)); + return ranges; +} + +Result ParquetFileBatchReader::RefineRowRangesByCoalescing( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups) const { + PAIMON_ASSIGN_OR_RAISE(const uint64_t hole_size_limit, + OptionsUtils::GetValueFromMap( + options_, PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT, + DEFAULT_PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT)); + + const auto& all_row_group_ranges = reader_->GetAllRowGroupRanges(); + TargetRowGroups target_row_groups; + target_row_groups.reserve(src_row_groups.size()); + + for (const auto& row_group : src_row_groups) { + int32_t rg_index = row_group.GetRowGroupIndex(); + uint64_t rg_start_row = all_row_group_ranges[rg_index].first; + uint64_t rg_end_row = all_row_group_ranges[rg_index].second; + + // Step 1: bitmap -> contiguous ranges (relative to row group start). + // Step 2: coalesce ranges with small gaps to reduce range count. + RowRanges contiguous = BitmapToContiguousRanges(bitmap, rg_start_row, rg_end_row); + RowRanges coalesced = CoalesceNearbyRanges(contiguous, hole_size_limit); + + auto rg_row_count = static_cast(rg_end_row - rg_start_row); + if (coalesced.IsEmpty()) { + continue; + } + if (coalesced.RowCount() == rg_row_count) { + target_row_groups.emplace_back(row_group); + } else { + target_row_groups.emplace_back(rg_index, true, std::move(coalesced)); + } + } + return target_row_groups; +} + +Result ParquetFileBatchReader::RefineRowRangesByTrimming( const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups, const std::vector& column_indices) const { auto page_index_reader = reader_->GetPageIndexReader(); @@ -308,13 +415,16 @@ Result ParquetFileBatchReader::FilterPagesByBitmap( TargetRowGroups target_row_groups; target_row_groups.reserve(src_row_groups.size()); for (const auto& row_group : src_row_groups) { - target_row_groups.emplace_back( - FilterRowGroupPagesByBitmap(bitmap, row_group, column_indices, page_index_reader)); + auto filtered = + TrimRowGroupPageRanges(bitmap, row_group, column_indices, page_index_reader); + if (!filtered.GetRowRanges().IsEmpty()) { + target_row_groups.emplace_back(std::move(filtered)); + } } return target_row_groups; } -TargetRowGroup ParquetFileBatchReader::FilterRowGroupPagesByBitmap( +TargetRowGroup ParquetFileBatchReader::TrimRowGroupPageRanges( const RoaringBitmap32& bitmap, const TargetRowGroup& row_group, const std::vector& column_indices, const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader) const { diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 7f6cab1f..308e7dfa 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -202,18 +202,30 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { Result FilterRowGroupsByBitmap(const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups) const; - Result FilterPagesByBitmap(const RoaringBitmap32& bitmap, - const TargetRowGroups& src_row_groups, - const std::vector& column_indices) const; + // Apply bitmap filtering to row ranges by trimming start and end rows in pages. + // Then apply intersection among all target columns. + Result RefineRowRangesByTrimming( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups, + const std::vector& column_indices) const; // Apply page-level bitmap filtering to a single row group across all // requested columns. Intersects the row group's existing ranges with the // per-column page ranges derived from the bitmap. - TargetRowGroup FilterRowGroupPagesByBitmap( + TargetRowGroup TrimRowGroupPageRanges( const RoaringBitmap32& bitmap, const TargetRowGroup& row_group, const std::vector& column_indices, const std::shared_ptr<::parquet::PageIndexReader>& page_index_reader) const; + // Apply bitmap filtering to row ranges by coalescing nearby ranges. + Result RefineRowRangesByCoalescing( + const RoaringBitmap32& bitmap, const TargetRowGroups& src_row_groups) const; + // Convert bitmap set bits within [start_row, end_row) to contiguous + // row ranges, stored relative to start_row. + static RowRanges BitmapToContiguousRanges(const RoaringBitmap32& bitmap, uint64_t start_row, + uint64_t end_row); + // Merge ranges whose inter-range gap is <= hole_size_limit. + static RowRanges CoalesceNearbyRanges(const RowRanges& input, uint64_t hole_size_limit); + // Compute the set of row ranges within a single column's pages that // overlap with the given bitmap. For each page, the bitmap is queried to // find the first/last matching row in each page, used to trim the page head/tail diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index 9ca4157f..1c40b230 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -905,8 +905,9 @@ TEST_F(ParquetFileBatchReaderTest, TestPredicateAndBitmapPagePushDown) { std::optional bitmap = RoaringBitmap32::From({100, 400, 600}); auto predicate = PredicateBuilder::GreaterThan(/*field_index=*/0, /*field_name=*/"f0", FieldType::INT, Literal(800)); - auto parquet_batch_reader = PrepareParquetFileBatchReader( - file_path_, arrow_schema, predicate, bitmap, /*batch_size=*/length); + auto parquet_batch_reader = + PrepareParquetFileBatchReader(file_path_, arrow_schema, predicate, bitmap, + /*batch_size=*/length, /*enable_page_level_filter=*/true); ASSERT_OK_AND_ASSIGN( std::shared_ptr result_array, paimon::test::ReadResultCollector::CollectResult(parquet_batch_reader.get())); diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h index 69894092..5cb6f2da 100644 --- a/src/paimon/format/parquet/parquet_format_defs.h +++ b/src/paimon/format/parquet/parquet_format_defs.h @@ -67,6 +67,23 @@ static inline const char PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT[] = "parquet.read.cache-option.prefetch-limit"; static inline const char PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT[] = "parquet.read.cache-option.range-size-limit"; +// Strategy for refining row ranges using the selection bitmap produced by pushed-down +// predicates. Two options: +// * "coalesce" (default): build row-level ranges from the bitmap, then merge nearby +// ranges whose gap is small enough (see PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT). +// * "trim": for each page with selected rows, trim its leading/trailing non-selected +// rows (and skip pages with no selected rows). Requires page index. Its advantage +// is stable, page-bounded ranges. After collected trimmed ranges for each column, +// the intersection of all columns' ranges is taken to produce the final row ranges. +static inline const char PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY[] = + "parquet.read.bitmap.row-range-refining-strategy"; +// When strategy = "coalesce", adjacent bitmap row ranges whose gap (in rows) is +// <= this limit are merged into one range; larger gaps are kept as real holes. +// A larger limit means fewer (larger) ranges and more wasted rows read; a smaller +// limit keeps the selection tighter at the cost of more (smaller) ranges. +// Only takes effect when PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY = "coalesce". +static inline const char PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT[] = + "parquet.read.bitmap.coalesce-hole-size-limit"; // stack-overflow may happen while the number of predicate node is too large, limit the number of // predicate nodes. Predicate will not be pushdown when exceed limit. @@ -84,6 +101,8 @@ static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT = 0; static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT = 32 * 1024 * 1024; static constexpr uint32_t DEFAULT_PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT = 512; static constexpr bool DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER = true; +static constexpr char DEFAULT_PARQUET_READ_BITMAP_STRATEGY[] = "coalesce"; +static constexpr uint32_t DEFAULT_PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT = 32; class ParquetMetrics { public: From 535105886b3078b9840e69cc4d1fefbf46562424 Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Tue, 21 Jul 2026 09:45:03 +0800 Subject: [PATCH 105/138] feat(commit): support TruncateTable/Abort/RollbackToAsLatest and add ut --- include/paimon/file_store_commit.h | 24 + src/paimon/CMakeLists.txt | 1 + src/paimon/common/utils/vector_store_utils.h | 40 + src/paimon/core/core_options.cpp | 2 +- .../io/append_data_file_writer_factory.cpp | 7 +- .../core/io/append_data_file_writer_factory.h | 5 + ...edding_append_data_file_writer_factory.cpp | 3 +- .../commit/commit_changes_provider_test.cpp | 8 +- .../core/operation/commit/commit_scanner.h | 2 +- .../operation/commit/commit_scanner_test.cpp | 6 +- .../compacted_changelog_path_resolver.h | 34 +- .../operation/commit/conflict_detection.cpp | 7 +- .../commit/manifest_entry_changes_test.cpp | 28 +- .../row_id_column_conflict_checker_test.cpp | 16 +- .../commit/row_tracking_commit_utils.cpp | 7 +- .../sequence_snapshot_properties_test.cpp | 179 ++++ .../core/operation/file_store_commit_impl.cpp | 166 ++++ .../core/operation/file_store_commit_impl.h | 8 + .../operation/file_store_commit_impl_test.cpp | 883 ++++++++++++++++++ 19 files changed, 1379 insertions(+), 47 deletions(-) create mode 100644 src/paimon/common/utils/vector_store_utils.h create mode 100644 src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h index b4cd2896..cb4c2f26 100644 --- a/include/paimon/file_store_commit.h +++ b/include/paimon/file_store_commit.h @@ -143,6 +143,30 @@ class PAIMON_EXPORT FileStoreCommit { virtual Status DropPartition(const std::vector>& partitions, int64_t commit_identifier) = 0; + /// Truncate the whole table by overwriting all partitions with empty data. The generated + /// snapshot has commit kind OVERWRITE. + /// + /// @param commit_identifier An identifier for the commit operation. + /// @return Status indicating the success or failure of the truncate operation. + virtual Status TruncateTable(int64_t commit_identifier) = 0; + + /// Abort an unsuccessful commit. The data and index files described by the given commit + /// messages will be deleted on a best-effort basis (delete failures are ignored). + /// + /// @param commit_messages A vector of commit messages whose files should be cleaned up. + /// @return Status indicating the success or failure of the abort operation. + virtual Status Abort(const std::vector>& commit_messages) = 0; + + /// Roll back to the target snapshot and materialize it as the latest snapshot. + /// + /// Reads the surviving files of both the current latest snapshot and the target + /// snapshot, then commits an OVERWRITE snapshot whose visible state equals the target. + /// + /// @param target_snapshot_id The snapshot id to roll back to. + /// @return Result; true if the atomic commit succeeded. Returns an error status if + /// there is no latest snapshot or the target snapshot does not exist. + virtual Result RollbackToAsLatest(int64_t target_snapshot_id) = 0; + /// Configure row-id conflict checking from a specific snapshot id. /// /// If set to a snapshot id, commit conflict detection will additionally validate row-id diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 5c0c7f44..72009c60 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -722,6 +722,7 @@ if(PAIMON_BUILD_TESTS) core/operation/commit/overwrite_changes_provider_test.cpp core/operation/commit/row_id_column_conflict_checker_test.cpp core/operation/commit/row_tracking_commit_utils_test.cpp + core/operation/commit/sequence_snapshot_properties_test.cpp core/operation/commit/retry_waiter_test.cpp core/operation/key_value_file_store_write_test.cpp core/operation/internal_read_context_test.cpp diff --git a/src/paimon/common/utils/vector_store_utils.h b/src/paimon/common/utils/vector_store_utils.h new file mode 100644 index 00000000..c50c0967 --- /dev/null +++ b/src/paimon/common/utils/vector_store_utils.h @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +namespace paimon { + +/// Utils for vector-store files. +/// +/// A vector-store file is identified by the `.vector.` marker in its name. +class VectorStoreUtils { + public: + VectorStoreUtils() = delete; + ~VectorStoreUtils() = delete; + + /// Returns true if `file_name` is a vector-store file (contains the `.vector.` marker). + static bool IsVectorStoreFile(const std::string& file_name) { + return file_name.find(".vector.") != std::string::npos; + } +}; + +} // namespace paimon diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 18987472..9e261fc3 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -688,7 +688,7 @@ struct CoreOptions::Impl { Options::SEQUENCE_FIELD, Options::FIELDS_SEPARATOR, &sequence_field)); // Parse sequence.field.sort-order - order of sequence field, default "ascending" PAIMON_RETURN_NOT_OK(parser.ParseSortOrder(&sequence_field_sort_order)); - // Parse write-sequence-number-init-mode - sequence init mode for write path + // Parse write.sequence-number-init-mode - sequence init mode for write path std::string write_sequence_init_mode_str = "scan"; PAIMON_RETURN_NOT_OK( parser.Parse(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, &write_sequence_init_mode_str)); diff --git a/src/paimon/core/io/append_data_file_writer_factory.cpp b/src/paimon/core/io/append_data_file_writer_factory.cpp index 20e9daec..e12374dd 100644 --- a/src/paimon/core/io/append_data_file_writer_factory.cpp +++ b/src/paimon/core/io/append_data_file_writer_factory.cpp @@ -43,10 +43,13 @@ AppendDataFileWriterFactory::AppendDataFileWriterFactory( file_source_(file_source), path_factory_(path_factory) {} +std::shared_ptr AppendDataFileWriterFactory::ResolveSeqNumCounter() const { + return options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; +} + Result>>> AppendDataFileWriterFactory::CreateWriter() const { - std::shared_ptr seq_num_counter = - options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; + std::shared_ptr seq_num_counter = ResolveSeqNumCounter(); PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*options_.GetFileFormat(), write_schema_, /*create_stats_extractor=*/true)); diff --git a/src/paimon/core/io/append_data_file_writer_factory.h b/src/paimon/core/io/append_data_file_writer_factory.h index be944fc9..e5e9c3d6 100644 --- a/src/paimon/core/io/append_data_file_writer_factory.h +++ b/src/paimon/core/io/append_data_file_writer_factory.h @@ -60,6 +60,11 @@ class AppendDataFileWriterFactory CreateWriter() const override; protected: + // Resolves the sequence-number counter for a newly created writer. When data evolution is + // enabled each file gets a fresh counter starting at 0; otherwise the factory's shared counter + // is reused. + std::shared_ptr ResolveSeqNumCounter() const; + std::shared_ptr write_schema_; std::optional> write_cols_; std::shared_ptr seq_num_counter_; diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp index 0622bb02..67bfef68 100644 --- a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp +++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp @@ -50,8 +50,7 @@ ShreddingAppendDataFileWriterFactory::CreateWriter() const { if (!shredding_context_) { return Status::Invalid("Shared-shredding append writer requires a shredding context."); } - std::shared_ptr seq_num_counter = - options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; + std::shared_ptr seq_num_counter = ResolveSeqNumCounter(); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, MapSharedShreddingBatchConverter::Create( write_schema_, shredding_context_, options_, pool_)); diff --git a/src/paimon/core/operation/commit/commit_changes_provider_test.cpp b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp index 43de67b5..edbb1a48 100644 --- a/src/paimon/core/operation/commit/commit_changes_provider_test.cpp +++ b/src/paimon/core/operation/commit/commit_changes_provider_test.cpp @@ -102,9 +102,9 @@ TEST(CommitChangesProviderTest, TestProvideReturnsGivenEntries) { ASSERT_EQ(delta_files.size(), provided_delta.size()); ASSERT_EQ(changelog_files.size(), provided_changelog.size()); ASSERT_EQ(index_entries.size(), provided_index.size()); - EXPECT_EQ("delta-1", provided_delta[0].FileName()); - EXPECT_EQ("changelog-1", provided_changelog[0].FileName()); - EXPECT_EQ("index-1", provided_index[0].index_file->FileName()); + ASSERT_EQ("delta-1", provided_delta[0].FileName()); + ASSERT_EQ("changelog-1", provided_changelog[0].FileName()); + ASSERT_EQ("index-1", provided_index[0].index_file->FileName()); } TEST(CommitChangesProviderTest, TestProvideUsesCopiedInputs) { @@ -126,7 +126,7 @@ TEST(CommitChangesProviderTest, TestProvideUsesCopiedInputs) { ASSERT_EQ(1u, provided->delta_files.size()); ASSERT_EQ(0u, provided->changelog_files.size()); ASSERT_EQ(0u, provided->index_entries.size()); - EXPECT_EQ("delta-1", provided->delta_files[0].FileName()); + ASSERT_EQ("delta-1", provided->delta_files[0].FileName()); } } // namespace paimon::test diff --git a/src/paimon/core/operation/commit/commit_scanner.h b/src/paimon/core/operation/commit/commit_scanner.h index 6704da6f..552cc648 100644 --- a/src/paimon/core/operation/commit/commit_scanner.h +++ b/src/paimon/core/operation/commit/commit_scanner.h @@ -53,7 +53,7 @@ class Snapshot; class SnapshotManager; class TableSchema; -// Manifest entries scanner for commit operations. +/// Manifest entries scanner for commit operations. class CommitScanner { public: using ScanSupplier = diff --git a/src/paimon/core/operation/commit/commit_scanner_test.cpp b/src/paimon/core/operation/commit/commit_scanner_test.cpp index 4d9f69bd..869da737 100644 --- a/src/paimon/core/operation/commit/commit_scanner_test.cpp +++ b/src/paimon/core/operation/commit/commit_scanner_test.cpp @@ -115,8 +115,8 @@ TEST_F(CommitScannerTest, TestReadAllEntriesFromChangedPartitionsEmptyFastExit) ASSERT_OK_AND_ASSIGN(std::vector entries, scanner.ReadAllEntriesFromChangedPartitions(MakeSnapshot(), /*changed_partitions=*/{})); - EXPECT_TRUE(entries.empty()); - EXPECT_FALSE(supplier_called); + ASSERT_TRUE(entries.empty()); + ASSERT_FALSE(supplier_called); } TEST_F(CommitScannerTest, TestReadAllEntriesFromPartitionsRequiresSupplier) { @@ -147,7 +147,7 @@ TEST_F(CommitScannerTest, TestReadAllEntriesFromChangedPartitionsBuildsScanFilte ASSERT_TRUE(supplier_called); ASSERT_EQ(1u, captured_partition_filters.size()); ASSERT_EQ(1u, captured_partition_filters[0].size()); - EXPECT_EQ("42", captured_partition_filters[0]["pt"]); + ASSERT_EQ("42", captured_partition_filters[0]["pt"]); } } // namespace paimon::test diff --git a/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h index b9232874..427e8f35 100644 --- a/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h +++ b/src/paimon/core/operation/commit/compacted_changelog_path_resolver.h @@ -23,11 +23,41 @@ namespace paimon { +/// Utility class for resolving compacted changelog file paths. +/// +/// This class provides functionality to resolve fake compacted changelog file paths to their real +/// file paths. +/// +/// File Name Protocol +/// +/// There are two kinds of file name. In the following description, `bid1` and `bid2` are bucket +/// id, `off` is offset, `len1` and `len2` are lengths. +/// +/// - `bucket-bid1/compacted-changelog-xxx$bid1-len1`: This is the real file name. If this file +/// name is recorded in manifest file meta, reader should read the bytes of this file starting +/// from offset `0` with length `len1`. +/// - `bucket-bid2/compacted-changelog-xxx$bid1-len1-off-len2`: This is the fake file name. Reader +/// should read the bytes of file `bucket-bid1/compacted-changelog-xxx$bid1-len1` starting from +/// offset `off` with length `len2`. class CompactedChangelogPathResolver { public: - static bool IsCompactedChangelogPath(const std::string& path); - + /// Resolves a file path, handling compacted changelog file path resolution if applicable. + /// + /// For compacted changelog files, resolves fake file paths to their real file paths as + /// described in the protocol above. For non-compacted changelog files, returns the path + /// unchanged. + /// + /// @param path The file path to resolve. + /// @return The resolved real file path for compacted changelog files, or the original path + /// unchanged for other files. static std::string Resolve(const std::string& path); + + private: + /// Checks if the given path is a compacted changelog file path. + /// + /// @param path The file path to check. + /// @return true if the path is a compacted changelog file, false otherwise. + static bool IsCompactedChangelogPath(const std::string& path); }; } // namespace paimon diff --git a/src/paimon/core/operation/commit/conflict_detection.cpp b/src/paimon/core/operation/commit/conflict_detection.cpp index 883cfb4f..1d62fc47 100644 --- a/src/paimon/core/operation/commit/conflict_detection.cpp +++ b/src/paimon/core/operation/commit/conflict_detection.cpp @@ -35,6 +35,7 @@ #include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/fields_comparator.h" #include "paimon/common/utils/range_helper.h" +#include "paimon/common/utils/vector_store_utils.h" #include "paimon/core/deletionvectors/deletion_vectors_index_file.h" #include "paimon/core/manifest/file_entry.h" #include "paimon/core/manifest/file_kind.h" @@ -58,12 +59,8 @@ namespace paimon { namespace { -bool IsVectorStoreFile(const std::string& file_name) { - return file_name.find(".vector.") != std::string::npos; -} - bool IsDedicatedStorageFile(const std::string& file_name) { - return BlobUtils::IsBlobFile(file_name) || IsVectorStoreFile(file_name); + return BlobUtils::IsBlobFile(file_name) || VectorStoreUtils::IsVectorStoreFile(file_name); } struct PartitionBucketKey { diff --git a/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp index e0caf2a4..434f7b4c 100644 --- a/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp +++ b/src/paimon/core/operation/commit/manifest_entry_changes_test.cpp @@ -112,18 +112,18 @@ TEST_F(ManifestEntryChangesTest, TestCollectAndSummary) { ASSERT_EQ(1u, changes.compact_changelog.size()); ASSERT_EQ(2u, changes.compact_index_files.size()); - EXPECT_TRUE(changes.HasAppendChanges()); - EXPECT_FALSE(changes.HasGlobalIndexFileAdditions()); - EXPECT_TRUE(changes.HasCompactChanges()); + ASSERT_TRUE(changes.HasAppendChanges()); + ASSERT_FALSE(changes.HasGlobalIndexFileAdditions()); + ASSERT_TRUE(changes.HasCompactChanges()); - EXPECT_EQ(FileKind::Add(), changes.append_table_files[0].Kind()); - EXPECT_EQ(FileKind::Delete(), changes.append_table_files[1].Kind()); - EXPECT_EQ(4, changes.append_table_files[0].TotalBuckets()); + ASSERT_EQ(FileKind::Add(), changes.append_table_files[0].Kind()); + ASSERT_EQ(FileKind::Delete(), changes.append_table_files[1].Kind()); + ASSERT_EQ(4, changes.append_table_files[0].TotalBuckets()); std::string summary = changes.ToString(); - EXPECT_NE(std::string::npos, summary.find("2 append table files")); - EXPECT_NE(std::string::npos, summary.find("1 append Changelogs")); - EXPECT_NE(std::string::npos, summary.find("2 compact index files")); + ASSERT_NE(std::string::npos, summary.find("2 append table files")); + ASSERT_NE(std::string::npos, summary.find("1 append Changelogs")); + ASSERT_NE(std::string::npos, summary.find("2 compact index files")); } TEST_F(ManifestEntryChangesTest, TestHasGlobalIndexFileAdditions) { @@ -144,7 +144,7 @@ TEST_F(ManifestEntryChangesTest, TestHasGlobalIndexFileAdditions) { ManifestEntryChanges changes(/*default_num_bucket=*/8); ASSERT_OK(changes.Collect(message)); - EXPECT_TRUE(changes.HasGlobalIndexFileAdditions()); + ASSERT_TRUE(changes.HasGlobalIndexFileAdditions()); } TEST_F(ManifestEntryChangesTest, TestCollectInvalidCommitMessageType) { @@ -179,10 +179,10 @@ TEST_F(ManifestEntryChangesTest, TestChangedPartitionsIncludesDvAndGlobalIndex) return std::find(changed.begin(), changed.end(), target) != changed.end(); }; - EXPECT_TRUE(contains(partition_data)); - EXPECT_TRUE(contains(partition_dv)); - EXPECT_TRUE(contains(partition_global)); - EXPECT_FALSE(contains(partition_plain_index)); + ASSERT_TRUE(contains(partition_data)); + ASSERT_TRUE(contains(partition_dv)); + ASSERT_TRUE(contains(partition_global)); + ASSERT_FALSE(contains(partition_plain_index)); } } // namespace paimon::test diff --git a/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp index b8e0cee1..e9091b0f 100644 --- a/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp +++ b/src/paimon/core/operation/commit/row_id_column_conflict_checker_test.cpp @@ -77,7 +77,7 @@ TEST_F(RowIdColumnConflictCheckerTest, TestAllowsDisjointWriteColumns) { auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, /*schema_id=*/0, std::vector{"c"}); ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); - EXPECT_FALSE(conflicts); + ASSERT_FALSE(conflicts); } TEST_F(RowIdColumnConflictCheckerTest, TestDetectsSameWriteColumns) { @@ -88,7 +88,7 @@ TEST_F(RowIdColumnConflictCheckerTest, TestDetectsSameWriteColumns) { auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, /*schema_id=*/0, std::vector{"b"}); ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); - EXPECT_TRUE(conflicts); + ASSERT_TRUE(conflicts); } TEST_F(RowIdColumnConflictCheckerTest, TestUsesFieldIdAcrossRename) { @@ -99,7 +99,7 @@ TEST_F(RowIdColumnConflictCheckerTest, TestUsesFieldIdAcrossRename) { auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, /*schema_id=*/0, std::vector{"b"}); ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); - EXPECT_TRUE(conflicts); + ASSERT_TRUE(conflicts); } TEST_F(RowIdColumnConflictCheckerTest, TestTreatsNullWriteColumnsAsFullSchemaWrite) { @@ -110,7 +110,7 @@ TEST_F(RowIdColumnConflictCheckerTest, TestTreatsNullWriteColumnsAsFullSchemaWri auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, /*schema_id=*/0, std::vector{"b"}); ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); - EXPECT_TRUE(conflicts); + ASSERT_TRUE(conflicts); } TEST_F(RowIdColumnConflictCheckerTest, TestMergesOverlappedDeltaRangesAndWriteColumns) { @@ -126,8 +126,8 @@ TEST_F(RowIdColumnConflictCheckerTest, TestMergesOverlappedDeltaRangesAndWriteCo /*schema_id=*/0, std::vector{"c"}); ASSERT_OK_AND_ASSIGN(bool conflicts_b, checker->ConflictsWith(historical_b)); ASSERT_OK_AND_ASSIGN(bool conflicts_c, checker->ConflictsWith(historical_c)); - EXPECT_TRUE(conflicts_b); - EXPECT_TRUE(conflicts_c); + ASSERT_TRUE(conflicts_b); + ASSERT_TRUE(conflicts_c); } TEST_F(RowIdColumnConflictCheckerTest, TestScansAllOverlappedRangesAfterBinarySearch) { @@ -140,7 +140,7 @@ TEST_F(RowIdColumnConflictCheckerTest, TestScansAllOverlappedRangesAfterBinarySe auto historical = CreateFile("historical", /*first_row_id=*/3, /*row_count=*/10, /*schema_id=*/0, std::vector{"c"}); ASSERT_OK_AND_ASSIGN(bool conflicts, checker->ConflictsWith(historical)); - EXPECT_TRUE(conflicts); + ASSERT_TRUE(conflicts); } TEST_F(RowIdColumnConflictCheckerTest, TestIgnoreUnknownNonSystemWriteColumn) { @@ -151,7 +151,7 @@ TEST_F(RowIdColumnConflictCheckerTest, TestIgnoreUnknownNonSystemWriteColumn) { auto historical = CreateFile("historical", /*first_row_id=*/0, /*row_count=*/10, /*schema_id=*/0, std::vector{"missing"}); auto conflicts = checker->ConflictsWith(historical); - EXPECT_FALSE(conflicts.ok()); + ASSERT_FALSE(conflicts.ok()); } } // namespace paimon::test diff --git a/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp b/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp index 8fc61394..1a337705 100644 --- a/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp +++ b/src/paimon/core/operation/commit/row_tracking_commit_utils.cpp @@ -26,6 +26,7 @@ #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/vector_store_utils.h" #include "paimon/core/manifest/file_source.h" #include "paimon/status.h" @@ -33,10 +34,6 @@ namespace paimon { namespace { -bool IsVectorStoreFile(const std::string& file_name) { - return file_name.find(".vector.") != std::string::npos; -} - ManifestEntry CloneEntryWithClonedFileMeta(const ManifestEntry& entry) { auto cloned_file = std::make_shared(*entry.File()); return ManifestEntry(entry.Kind(), entry.Partition(), entry.Bucket(), entry.TotalBuckets(), @@ -127,7 +124,7 @@ Result RowTrackingCommitUtils::AssignRowTrackingMeta( } assigned_entry.AssignFirstRowId(blob_start); blob_starts[blob_field_name] = blob_start + row_count; - } else if (IsVectorStoreFile(entry.File()->file_name)) { + } else if (VectorStoreUtils::IsVectorStoreFile(entry.File()->file_name)) { if (vector_store_start >= start) { return Status::Invalid(fmt::format( "This is a bug, vectorStoreStart {} should be less than start {} " diff --git a/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp new file mode 100644 index 00000000..af572b72 --- /dev/null +++ b/src/paimon/core/operation/commit/sequence_snapshot_properties_test.cpp @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/operation/commit/sequence_snapshot_properties.h" + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/stats/simple_stats.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class SequenceSnapshotPropertiesTest : public testing::Test { + protected: + Snapshot MakeSnapshot( + const std::optional>& properties) const { + return Snapshot( + /*id=*/1, + /*schema_id=*/1, + /*base_manifest_list=*/"base-manifest-list", + /*base_manifest_list_size=*/std::nullopt, + /*delta_manifest_list=*/"delta-manifest-list", + /*delta_manifest_list_size=*/std::nullopt, + /*changelog_manifest_list=*/std::nullopt, + /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, + /*commit_user=*/"test-user", + /*commit_identifier=*/1, Snapshot::CommitKind::Append(), + /*time_millis=*/0, + /*total_record_count=*/0, + /*delta_record_count=*/0, + /*changelog_record_count=*/std::nullopt, + /*watermark=*/std::nullopt, + /*statistics=*/std::nullopt, properties, + /*next_row_id=*/std::nullopt); + } + + std::shared_ptr CreateDataFileMeta(int64_t max_sequence_number) const { + return std::make_shared( + "data-file", 1024, 8, DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), + SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_seq_no=*/0, + /*max_seq_no=*/max_sequence_number, + /*schema_id=*/1, /*level=*/0, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, /*file_source=*/std::nullopt, + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + } + + ManifestEntry CreateEntry(const FileKind& kind, int64_t max_sequence_number) const { + return ManifestEntry(kind, BinaryRow(0), /*bucket=*/0, /*total_buckets=*/1, + CreateDataFileMeta(max_sequence_number)); + } +}; + +TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberEmptySnapshot) { + ASSERT_OK_AND_ASSIGN(std::optional result, + SequenceSnapshotProperties::MaxSequenceNumber(std::nullopt)); + ASSERT_FALSE(result.has_value()); +} + +TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberSnapshotWithoutProperties) { + ASSERT_OK_AND_ASSIGN(std::optional result, + SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(std::nullopt))); + ASSERT_FALSE(result.has_value()); +} + +TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberKeyMissing) { + std::map properties{{"other-key", "42"}}; + ASSERT_OK_AND_ASSIGN(std::optional result, + SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(properties))); + ASSERT_FALSE(result.has_value()); +} + +TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberValid) { + std::map properties{ + {SequenceSnapshotProperties::kMaxSequenceNumberKey, "123"}}; + ASSERT_OK_AND_ASSIGN(std::optional result, + SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(properties))); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(123, result.value()); +} + +TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberTrailingCharacters) { + std::map properties{ + {SequenceSnapshotProperties::kMaxSequenceNumberKey, "123abc"}}; + ASSERT_NOK_WITH_MSG(SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(properties)), + "trailing characters are not allowed"); +} + +TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberNotANumber) { + std::map properties{ + {SequenceSnapshotProperties::kMaxSequenceNumberKey, "not-a-number"}}; + ASSERT_NOK_WITH_MSG(SequenceSnapshotProperties::MaxSequenceNumber(MakeSnapshot(properties)), + "Invalid"); +} + +TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberFromFilesEmpty) { + ASSERT_FALSE(SequenceSnapshotProperties::MaxSequenceNumberFromFiles({}).has_value()); +} + +TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberFromFilesOnlyDelete) { + std::vector files{CreateEntry(FileKind::Delete(), 100)}; + ASSERT_FALSE(SequenceSnapshotProperties::MaxSequenceNumberFromFiles(files).has_value()); +} + +TEST_F(SequenceSnapshotPropertiesTest, MaxSequenceNumberFromFilesSkipsDelete) { + std::vector files{CreateEntry(FileKind::Add(), 10), + CreateEntry(FileKind::Delete(), 999), + CreateEntry(FileKind::Add(), 42)}; + std::optional result = SequenceSnapshotProperties::MaxSequenceNumberFromFiles(files); + ASSERT_TRUE(result.has_value()); + ASSERT_EQ(42, result.value()); +} + +TEST_F(SequenceSnapshotPropertiesTest, MergeMaxSequenceNumberNoInput) { + std::map properties{{"existing", "value"}}; + std::map merged = SequenceSnapshotProperties::MergeMaxSequenceNumber( + properties, /*latest_max_sequence_number=*/std::nullopt, /*delta_files=*/{}); + ASSERT_EQ(properties, merged); + ASSERT_EQ(0u, merged.count(SequenceSnapshotProperties::kMaxSequenceNumberKey)); +} + +TEST_F(SequenceSnapshotPropertiesTest, MergeMaxSequenceNumberLatestOnly) { + std::map merged = SequenceSnapshotProperties::MergeMaxSequenceNumber( + /*properties=*/{}, /*latest_max_sequence_number=*/50, /*delta_files=*/{}); + ASSERT_EQ("50", merged.at(SequenceSnapshotProperties::kMaxSequenceNumberKey)); +} + +TEST_F(SequenceSnapshotPropertiesTest, MergeMaxSequenceNumberDeltaOnly) { + std::vector delta_files{CreateEntry(FileKind::Add(), 77)}; + std::map merged = SequenceSnapshotProperties::MergeMaxSequenceNumber( + /*properties=*/{}, /*latest_max_sequence_number=*/std::nullopt, delta_files); + ASSERT_EQ("77", merged.at(SequenceSnapshotProperties::kMaxSequenceNumberKey)); +} + +TEST_F(SequenceSnapshotPropertiesTest, MergeMaxSequenceNumberTakesMaximum) { + std::vector delta_files{CreateEntry(FileKind::Add(), 30)}; + std::map merged = SequenceSnapshotProperties::MergeMaxSequenceNumber( + /*properties=*/{}, /*latest_max_sequence_number=*/90, delta_files); + ASSERT_EQ("90", merged.at(SequenceSnapshotProperties::kMaxSequenceNumberKey)); + + std::vector larger_delta{CreateEntry(FileKind::Add(), 150)}; + std::map merged2 = SequenceSnapshotProperties::MergeMaxSequenceNumber( + /*properties=*/{}, /*latest_max_sequence_number=*/90, larger_delta); + ASSERT_EQ("150", merged2.at(SequenceSnapshotProperties::kMaxSequenceNumberKey)); +} + +} // namespace paimon::test diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index ea5c5e38..0131e1a9 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp @@ -205,6 +205,172 @@ Status FileStoreCommitImpl::DropPartition( return Status::OK(); } +Status FileStoreCommitImpl::TruncateTable(int64_t commit_identifier) { + // An empty partition list means "all partitions", so this overwrites the whole table with + // no new files, effectively truncating it. Mirrors Java tryOverwritePartition(null, ...). + PAIMON_ASSIGN_OR_RAISE([[maybe_unused]] int32_t attempt, + TryOverwrite(/*partition=*/{}, /*changes=*/{}, /*index_entries=*/{}, + commit_identifier, std::nullopt, /*properties=*/{})); + return Status::OK(); +} + +Status FileStoreCommitImpl::Abort( + const std::vector>& commit_messages) { + for (const auto& message : commit_messages) { + auto* msg = dynamic_cast(message.get()); + if (msg == nullptr) { + return Status::Invalid("fail to cast commit message to impl"); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr data_file_path_factory, + path_factory_->CreateDataFilePathFactory(msg->Partition(), msg->Bucket())); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_file_path_factory, + path_factory_->CreateIndexFileFactory(msg->Partition(), msg->Bucket())); + + const DataIncrement& new_files_increment = msg->GetNewFilesIncrement(); + const CompactIncrement& compact_increment = msg->GetCompactIncrement(); + + std::vector> data_files_to_delete; + auto append_data_files = + [&data_files_to_delete](const std::vector>& files) { + data_files_to_delete.insert(data_files_to_delete.end(), files.begin(), files.end()); + }; + append_data_files(new_files_increment.NewFiles()); + append_data_files(new_files_increment.ChangelogFiles()); + append_data_files(compact_increment.CompactAfter()); + append_data_files(compact_increment.ChangelogFiles()); + for (const auto& file : data_files_to_delete) { + // Best-effort cleanup: ignore delete failures, aligning with Java deleteQuietly. + [[maybe_unused]] Status status = + fs_->Delete(data_file_path_factory->ToPath(file), /*recursive=*/false); + } + + std::vector> index_files_to_delete; + auto append_index_files = [&index_files_to_delete]( + const std::vector>& files) { + index_files_to_delete.insert(index_files_to_delete.end(), files.begin(), files.end()); + }; + append_index_files(new_files_increment.NewIndexFiles()); + append_index_files(compact_increment.NewIndexFiles()); + for (const auto& file : index_files_to_delete) { + [[maybe_unused]] Status status = + fs_->Delete(index_file_path_factory->ToPath(file), /*recursive=*/false); + } + } + return Status::OK(); +} + +Result> FileStoreCommitImpl::ReadAddManifestEntries( + const Snapshot& snapshot) const { + std::vector data_manifests; + PAIMON_RETURN_NOT_OK(manifest_list_->ReadDataManifests(snapshot, &data_manifests)); + std::vector unmerged_entries; + for (const auto& meta : data_manifests) { + std::vector entries; + PAIMON_RETURN_NOT_OK(manifest_file_->Read( + meta.FileName(), [](const ManifestEntry&) -> Result { return true; }, &entries)); + unmerged_entries.insert(unmerged_entries.end(), std::make_move_iterator(entries.begin()), + std::make_move_iterator(entries.end())); + } + std::vector merged_entries; + PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(unmerged_entries, &merged_entries)); + std::vector add_entries; + for (auto& entry : merged_entries) { + if (entry.Kind() == FileKind::Add()) { + add_entries.push_back(std::move(entry)); + } + } + return add_entries; +} + +Result FileStoreCommitImpl::RollbackToAsLatest(int64_t target_snapshot_id) { + PAIMON_ASSIGN_OR_RAISE(std::optional latest_opt, snapshot_manager_->LatestSnapshot()); + if (!latest_opt) { + return Status::Invalid("Latest snapshot is null, can not roll back."); + } + const Snapshot& latest = latest_opt.value(); + PAIMON_ASSIGN_OR_RAISE(Snapshot target_snapshot, + snapshot_manager_->LoadSnapshot(target_snapshot_id)); + + PAIMON_ASSIGN_OR_RAISE(std::vector latest_entries, + ReadAddManifestEntries(latest)); + PAIMON_ASSIGN_OR_RAISE(std::vector target_entries, + ReadAddManifestEntries(target_snapshot)); + + std::unordered_set latest_identifiers; + latest_identifiers.reserve(latest_entries.size()); + for (const auto& entry : latest_entries) { + latest_identifiers.insert(entry.CreateIdentifier()); + } + std::unordered_set target_identifiers; + target_identifiers.reserve(target_entries.size()); + for (const auto& entry : target_entries) { + target_identifiers.insert(entry.CreateIdentifier()); + } + + std::vector delta_files; + for (const auto& entry : latest_entries) { + if (target_identifiers.find(entry.CreateIdentifier()) == target_identifiers.end()) { + delta_files.emplace_back(FileKind::Delete(), entry.Partition(), entry.Bucket(), + entry.TotalBuckets(), entry.File()); + } + } + for (const auto& entry : target_entries) { + if (latest_identifiers.find(entry.CreateIdentifier()) == latest_identifiers.end()) { + delta_files.emplace_back(FileKind::Add(), entry.Partition(), entry.Bucket(), + entry.TotalBuckets(), entry.File()); + } + } + + std::pair base_manifest_list; + std::pair delta_manifest_list; + PAIMON_ASSIGN_OR_RAISE(std::vector base_manifests, + manifest_file_->Write(latest_entries)); + PAIMON_ASSIGN_OR_RAISE(base_manifest_list, manifest_list_->Write(base_manifests)); + PAIMON_ASSIGN_OR_RAISE(std::vector delta_manifests, + manifest_file_->Write(delta_files)); + PAIMON_ASSIGN_OR_RAISE(delta_manifest_list, manifest_list_->Write(delta_manifests)); + + // For row-tracking tables nextRowId must stay monotonic: a rollback to an older snapshot must + // not move it backwards, otherwise new appends would reuse row ids already assigned by the + // snapshots between the target and the previous latest, breaking the global uniqueness of + // _ROW_ID. Keep the larger of the previous latest and the target nextRowId. + std::optional next_row_id = std::max(latest.NextRowId(), target_snapshot.NextRowId()); + + int64_t delta_record_count = + ManifestEntry::RecordCountAdd(delta_files) - ManifestEntry::RecordCountDelete(delta_files); + Snapshot new_snapshot( + latest.Id() + 1, target_snapshot.SchemaId(), base_manifest_list.first, + base_manifest_list.second, delta_manifest_list.first, delta_manifest_list.second, + /*changelog_manifest_list=*/std::nullopt, /*changelog_manifest_list_size=*/std::nullopt, + target_snapshot.IndexManifest(), commit_user_, + /*commit_identifier=*/std::numeric_limits::max(), + Snapshot::CommitKind::Overwrite(), DateTimeUtils::GetCurrentUTCTimeUs() / 1000, + target_snapshot.TotalRecordCount(), delta_record_count, + /*changelog_record_count=*/std::nullopt, target_snapshot.Watermark(), + target_snapshot.Statistics(), target_snapshot.Properties(), next_row_id); + + std::unordered_map partition_entry_map; + PAIMON_RETURN_NOT_OK(PartitionEntry::Merge(delta_files, &partition_entry_map)); + std::vector delta_statistics; + delta_statistics.reserve(partition_entry_map.size()); + for (const auto& [_, partition_entry] : partition_entry_map) { + delta_statistics.push_back(partition_entry); + } + + PAIMON_ASSIGN_OR_RAISE(bool success, CommitSnapshotImpl(new_snapshot, delta_statistics)); + if (success) { + PAIMON_LOG_INFO(logger_, + "Successfully rolled back table %s to snapshot %ld as new snapshot %ld by " + "user %s.", + root_path_.c_str(), target_snapshot_id, new_snapshot.Id(), + commit_user_.c_str()); + last_committed_snapshot_id_ = new_snapshot.Id(); + } + return success; +} + FileStoreCommit& FileStoreCommitImpl::RowIdCheckConflict( std::optional row_id_check_from_snapshot) { conflict_detection_.SetRowIdCheckFromSnapshot(row_id_check_from_snapshot); diff --git a/src/paimon/core/operation/file_store_commit_impl.h b/src/paimon/core/operation/file_store_commit_impl.h index f1b40f80..ac1dc873 100644 --- a/src/paimon/core/operation/file_store_commit_impl.h +++ b/src/paimon/core/operation/file_store_commit_impl.h @@ -128,6 +128,12 @@ class FileStoreCommitImpl : public FileStoreCommit { Status DropPartition(const std::vector>& partitions, int64_t commit_identifier) override; + Status TruncateTable(int64_t commit_identifier) override; + + Status Abort(const std::vector>& commit_messages) override; + + Result RollbackToAsLatest(int64_t target_snapshot_id) override; + FileStoreCommit& RowIdCheckConflict(std::optional row_id_check_from_snapshot) override; std::shared_ptr GetCommitMetrics() const override { @@ -199,6 +205,8 @@ class FileStoreCommitImpl : public FileStoreCommit { Result CommitSnapshotImpl(const Snapshot& new_snapshot, const std::vector& delta_statistics); + Result> ReadAddManifestEntries(const Snapshot& snapshot) const; + void CleanUpTmpManifests(const std::string& previous_changes_list_name, const std::string& new_changes_list_name, const std::vector& old_metas, diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index f1f4ad8b..6bbe7c54 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -39,11 +39,20 @@ #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" #include "paimon/common/factories/io_hook.h" +#include "paimon/common/utils/linked_hash_map.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/catalog/commit_table_request.h" +#include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/deletion_vectors_index_file.h" +#include "paimon/core/index/deletion_vector_meta.h" #include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/index_path_factory.h" +#include "paimon/core/io/compact_increment.h" #include "paimon/core/io/data_file_meta.h" +#include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/data_increment.h" #include "paimon/core/manifest/file_kind.h" #include "paimon/core/manifest/file_source.h" #include "paimon/core/manifest/index_manifest_entry.h" @@ -57,6 +66,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/table/sink/commit_message_impl.h" +#include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/file_utils.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/timestamp.h" @@ -84,6 +94,7 @@ class GmockFileSystem : public LocalFileSystem { (const, override)); MOCK_METHOD(Status, AtomicStore, (const std::string& path, const std::string& content), (override)); + MOCK_METHOD(Result, Exists, (const std::string& path), (const, override)); }; class GmockFileSystemFactory : public LocalFileSystemFactory { @@ -117,6 +128,11 @@ class GmockFileSystemFactory : public LocalFileSystemFactory { return fs_ptr->FileSystem::AtomicStore(path, content); })); + ON_CALL(*fs, Exists(A())) + .WillByDefault(Invoke([fs_ptr](const std::string& path) { + return fs_ptr->LocalFileSystem::Exists(path); + })); + return fs; } }; @@ -221,6 +237,17 @@ class FileStoreCommitImplTest : public testing::Test { return result; } + size_t CountFiles(const std::string& dir) const { + size_t count = 0; + std::error_code ec; + for (std::filesystem::directory_iterator it(dir, ec), end; it != end; it.increment(ec)) { + if (it->is_regular_file(ec)) { + ++count; + } + } + return count; + } + std::shared_ptr CreateIndexFileMeta(const std::string& file_name, const std::string& index_type = "bitmap") { return std::make_shared(index_type, file_name, /*file_size=*/100, @@ -240,6 +267,22 @@ class FileStoreCommitImplTest : public testing::Test { /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, global_index); } + std::shared_ptr CreateLeveledDataFileMeta(const std::string& file_name, + const BinaryRow& min_key, + const BinaryRow& max_key, int32_t level, + int32_t schema_id = 0) { + return std::make_shared( + file_name, 1024, 8, min_key, max_key, SimpleStats::EmptyStats(), + SimpleStats::EmptyStats(), /*min_seq_no=*/16, /*max_seq_no=*/32, schema_id, level, + /*extra_files=*/std::vector>(), + /*creation_time=*/Timestamp(0, 0), + /*delete_row_count=*/std::nullopt, + /*embedded_index=*/nullptr, FileSource::Append(), + /*external_path=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*write_cols=*/std::nullopt); + } + std::shared_ptr CreateAppendDataFileMeta(const std::string& file_name, int64_t row_count) { return std::make_shared( @@ -765,6 +808,224 @@ TEST_F(FileStoreCommitImplTest, TestCommitMultipleTimes) { } } +TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatest) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = dynamic_cast(commit.get()); + ASSERT_TRUE(commit_impl); + + // Append three times so the latest snapshot (3) is a superset of snapshot 1. + const std::string base_path = paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-0"; + for (int32_t i = 1; i <= 3; ++i) { + std::vector> msgs = + GetCommitMessages(base_path + std::to_string(i), /*version=*/3); + ASSERT_GT(msgs.size(), 0); + ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/i)); + } + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot1, commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot3, commit_impl->snapshot_manager_->LoadSnapshot(3)); + ASSERT_OK_AND_ASSIGN(std::vector snapshot1_entries, + commit_impl->ReadAddManifestEntries(snapshot1)); + ASSERT_OK_AND_ASSIGN(std::vector snapshot3_entries, + commit_impl->ReadAddManifestEntries(snapshot3)); + + // Roll back to snapshot 1: the delta only removes files (DELETE branch). + ASSERT_OK_AND_ASSIGN(bool rolled_back, commit->RollbackToAsLatest(/*target_snapshot_id=*/1)); + ASSERT_TRUE(rolled_back); + ASSERT_OK_AND_ASSIGN(bool snapshot4_exist, file_system_->Exists(PathUtil::JoinPath( + table_path_, "snapshot/snapshot-4"))); + ASSERT_TRUE(snapshot4_exist); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot4, commit_impl->snapshot_manager_->LoadSnapshot(4)); + ASSERT_EQ(snapshot4.Id(), 4); + ASSERT_TRUE(snapshot4.GetCommitKind() == Snapshot::CommitKind::Overwrite()); + ASSERT_EQ(snapshot4.SchemaId(), snapshot1.SchemaId()); + ASSERT_EQ(snapshot4.TotalRecordCount(), snapshot1.TotalRecordCount()); + ASSERT_EQ(snapshot4.NextRowId(), std::max(snapshot3.NextRowId(), snapshot1.NextRowId())); + ASSERT_EQ(snapshot4.IndexManifest(), snapshot1.IndexManifest()); + ASSERT_OK_AND_ASSIGN(std::vector snapshot4_entries, + commit_impl->ReadAddManifestEntries(snapshot4)); + ASSERT_EQ(CollectFileNames(snapshot4_entries), CollectFileNames(snapshot1_entries)); + + // Roll back forward to snapshot 3: the delta only adds files (ADD branch). + ASSERT_OK_AND_ASSIGN(bool rolled_forward, commit->RollbackToAsLatest(/*target_snapshot_id=*/3)); + ASSERT_TRUE(rolled_forward); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot5, commit_impl->snapshot_manager_->LoadSnapshot(5)); + ASSERT_EQ(snapshot5.Id(), 5); + ASSERT_TRUE(snapshot5.GetCommitKind() == Snapshot::CommitKind::Overwrite()); + ASSERT_EQ(snapshot5.TotalRecordCount(), snapshot3.TotalRecordCount()); + ASSERT_EQ(snapshot5.IndexManifest(), snapshot3.IndexManifest()); + ASSERT_OK_AND_ASSIGN(std::vector snapshot5_entries, + commit_impl->ReadAddManifestEntries(snapshot5)); + ASSERT_EQ(CollectFileNames(snapshot5_entries), CollectFileNames(snapshot3_entries)); +} + +TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestNoLatestSnapshotReturnsError) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_NOK_WITH_MSG(commit->RollbackToAsLatest(/*target_snapshot_id=*/1), + "Latest snapshot is null, can not roll back."); +} + +TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestTargetNotExistReturnsError) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + std::vector> msgs = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + ASSERT_GT(msgs.size(), 0); + ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/1)); + ASSERT_FALSE(commit->RollbackToAsLatest(/*target_snapshot_id=*/999).ok()); +} + +TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestDeletionVectorOnlyChange) { + // Mirror Java testRollbackToAsLatestDeletionVectorChangeIsInvisibleToStreaming: when the target + // and the latest snapshot share identical data files and differ only in the index (deletion + // vector) manifest, the rollback produces an empty data delta and the new snapshot inherits the + // target's index manifest. + std::map table_options = {{Options::FILE_FORMAT, "orc"}, + {Options::FILE_SYSTEM, "local"}, + {Options::BUCKET, "2"}, + {Options::BUCKET_KEY, "f2"}}; + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(test_root_, table_options)); + arrow::Schema typed_schema(fields_); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("foo", "dv_bar"), &schema, + /*partition_keys=*/{"f1"}, + /*primary_keys=*/{}, table_options, + /*ignore_if_exists=*/false)); + std::string dv_table_path = PathUtil::JoinPath(test_root_, "foo.db/dv_bar"); + + CommitContextBuilder context_builder(dv_table_path, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = dynamic_cast(commit.get()); + ASSERT_TRUE(commit_impl); + + BinaryRow partition = BinaryRowGenerator::GenerateRow({10}, GetDefaultPool().get()); + + // snapshot 1 (target): a single data file, no deletion vectors. + std::vector> new_files; + new_files.push_back(CreateAppendDataFileMeta("data-file-1", /*row_count=*/10)); + DataIncrement data_increment(std::move(new_files), {}, {}); + std::shared_ptr data_msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, CompactIncrement({}, {}, {})); + ASSERT_OK(commit->Commit({data_msg}, /*commit_identifier=*/1)); + ASSERT_OK_AND_ASSIGN(Snapshot target_snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1)); + + // snapshot 2 (latest): a deletion-vector-only change on the same data file. The data file is + // unchanged, only the index manifest gains a deletion vector. + LinkedHashMap dv_ranges; + dv_ranges.insert_or_assign("data-file-1", + DeletionVectorMeta(/*data_file_name=*/"data-file-1", /*offset=*/0, + /*length=*/10, /*cardinality=*/1)); + std::vector> new_index_files; + new_index_files.push_back(std::make_shared( + DeletionVectorsIndexFile::DELETION_VECTORS_INDEX, "dv-index-1", /*file_size=*/100, + /*row_count=*/1, /*dv_ranges=*/dv_ranges, /*external_path=*/std::nullopt)); + DataIncrement dv_increment({}, {}, {}, std::move(new_index_files), {}); + std::shared_ptr dv_msg = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, dv_increment, CompactIncrement({}, {}, {})); + ASSERT_OK(commit->Commit({dv_msg}, /*commit_identifier=*/2)); + ASSERT_OK_AND_ASSIGN(Snapshot latest_snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_TRUE(latest_snapshot.IndexManifest().has_value()); + + // Roll back to snapshot 1: the data files are identical, so the delta carries no data change + // and the new snapshot inherits the target's (empty) index manifest, dropping the deletion + // vector. + ASSERT_OK_AND_ASSIGN(bool rolled_back, commit->RollbackToAsLatest(/*target_snapshot_id=*/1)); + ASSERT_TRUE(rolled_back); + ASSERT_OK_AND_ASSIGN(Snapshot rolled_back_snapshot, + commit_impl->snapshot_manager_->LoadSnapshot(3)); + ASSERT_EQ(rolled_back_snapshot.Id(), 3); + ASSERT_TRUE(rolled_back_snapshot.GetCommitKind() == Snapshot::CommitKind::Overwrite()); + // Empty data delta: no records are added or removed by the rollback. + ASSERT_EQ(rolled_back_snapshot.DeltaRecordCount(), 0); + // The new snapshot points back to the target's index manifest. + ASSERT_EQ(rolled_back_snapshot.IndexManifest(), target_snapshot.IndexManifest()); + // The surviving data files are unchanged relative to the target. + ASSERT_OK_AND_ASSIGN(std::vector target_entries, + commit_impl->ReadAddManifestEntries(target_snapshot)); + ASSERT_OK_AND_ASSIGN(std::vector rolled_back_entries, + commit_impl->ReadAddManifestEntries(rolled_back_snapshot)); + ASSERT_EQ(CollectFileNames(rolled_back_entries), CollectFileNames(target_entries)); +} + +TEST_F(FileStoreCommitImplTest, TestRollbackToAsLatestConcurrentConflictReturnsFalse) { + // When a concurrent writer wins the race for the next snapshot id, the atomic snapshot commit + // reports failure (returns false); RollbackToAsLatest surfaces that without error and without + // advancing the committed snapshot id. The temporary base/delta manifest lists written before + // the failed commit are intentionally left behind, matching Java (no cleanup on this path). + // + // The conflict is a TOCTOU race: snapshot-2 does not exist when LatestSnapshot() resolves the + // current latest (so it stays snapshot-1), but exists by the time RenamingSnapshotCommit checks + // before writing. A stateful Exists() mock reproduces exactly that ordering. + ASSERT_OK_AND_ASSIGN(std::shared_ptr fs, + FileSystemFactory::Get("gmock_fs", table_path_, {})); + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .WithFileSystem(fs) + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = dynamic_cast(commit.get()); + ASSERT_TRUE(commit_impl); + + std::vector> msgs = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + ASSERT_GT(msgs.size(), 0); + ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/1)); + ASSERT_EQ(commit_impl->last_committed_snapshot_id_, 1); + + // snapshot-2 is invisible the first time it is probed (LatestSnapshot keeps snapshot-1 as the + // latest) and visible afterwards (RenamingSnapshotCommit sees the conflicting file). + const std::string next_snapshot = PathUtil::JoinPath(table_path_, "snapshot/snapshot-2"); + auto* mock_fs = dynamic_cast(fs.get()); + ASSERT_TRUE(mock_fs); + EXPECT_CALL(*mock_fs, Exists(testing::_)) + .Times(testing::AnyNumber()) + .WillRepeatedly(testing::Invoke( + [mock_fs](const std::string& path) { return mock_fs->LocalFileSystem::Exists(path); })); + EXPECT_CALL(*mock_fs, Exists(testing::StrEq(next_snapshot))) + .WillOnce(testing::Return(Result(false))) + .WillRepeatedly(testing::Return(Result(true))); + + const std::string manifest_dir = PathUtil::JoinPath(table_path_, "manifest"); + const size_t manifest_count_before = CountFiles(manifest_dir); + + ASSERT_OK_AND_ASSIGN(bool rolled_back, commit->RollbackToAsLatest(/*target_snapshot_id=*/1)); + ASSERT_FALSE(rolled_back); + // The committed snapshot id must not advance on the failed rollback. + ASSERT_EQ(commit_impl->last_committed_snapshot_id_, 1); + // The temporary base/delta manifest lists are leaked (not cleaned up) on the failure path. + ASSERT_GT(CountFiles(manifest_dir), manifest_count_before); +} + TEST_F(FileStoreCommitImplTest, TestCommitAndOverwriteWithNoPartitionKey) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -1248,6 +1509,182 @@ TEST_F(FileStoreCommitImplTest, TestDropMultiPartitionAndExpireSnapshot) { ASSERT_EQ(3, manifests[0].NumDeletedFiles()); } +TEST_F(FileStoreCommitImplTest, TestTruncateTable) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .IgnoreEmptyCommit(true) + .Finish()); + + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + std::vector> msgs = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/" + "commit_messages-01", + /*version=*/3); + ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/0)); + + // Truncate overwrites all partitions with no new files, deleting every existing file. + ASSERT_OK(commit->TruncateTable(/*commit_identifier=*/1)); + + ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(table_path_ + "/snapshot/snapshot-2")); + ASSERT_TRUE(exist); + auto commit_impl = dynamic_cast(commit.get()); + ASSERT_TRUE(commit_impl); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot.GetCommitKind()); + std::vector manifests; + ASSERT_OK(commit_impl->manifest_list_->ReadDeltaManifests(snapshot, &manifests)); + ASSERT_EQ(1, manifests.size()); + ASSERT_EQ(0, manifests[0].NumAddedFiles()); + // The append_09 fixture contains 3 data files spread across partitions f1=10 and f1=20. + ASSERT_EQ(3, manifests[0].NumDeletedFiles()); +} + +TEST_F(FileStoreCommitImplTest, TestAbortDeletesDataAndIndexFiles) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + const BinaryRow partition = CreateIntRow(10); + const int32_t bucket = 0; + auto new_data_file = CreateAppendDataFileMeta("abort-new-data", 1); + auto compact_data_file = CreateAppendDataFileMeta("abort-compact-data", 1); + auto new_index_file = CreateIndexFileMeta("abort-new-index"); + auto compact_index_file = CreateIndexFileMeta("abort-compact-index"); + + DataIncrement data_increment(/*new_files=*/{new_data_file}, /*deleted_files=*/{}, + /*changelog_files=*/{}, /*new_index_files=*/{new_index_file}, + /*deleted_index_files=*/{}); + CompactIncrement compact_increment(/*compact_before=*/{}, /*compact_after=*/{compact_data_file}, + /*changelog_files=*/{}, + /*new_index_files=*/{compact_index_file}, + /*deleted_index_files=*/{}); + std::shared_ptr message = std::make_shared( + partition, bucket, /*total_buckets=*/2, data_increment, compact_increment); + + // Materialize the referenced files so we can observe them being cleaned up. + ASSERT_OK_AND_ASSIGN(std::shared_ptr data_pf, + commit_impl->path_factory_->CreateDataFilePathFactory(partition, bucket)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr index_pf, + commit_impl->path_factory_->CreateIndexFileFactory(partition, bucket)); + std::vector paths = { + data_pf->ToPath(new_data_file), data_pf->ToPath(compact_data_file), + index_pf->ToPath(new_index_file), index_pf->ToPath(compact_index_file)}; + for (const auto& path : paths) { + ASSERT_OK(file_system_->WriteFile(path, /*content=*/"", /*overwrite=*/false)); + ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(path)); + ASSERT_TRUE(exist); + } + + ASSERT_OK(commit_impl->Abort({message})); + + for (const auto& path : paths) { + ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(path)); + ASSERT_FALSE(exist); + } +} + +TEST_F(FileStoreCommitImplTest, AbortIgnoresMissingFilesAndFailsForNonImplMessage) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + // Deleting files that were never written is a best-effort no-op, not an error. + DataIncrement data_increment(/*new_files=*/{CreateAppendDataFileMeta("abort-missing", 1)}, + /*deleted_files=*/{}, /*changelog_files=*/{}); + std::shared_ptr message = + std::make_shared(CreateIntRow(10), /*bucket=*/0, /*total_buckets=*/2, + data_increment, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Abort({message})); + + // A commit message that is not a CommitMessageImpl is rejected. + ASSERT_NOK_WITH_MSG(commit_impl->Abort({std::make_shared()}), + "fail to cast commit message to impl"); +} + +TEST_F(FileStoreCommitImplTest, AbortIgnoresDeleteFailures) { + // A delete that fails with an IO error (e.g. permission denied) must be swallowed by + // best-effort cleanup, mirroring Java deleteQuietly: Abort still returns OK and does not + // propagate the failure. + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = dynamic_cast(commit.get()); + ASSERT_TRUE(commit_impl); + + const BinaryRow partition = CreateIntRow(10); + const int32_t bucket = 0; + auto new_data_file = CreateAppendDataFileMeta("abort-io-fail-data", 1); + DataIncrement data_increment(/*new_files=*/{new_data_file}, /*deleted_files=*/{}, + /*changelog_files=*/{}); + std::shared_ptr message = std::make_shared( + partition, bucket, /*total_buckets=*/2, data_increment, CompactIncrement({}, {}, {})); + + // Materialize the file so we can observe that a failed delete leaves it untouched. + ASSERT_OK_AND_ASSIGN(std::shared_ptr data_pf, + commit_impl->path_factory_->CreateDataFilePathFactory(partition, bucket)); + const std::string data_path = data_pf->ToPath(new_data_file); + ASSERT_OK(file_system_->WriteFile(data_path, /*content=*/"", /*overwrite=*/false)); + + // Fault-inject an IO error on the delete. Abort performs no local file IO before its delete + // loop, so position 0 lands on the delete itself. The error must be swallowed. Clearing on + // scope exit keeps the process-global hook from leaking into other tests. + auto io_hook = IOHook::GetInstance(); + ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); + io_hook->Reset(/*pos=*/0, IOHook::Mode::RETURN_ERROR); + ASSERT_OK(commit_impl->Abort({message})); + io_hook->Clear(); + + // Because the delete failed, the file remains on disk. + ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(data_path)); + ASSERT_TRUE(exist); +} + +TEST_F(FileStoreCommitImplTest, TestTruncateEmptyTable) { + // Truncating a table that has never been committed succeeds and produces an OVERWRITE snapshot + // with no added and no deleted files: there is nothing to overwrite, but the overwrite is still + // materialized. + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .IgnoreEmptyCommit(true) + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + ASSERT_OK(commit->TruncateTable(/*commit_identifier=*/1)); + + ASSERT_OK_AND_ASSIGN(bool exist, file_system_->Exists(table_path_ + "/snapshot/snapshot-1")); + ASSERT_TRUE(exist); + auto commit_impl = dynamic_cast(commit.get()); + ASSERT_TRUE(commit_impl); + ASSERT_OK_AND_ASSIGN(Snapshot snapshot, commit_impl->snapshot_manager_->LoadSnapshot(1)); + ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot.GetCommitKind()); + std::vector manifests; + ASSERT_OK(commit_impl->manifest_list_->ReadDeltaManifests(snapshot, &manifests)); + for (const auto& manifest : manifests) { + ASSERT_EQ(0, manifest.NumAddedFiles()); + ASSERT_EQ(0, manifest.NumDeletedFiles()); + } +} + TEST_F(FileStoreCommitImplTest, TestCreateManifestCommittable) { CommitContextBuilder context_builder(table_path_, "commit_user_1"); ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, @@ -2100,4 +2537,450 @@ TEST_F(FileStoreCommitImplTest, TestFixedBucketPKTableCommitAllowed) { ASSERT_TRUE(committer != nullptr); } +TEST_F(FileStoreCommitImplTest, ValidateCommitOptionsRejectsUnsupportedOptions) { + const std::vector unsupported_keys = { + "commit.strict-mode.last-safe-snapshot", "manifest.delete-file-drop-stats", + "sequence.snapshot-ordering", "pk-clustering-override"}; + for (const auto& key : unsupported_keys) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({{key, "true"}})); + ASSERT_NOK_WITH_MSG(FileStoreCommitImpl::ValidateCommitOptions(options), + "not supported by C++ commit path"); + } + + // Supported options should validate successfully. + ASSERT_OK_AND_ASSIGN(CoreOptions ok_options, + CoreOptions::FromMap({{Options::FILE_SYSTEM, "local"}})); + ASSERT_OK(FileStoreCommitImpl::ValidateCommitOptions(ok_options)); +} + +TEST_F(FileStoreCommitImplTest, DropPartitionWithEmptyPartitionsFails) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + ASSERT_NOK_WITH_MSG(commit->DropPartition({}, /*commit_identifier=*/1), + "partitions list cannot be empty"); +} + +TEST_F(FileStoreCommitImplTest, FilterAndCommitMultipleIdentifiersAndEmptyInput) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + // Empty input map takes the FilterCommitted fast-exit path (nothing to commit). + ASSERT_OK_AND_ASSIGN(int32_t committed_empty, commit_impl->FilterAndCommit({}, 1)); + ASSERT_EQ(0, committed_empty); + + std::vector data_files = { + "/f1=10/bucket-0/data-51a45441-6037-4af3-b67b-5cefd75dc6f2-0.orc", + "/f1=10/bucket-1/data-6828284c-e707-49b5-af6b-69be79af120c-0.orc", + "/f1=20/bucket-0/data-8dc7f04c-3c98-48b2-9d56-834d746c4a40-0.orc", + "/f1=10/bucket-1/data-fd1d2255-43f2-4534-b4cc-08b29e662940-0.orc", + "/f1=20/bucket-0/data-7b3f4cc7-116b-4d2f-9c62-5dadc1f11bcb-0.orc"}; + ASSERT_OK(PrepareFakeFiles(data_files)); + + std::vector> msgs1 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + std::vector> msgs2 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-02", + /*version=*/3); + + // Two identifiers in a single call exercises the sort-by-identifier comparator. + std::map>> inputs; + inputs[2] = msgs2; + inputs[1] = msgs1; + ASSERT_OK_AND_ASSIGN(int32_t committed, commit_impl->FilterAndCommit(inputs, 5)); + ASSERT_EQ(2, committed); +} + +TEST_F(FileStoreCommitImplTest, CheckFilesExistenceFailsForNonImplCommitMessage) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + auto committable = std::make_shared(/*identifier=*/1); + committable->AddFileCommittable(std::make_shared()); + ASSERT_NOK_WITH_MSG(commit_impl->CheckFilesExistence({committable}), + "fail to cast commit message to impl"); +} + +TEST_F(FileStoreCommitImplTest, CheckFilesExistenceCollectsIndexFilePaths) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + // New index files from both the data increment and the compact increment must be + // collected for the existence check. + DataIncrement data_increment(/*new_files=*/{}, /*deleted_files=*/{}, /*changelog_files=*/{}, + /*new_index_files=*/{CreateIndexFileMeta("new-index-missing")}, + /*deleted_index_files=*/{}); + CompactIncrement compact_increment( + /*compact_before=*/{}, /*compact_after=*/{}, /*changelog_files=*/{}, + /*new_index_files=*/{CreateIndexFileMeta("compact-index-missing")}, + /*deleted_index_files=*/{}); + std::shared_ptr message = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment); + + auto committable = std::make_shared(/*identifier=*/1); + committable->AddFileCommittable(message); + // The referenced index files were never written, so the existence check reports them missing. + ASSERT_NOK_WITH_MSG(commit_impl->CheckFilesExistence({committable}), "have been deleted"); +} + +TEST_F(FileStoreCommitImplTest, OverwriteStaticPartitionValidatesFileOwnership) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::DYNAMIC_PARTITION_OVERWRITE, "false") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + // Files whose partition matches the overwrite target are accepted. + DataIncrement matching_increment({CreateAppendDataFileMeta("static-f1-10", 1)}, {}, {}); + std::shared_ptr matching_msg = + std::make_shared(CreateIntRow(10), /*bucket=*/0, /*total_buckets=*/2, + matching_increment, CompactIncrement({}, {}, {})); + ASSERT_OK(commit_impl->Overwrite({{"f1", "10"}}, {matching_msg}, /*commit_identifier=*/1)); + + // Files belonging to a different partition than the overwrite target are rejected. + DataIncrement mismatching_increment({CreateAppendDataFileMeta("static-f1-20", 1)}, {}, {}); + std::shared_ptr mismatching_msg = + std::make_shared(CreateIntRow(20), /*bucket=*/0, /*total_buckets=*/2, + mismatching_increment, CompactIncrement({}, {}, {})); + ASSERT_NOK_WITH_MSG( + commit_impl->Overwrite({{"f1", "10"}}, {mismatching_msg}, /*commit_identifier=*/2), + "does not belong to this partition"); +} + +TEST_F(FileStoreCommitImplTest, OverwriteWithChangelogFilesLogsWarning) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + const BinaryRow partition = CreateIntRow(10); + + // Overwrite ignores changelog files but emits a warning listing them. + DataIncrement data_increment( + /*new_files=*/{CreateAppendDataFileMeta("overwrite-with-changelog", 1)}, + /*deleted_files=*/{}, + /*changelog_files=*/{CreateAppendDataFileMeta("ignored-append-changelog", 1)}, + /*new_index_files=*/{}, /*deleted_index_files=*/{}); + CompactIncrement compact_increment( + /*compact_before=*/{}, /*compact_after=*/{}, + /*changelog_files=*/{CreateAppendDataFileMeta("ignored-compact-changelog", 1)}); + std::shared_ptr message = std::make_shared( + partition, /*bucket=*/0, /*total_buckets=*/2, data_increment, compact_increment); + + ASSERT_OK(commit_impl->Overwrite({}, {message}, /*commit_identifier=*/1)); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, + commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot.has_value()); +} + +TEST_F(FileStoreCommitImplTest, OverwriteUpgradesNonOverlappingPrimaryKeyFiles) { + auto pk_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(pk_dir); + std::string pk_root = pk_dir->Str(); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(pk_root, {})); + ASSERT_OK(catalog->CreateDatabase("db", {}, false)); + + arrow::Schema pk_schema( + {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())}); + ::ArrowSchema arrow_schema; + ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok()); + std::map table_options = {{Options::BUCKET, "4"}}; + ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl"), &arrow_schema, + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options, + /*ignore_if_exists=*/false)); + std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); + + CommitContextBuilder builder(pk_table_path, "test_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::OVERWRITE_UPGRADE, "true") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + const BinaryRow partition = BinaryRow::EmptyRow(); + // Bucket 0: non-overlapping key ranges -> files are upgraded to a higher level. + DataIncrement bucket0_increment( + {CreateLeveledDataFileMeta("pk-b0-lo", CreateIntRow(0), CreateIntRow(10), /*level=*/0), + CreateLeveledDataFileMeta("pk-b0-hi", CreateIntRow(20), CreateIntRow(30), /*level=*/0)}, + {}, {}); + std::shared_ptr bucket0_msg = + std::make_shared(partition, /*bucket=*/0, /*total_buckets=*/4, + bucket0_increment, CompactIncrement({}, {}, {})); + // Bucket 1: overlapping key ranges -> files are kept at their original level. + DataIncrement bucket1_increment( + {CreateLeveledDataFileMeta("pk-b1-a", CreateIntRow(0), CreateIntRow(20), /*level=*/0), + CreateLeveledDataFileMeta("pk-b1-b", CreateIntRow(10), CreateIntRow(30), /*level=*/0)}, + {}, {}); + std::shared_ptr bucket1_msg = + std::make_shared(partition, /*bucket=*/1, /*total_buckets=*/4, + bucket1_increment, CompactIncrement({}, {}, {})); + + ASSERT_OK(commit_impl->Overwrite({}, {bucket0_msg, bucket1_msg}, /*commit_identifier=*/1)); + + ASSERT_OK_AND_ASSIGN(std::optional snapshot, + commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot.has_value()); + ASSERT_OK_AND_ASSIGN(std::vector entries, + commit_impl->GetAllFiles(snapshot.value(), {})); + ASSERT_EQ(4u, entries.size()); + + int32_t max_level = 0; + int32_t level0_count = 0; + for (const auto& entry : entries) { + max_level = std::max(max_level, entry.Level()); + if (entry.Level() == 0) { + level0_count++; + } + } + // Bucket 0 files were upgraded above level 0, bucket 1 files stayed at level 0. + ASSERT_GT(max_level, 0); + ASSERT_EQ(2, level0_count); +} + +TEST_F(FileStoreCommitImplTest, CommitWithAppendCommitCheckConflict) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .AppendCommitCheckConflict(true) + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + + std::vector> msgs = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + ASSERT_GT(msgs.size(), 0); + ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/1)); + ASSERT_OK_AND_ASSIGN( + bool exist, file_system_->Exists(PathUtil::JoinPath(table_path_, "snapshot/snapshot-1"))); + ASSERT_TRUE(exist); +} + +TEST_F(FileStoreCommitImplTest, SnapshotSequenceMaxFallsBackToManifestScan) { + // First snapshot is committed in the default (scan) mode, so it does NOT carry the + // max-sequence-number property. + { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + std::vector> msgs1 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + ASSERT_OK(commit->Commit(msgs1, /*commit_identifier=*/1)); + } + + // Second commit uses snapshot-init mode; since the previous snapshot lacks the property, + // the max sequence number is recomputed by scanning the base manifests. + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::WRITE_SEQUENCE_NUMBER_INIT_MODE, "snapshot") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot1, commit_impl->snapshot_manager_->LoadSnapshot(1)); + if (snapshot1.Properties()) { + ASSERT_EQ(snapshot1.Properties().value().end(), + snapshot1.Properties().value().find("sequence.generation.max-sequence-number")); + } + + std::vector> msgs2 = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-02", + /*version=*/3); + ASSERT_OK(commit_impl->Commit(msgs2, /*commit_identifier=*/2)); + + ASSERT_OK_AND_ASSIGN(Snapshot snapshot2, commit_impl->snapshot_manager_->LoadSnapshot(2)); + ASSERT_TRUE(snapshot2.Properties()); + auto iter = snapshot2.Properties().value().find("sequence.generation.max-sequence-number"); + ASSERT_TRUE(iter != snapshot2.Properties().value().end()); +} + +TEST_F(FileStoreCommitImplTest, FilterAndOverwriteWithSpecifiedPartition) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + std::vector> msgs = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + // A non-empty partition spec is forwarded to the overwrite so the partition list is populated. + std::map partition_spec = {{"f1", "10"}}; + ASSERT_OK_AND_ASSIGN(int32_t actual_commit, commit_impl->FilterAndOverwrite( + partition_spec, msgs, /*commit_identifier=*/1, + /*watermark=*/10)); + ASSERT_EQ(1, actual_commit); + + ASSERT_OK_AND_ASSIGN(auto snapshot, commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_EQ(Snapshot::CommitKind::Overwrite(), snapshot.value().GetCommitKind()); +} + +TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenOverwriteUpgradeDisabled) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::OVERWRITE_UPGRADE, "false") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + std::vector entries = { + CreateManifestEntry("upgrade-disabled-1", FileKind::Add())}; + // overwrite-upgrade disabled: TryUpgrade returns the input unchanged. + ASSERT_OK_AND_ASSIGN(std::vector result, commit_impl->TryUpgrade(entries)); + ASSERT_EQ(entries.size(), result.size()); +} + +TEST_F(FileStoreCommitImplTest, TryUpgradeReturnsInputWhenEntryLevelAboveZero) { + auto pk_dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(pk_dir); + std::string pk_root = pk_dir->Str(); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(pk_root, {})); + ASSERT_OK(catalog->CreateDatabase("db", {}, false)); + + arrow::Schema pk_schema( + {arrow::field("pk", arrow::int32()), arrow::field("val", arrow::utf8())}); + ::ArrowSchema arrow_schema; + ASSERT_TRUE(arrow::ExportSchema(pk_schema, &arrow_schema).ok()); + std::map table_options = {{Options::BUCKET, "4"}}; + ASSERT_OK(catalog->CreateTable(Identifier("db", "pk_tbl"), &arrow_schema, + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, table_options, + /*ignore_if_exists=*/false)); + std::string pk_table_path = PathUtil::JoinPath(pk_root, "db.db/pk_tbl"); + + CommitContextBuilder builder(pk_table_path, "test_user"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .AddOption(Options::OVERWRITE_UPGRADE, "true") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + // A PK-table entry already at a level above 0 bypasses the upgrade and returns the input. + std::vector entries = { + CreateManifestEntry("already-upgraded", BinaryRow::EmptyRow(), FileKind::Add(), + DataFileMeta::EmptyMinKey(), DataFileMeta::EmptyMaxKey(), /*level=*/2, + /*bucket=*/0)}; + ASSERT_OK_AND_ASSIGN(std::vector result, commit_impl->TryUpgrade(entries)); + ASSERT_EQ(entries.size(), result.size()); + ASSERT_EQ(2, result[0].Level()); +} + +TEST_F(FileStoreCommitImplTest, CheckSameBucketFromSnapshotReturnsOkForEmptyDelta) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::MANIFEST_TARGET_FILE_SIZE, "8mb") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + std::vector> msgs = + GetCommitMessages(paimon::test::GetDataDir() + + "/orc/append_09.db/append_09/commit_messages/commit_messages-01", + /*version=*/3); + ASSERT_OK(commit_impl->Commit(msgs, /*commit_identifier=*/1)); + + ASSERT_OK_AND_ASSIGN(std::optional snapshot, + commit_impl->snapshot_manager_->LatestSnapshot()); + ASSERT_TRUE(snapshot); + // No delta entries -> no buckets to verify -> returns OK without scanning the snapshot. + ASSERT_OK(commit_impl->CheckSameBucketFromSnapshot(/*delta_entries=*/{}, snapshot)); +} + +TEST_F(FileStoreCommitImplTest, MaxSequenceNumberReturnsNulloptForEmptyManifests) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + // No manifests to scan -> no sequence number found -> returns nullopt. + ASSERT_OK_AND_ASSIGN(std::optional max_seq, commit_impl->MaxSequenceNumber({})); + ASSERT_FALSE(max_seq.has_value()); +} + +TEST_F(FileStoreCommitImplTest, RowIdCheckConflictSetsCheckSnapshotAndReturnsSelf) { + CommitContextBuilder context_builder(table_path_, "commit_user_1"); + ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context, + context_builder.AddOption(Options::MANIFEST_FORMAT, "orc") + .AddOption(Options::FILE_SYSTEM, "local") + .Finish()); + ASSERT_OK_AND_ASSIGN(auto commit, FileStoreCommit::Create(std::move(commit_context))); + auto commit_impl = std::dynamic_pointer_cast( + std::shared_ptr(std::move(commit))); + + ASSERT_FALSE(commit_impl->conflict_detection_.HasRowIdCheckFromSnapshot()); + // RowIdCheckConflict records the snapshot to verify against and returns *this for chaining. + FileStoreCommit& returned = commit_impl->RowIdCheckConflict(/*row_id_check_from_snapshot=*/5); + ASSERT_EQ(commit_impl.get(), &returned); + ASSERT_TRUE(commit_impl->conflict_detection_.HasRowIdCheckFromSnapshot()); +} + } // namespace paimon::test From 4a187034238e21e44ba92b498e6e32b59a763779 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:00:05 +0800 Subject: [PATCH 106/138] build: fetch Lumina from release package --- third_party/versions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/versions.txt b/third_party/versions.txt index 6b80b7a8..1266761d 100644 --- a/third_party/versions.txt +++ b/third_party/versions.txt @@ -88,8 +88,8 @@ PAIMON_RAPIDJSON_BUILD_VERSION=232389d4f1012dddec4ef84861face2d2ba85709 PAIMON_RAPIDJSON_BUILD_SHA256_CHECKSUM=b9290a9a6d444c8e049bd589ab804e0ccf2b05dc5984a19ed5ae75d090064806 PAIMON_RAPIDJSON_PKG_NAME=rapidjson-${PAIMON_RAPIDJSON_BUILD_VERSION}.tar.gz -PAIMON_LUMINA_BUILD_VERSION=0.2.3 -PAIMON_LUMINA_BUILD_SHA256_CHECKSUM=49de9548a8b81f6a5e3794e8e65fb32023a1c4bdb019028ce55cbcc98c12be97 +PAIMON_LUMINA_BUILD_VERSION=0.3.0-rc1 +PAIMON_LUMINA_BUILD_SHA256_CHECKSUM=6bdb9eeeeb0c6192e480ea0523712df6f07ea58521ca5ca1abc023673dfa1fa5 PAIMON_LUMINA_PKG_NAME=lumina_release-${PAIMON_LUMINA_BUILD_VERSION}.tar.gz PAIMON_JINDOSDK_C_BUILD_VERSION=6.10.2 From 3da123341a15a894a8e13569635f1eff3a352cff Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Tue, 21 Jul 2026 15:34:29 +0800 Subject: [PATCH 107/138] fix(blob): identify a missing blob file with FileSystem::Exists --- benchmark/CMakeLists.txt | 1 - cmake_modules/ThirdpartyToolchain.cmake | 3 + include/paimon/defs.h | 9 +- src/paimon/CMakeLists.txt | 1 - src/paimon/common/file_index/CMakeLists.txt | 1 - src/paimon/common/global_index/CMakeLists.txt | 1 - src/paimon/format/avro/CMakeLists.txt | 1 - src/paimon/format/blob/CMakeLists.txt | 1 - src/paimon/format/blob/blob_format_writer.cpp | 108 ++++-- src/paimon/format/blob/blob_format_writer.h | 40 +- .../format/blob/blob_format_writer_test.cpp | 353 ++++++++++++++++-- src/paimon/format/orc/CMakeLists.txt | 1 - src/paimon/format/parquet/CMakeLists.txt | 1 - src/paimon/global_index/lucene/CMakeLists.txt | 1 - src/paimon/global_index/lumina/CMakeLists.txt | 1 - test/inte/blob_table_inte_test.cpp | 40 +- 16 files changed, 494 insertions(+), 69 deletions(-) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 9b160fcd..375e9896 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -59,7 +59,6 @@ if(PAIMON_BUILD_BENCHMARKS) ${PAIMON_BENCHMARK_STATIC_LINK_LIBS} test_utils_static Threads::Threads - ${CMAKE_DL_LIBS} ${PAIMON_BENCHMARK_PLATFORM_LINK_LIBS} ${PAIMON_BENCHMARK_LINK_TOOLCHAIN} EXTRA_INCLUDES diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index cb3d825b..255a89ab 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -1678,6 +1678,9 @@ macro(build_arrow) # libarrow.a calls dlsym; keep ${CMAKE_DL_LIBS} in the interface so -ldl is placed # after libarrow.a on linkers that resolve symbols strictly left-to-right. + # Every library that uses dl itself (arrow here; also lucene and jindosdk::nextarch) + # declares it on its own interface the same way; consumers inherit it transitively + # and must not list ${CMAKE_DL_LIBS} again themselves. target_link_libraries(arrow INTERFACE zstd snappy diff --git a/include/paimon/defs.h b/include/paimon/defs.h index b9b73cb4..ddc157ba 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -444,13 +444,14 @@ struct PAIMON_EXPORT Options { /// path and requires manual configuration by the user. No default value. static const char BLOB_VIEW_UPSTREAM_WAREHOUSE[]; /// "blob-write-null-on-missing-file" - Whether to write NULL for a descriptor BLOB value when - /// the referenced file does not exist at write time. When false, the write fails when the - /// descriptor is read. Default value is "false". + /// the referenced file does not exist at write time. When false, a missing file is treated + /// like any other fetch failure, following "blob-write-null-on-fetch-failure". Default value + /// is "false". static const char BLOB_WRITE_NULL_ON_MISSING_FILE[]; /// "blob-write-null-on-fetch-failure" - Whether to write NULL for a descriptor BLOB value when /// the referenced data cannot be fetched at write time (e.g. invalid descriptor or invalid - /// offset). A missing file is handled by "blob-write-null-on-missing-file". When false, the - /// write fails when the descriptor is read. Default value is "false". + /// offset). A missing file is handled by "blob-write-null-on-missing-file" when that option is + /// enabled. When false, the write fails when the descriptor is read. Default value is "false". static const char BLOB_WRITE_NULL_ON_FETCH_FAILURE[]; /// "global-index.enabled" - Whether to enable global index for scan. Default value is "true". static const char GLOBAL_INDEX_ENABLED[]; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 72009c60..3250babf 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -394,7 +394,6 @@ add_paimon_lib(paimon arrow tbb glog - ${CMAKE_DL_LIBS} fmt roaring_bitmap xxhash diff --git a/src/paimon/common/file_index/CMakeLists.txt b/src/paimon/common/file_index/CMakeLists.txt index 0bd1f167..7ab5a069 100644 --- a/src/paimon/common/file_index/CMakeLists.txt +++ b/src/paimon/common/file_index/CMakeLists.txt @@ -44,7 +44,6 @@ add_paimon_lib(paimon_file_index arrow fmt xxhash - ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/common/global_index/CMakeLists.txt b/src/paimon/common/global_index/CMakeLists.txt index c2f9fb51..0bfad32e 100644 --- a/src/paimon/common/global_index/CMakeLists.txt +++ b/src/paimon/common/global_index/CMakeLists.txt @@ -40,7 +40,6 @@ add_paimon_lib(paimon_global_index STATIC_LINK_LIBS arrow fmt - ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/format/avro/CMakeLists.txt b/src/paimon/format/avro/CMakeLists.txt index 00c78477..1c425674 100644 --- a/src/paimon/format/avro/CMakeLists.txt +++ b/src/paimon/format/avro/CMakeLists.txt @@ -40,7 +40,6 @@ if(PAIMON_ENABLE_AVRO) fmt avro tbb - ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/format/blob/CMakeLists.txt b/src/paimon/format/blob/CMakeLists.txt index 440effff..239a936a 100644 --- a/src/paimon/format/blob/CMakeLists.txt +++ b/src/paimon/format/blob/CMakeLists.txt @@ -27,7 +27,6 @@ add_paimon_lib(paimon_blob_file_format STATIC_LINK_LIBS arrow fmt - ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index ff20e1d1..22db5e2f 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -32,6 +32,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/delta_varint_compressor.h" #include "paimon/data/blob.h" +#include "paimon/fs/file_system.h" #include "paimon/io/byte_array_input_stream.h" #include "paimon/logging.h" @@ -50,6 +51,8 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con pool_(pool), write_null_on_missing_file_(write_null_on_missing_file), write_null_on_fetch_failure_(write_null_on_fetch_failure) { + // Create() has already checked that data_type has exactly one BLOB field. + blob_field_name_ = data_type_->field(0)->name(); metrics_ = std::make_shared(); tmp_buffer_ = Bytes::AllocateBytes(kTmpBufferSize, pool_.get()); magic_number_bytes_ = IntegerToLittleEndian(BlobDefs::kMagicNumber, pool_); @@ -69,6 +72,9 @@ Result> BlobFormatWriter::Create( if (pool == nullptr) { return Status::Invalid("blob format writer create failed. pool is nullptr"); } + if (fs == nullptr) { + return Status::Invalid("blob format writer create failed. fs is nullptr"); + } if (data_type->num_fields() != 1) { return Status::Invalid( fmt::format("blob data type field number {} is not 1", data_type->num_fields())); @@ -119,6 +125,10 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { } Status BlobFormatWriter::Flush() { + metrics_->SetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT, + null_on_missing_file_count_); + metrics_->SetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT, + null_on_fetch_failure_count_); return out_->Flush(); } @@ -142,29 +152,20 @@ Status BlobFormatWriter::Finish() { Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { // Open the blob input stream before writing any bytes, so that a failed fetch can be // converted to a NULL element without leaving partial data in the output stream. - // Dynamically check whether blob_data is a serialized BlobDescriptor (by magic header) - // rather than relying on blob_as_descriptor_ config. This is consistent with Java behavior: - // at write time, the input bytes are auto-detected as descriptor or raw data. + // Whether blob_data is a serialized BlobDescriptor is detected by its magic header rather + // than taken from a blob_as_descriptor option, so each row may hold either form. std::unique_ptr in; PAIMON_ASSIGN_OR_RAISE(bool is_descriptor, BlobDescriptor::IsBlobDescriptor(blob_data.data(), blob_data.size())); if (is_descriptor) { - Result> opened = OpenDescriptorInputStream(blob_data); - if (!opened.ok()) { - const Status& status = opened.status(); - // A missing file is only handled by 'blob-write-null-on-missing-file'; other fetch - // failures are only handled by 'blob-write-null-on-fetch-failure' (aligned with Java). - bool write_null = - status.IsNotExist() ? write_null_on_missing_file_ : write_null_on_fetch_failure_; - if (write_null) { - PAIMON_LOG_WARN(logger_, "Failed to open blob, writing NULL for BLOB field: %s", - status.ToString().c_str()); - bin_lengths_.push_back(BlobDefs::kNullBinLength); - return Status::OK(); - } - return status; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr descriptor_in, + OpenDescriptorInputStream(blob_data)); + // A null stream means a write-null option already converted the failure. + if (descriptor_in == nullptr) { + bin_lengths_.push_back(BlobDefs::kNullBinLength); + return Status::OK(); } - in = std::move(opened).value(); + in = std::move(descriptor_in); } else { in = std::make_unique(blob_data.data(), blob_data.size()); } @@ -208,10 +209,73 @@ Status BlobFormatWriter::WriteBlob(std::string_view blob_data) { } Result> BlobFormatWriter::OpenDescriptorInputStream( - std::string_view blob_data) const { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr blob, - Blob::FromDescriptor(blob_data.data(), blob_data.size())); - return blob->NewInputStream(fs_); + std::string_view blob_data) { + // A descriptor that cannot be deserialized is a fetch failure: the referenced data cannot be + // reached. Its URI is inside the unreadable bytes, hence the placeholder; the underlying + // status comes from the byte reader and never mentions blobs, hence the added context. + Result> blob_result = + Blob::FromDescriptor(blob_data.data(), blob_data.size()); + if (!blob_result.ok()) { + const Status& status = blob_result.status(); + return HandleFetchFailure( + "", status.WithMessage("invalid blob descriptor: ", status.message())); + } + std::unique_ptr blob = std::move(blob_result).value(); + + // A missing file is identified by FileSystem::Exists rather than by the status of a failed + // open, since file system implementations disagree on which status a missing file maps to. + // The check runs only when `write_null_on_missing_file_` needs the classification; otherwise + // a missing file gets the same treatment as any other failed open. + if (write_null_on_missing_file_) { + Result exists = fs_->Exists(blob->Uri()); + if (exists.ok()) { + if (!exists.value()) { + return HandleMissingFile(blob->Uri()); + } + } else if (!write_null_on_fetch_failure_) { + // The check cannot answer whether the file is there; with no fetch-failure handling + // to defer to, fail rather than assume either answer. + const Status& status = exists.status(); + return status.WithMessage("failed to check existence of blob file '", blob->Uri(), + "': ", status.message()); + } + // A failed check is otherwise deferred to the open below, which can still succeed. + } + + Result> opened = blob->NewInputStream(fs_); + if (!opened.ok()) { + // The file can be deleted between the check above and this open. Classifying that from + // `opened.status()` would reintroduce the plugin-specific status codes this writer avoids, + // so ask FileSystem::Exists once more. This narrows the window rather than closing it; a + // check that cannot answer falls through to the open failure. + if (write_null_on_missing_file_) { + Result exists = fs_->Exists(blob->Uri()); + if (exists.ok() && !exists.value()) { + return HandleMissingFile(blob->Uri()); + } + } + return HandleFetchFailure(blob->Uri(), opened.status()); + } + return std::move(opened).value(); +} + +std::unique_ptr BlobFormatWriter::HandleMissingFile(const std::string& blob_uri) { + PAIMON_LOG_WARN(logger_, "Blob file %s does not exist, writing NULL for BLOB field %s into %s", + blob_uri.c_str(), blob_field_name_.c_str(), uri_.c_str()); + ++null_on_missing_file_count_; + return std::unique_ptr(); +} + +Result> BlobFormatWriter::HandleFetchFailure( + const std::string& blob_uri, const Status& status) { + if (!write_null_on_fetch_failure_) { + return status; + } + PAIMON_LOG_WARN(logger_, "Failed to fetch blob %s, writing NULL for BLOB field %s into %s: %s", + blob_uri.c_str(), blob_field_name_.c_str(), uri_.c_str(), + status.ToString().c_str()); + ++null_on_fetch_failure_count_; + return std::unique_ptr(); } Status BlobFormatWriter::WriteBytes(const char* data, int64_t length) { diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index a95d0715..78535c32 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -49,14 +49,27 @@ class OutputStream; namespace paimon::blob { +class BlobMetrics { + public: + /// Number of rows written as NULL because their referenced file did not exist. + static inline const char WRITE_NULL_ON_MISSING_FILE_COUNT[] = + "blob.write.null-on-missing-file.count"; + /// Number of rows written as NULL because their referenced data could not be reached. + static inline const char WRITE_NULL_ON_FETCH_FAILURE_COUNT[] = + "blob.write.null-on-fetch-failure.count"; +}; + // Blob format: // https://cwiki.apache.org/confluence/display/PAIMON/PIP-35%3A+Introduce+Blob+to+store+multimodal+data class BlobFormatWriter : public FormatWriter { public: - /// When opening a descriptor input fails, `write_null_on_missing_file` converts a - /// missing file (Status::NotExist) to a NULL element and `write_null_on_fetch_failure` - /// converts any other open failure; failures during the streaming copy always fail the - /// write. See Options::BLOB_WRITE_NULL_ON_MISSING_FILE / BLOB_WRITE_NULL_ON_FETCH_FAILURE. + /// `write_null_on_missing_file` converts a descriptor whose referenced file does not exist + /// (as reported by FileSystem::Exists) to a NULL element, and `write_null_on_fetch_failure` + /// converts any other failure to access the referenced data; failures during the streaming + /// copy always fail the write. The existence check runs only when + /// `write_null_on_missing_file` is enabled; otherwise a missing file follows + /// `write_null_on_fetch_failure` like any other failed open. + /// See Options::BLOB_WRITE_NULL_ON_MISSING_FILE / BLOB_WRITE_NULL_ON_FETCH_FAILURE. static Result> Create( const std::shared_ptr& out, const std::shared_ptr& data_type, bool write_null_on_missing_file, bool write_null_on_fetch_failure, @@ -86,8 +99,19 @@ class BlobFormatWriter : public FormatWriter { Status WriteBlob(std::string_view blob_data); /// Deserialize the descriptor and open an input stream on the referenced data. - Result> OpenDescriptorInputStream( - std::string_view blob_data) const; + /// Returns a null stream when the failure is converted to a NULL element by + /// `write_null_on_missing_file_` or `write_null_on_fetch_failure_`. + Result> OpenDescriptorInputStream(std::string_view blob_data); + + /// Convert a file that FileSystem::Exists reported as absent to a NULL element: count it and + /// return a null stream. Only reached under `write_null_on_missing_file_`, which callers check. + std::unique_ptr HandleMissingFile(const std::string& blob_uri); + + /// Apply `write_null_on_fetch_failure_` to a failure to reach the referenced data: returns + /// `status` when the option is disabled, and a null stream when it converts the failure to a + /// NULL element. `blob_uri` is only used for logging. + Result> HandleFetchFailure(const std::string& blob_uri, + const Status& status); Status WriteBytes(const char* data, int64_t length); Status WriteWithCrc32(const char* data, int64_t length); @@ -102,15 +126,19 @@ class BlobFormatWriter : public FormatWriter { uint32_t crc32_ = 0; std::vector bin_lengths_; std::shared_ptr out_; + /// Path of the blob file being written, not of any referenced blob. std::string uri_; PAIMON_UNIQUE_PTR tmp_buffer_; PAIMON_UNIQUE_PTR magic_number_bytes_; std::shared_ptr data_type_; + std::string blob_field_name_; std::shared_ptr fs_; std::shared_ptr pool_; std::shared_ptr metrics_; bool write_null_on_missing_file_ = false; bool write_null_on_fetch_failure_ = false; + uint64_t null_on_missing_file_count_ = 0; + uint64_t null_on_fetch_failure_count_ = 0; std::unique_ptr logger_; }; diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index 51e70ef1..f530bda1 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -24,6 +24,7 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" +#include "paimon/common/data/blob_descriptor.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/stream_utils.h" #include "paimon/data/blob.h" @@ -35,18 +36,78 @@ namespace paimon::blob::test { -/// A file system whose Open() always fails with the configured status, for verifying how the -/// writer classifies open failures by status code. +/// A file system whose Open() always fails with the configured status while Exists() keeps the +/// real local check, standing in for a plugin that reports a missing file as something other than +/// Status::NotExist. Open() calls are counted so a test can assert a missing file is never opened. class OpenFailFileSystem : public LocalFileSystem { public: explicit OpenFailFileSystem(Status open_status) : open_status_(std::move(open_status)) {} Result> Open(const std::string& path) const override { + ++open_call_count_; return open_status_; } + int64_t OpenCallCount() const { + return open_call_count_; + } + + private: + Status open_status_; + mutable int64_t open_call_count_ = 0; +}; + +/// A file system whose Exists() always fails with the configured status, counting the calls. By +/// default Open() is delegated to a separate LocalFileSystem so that it still succeeds +/// (LocalFileSystem::Open() calls Exists() on itself, so without the delegation a failed check +/// would also fail the open); a non-OK `open_status` makes Open() fail with it instead. +class ExistsFailFileSystem : public LocalFileSystem { + public: + explicit ExistsFailFileSystem(Status exists_status, Status open_status = Status::OK()) + : exists_status_(std::move(exists_status)), open_status_(std::move(open_status)) {} + + Result Exists(const std::string& path) const override { + ++exists_call_count_; + return exists_status_; + } + + Result> Open(const std::string& path) const override { + if (!open_status_.ok()) { + return open_status_; + } + return real_fs_.Open(path); + } + + int64_t ExistsCallCount() const { + return exists_call_count_; + } + private: + Status exists_status_; Status open_status_; + LocalFileSystem real_fs_; + mutable int64_t exists_call_count_ = 0; +}; + +/// A file system that reports a file as present on the first Exists() and absent afterwards, +/// standing in for a file deleted between the check and the open. Open() fails with a plain +/// IOError, so a test can tell a re-checked classification apart from one taken from the open. +class VanishingFileSystem : public LocalFileSystem { + public: + Result Exists(const std::string& path) const override { + return ++exists_call_count_ == 1; + } + + Result> Open(const std::string& path) const override { + return Status::IOError("mock io error"); + } + + int64_t ExistsCallCount() const { + return exists_call_count_; + } + + private: + mutable int64_t exists_call_count_ = 0; }; class BlobFormatWriterTestBase : public ::testing::Test { @@ -85,6 +146,20 @@ class BlobFormatWriterTestBase : public ::testing::Test { return paimon::test::TestHelper::MakeBlobDescriptorArray(struct_type_, blob, pool_); } + /// Build a single-row blob array holding `bytes` verbatim, for bytes no Blob can produce, + /// such as a truncated descriptor. + Result> MakeBlobArrayFromBytes(const std::string& bytes) const { + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Append()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(blob_builder->Append(bytes.data(), bytes.size())); + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder.Finish(&array)); + return array; + } + Result> ReadBackAsData() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, file_system_->Open(dir_->Str() + "/file.blob")); @@ -226,6 +301,12 @@ TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) { /*write_null_on_fetch_failure=*/false, file_system_, nullptr), "blob format writer create failed. pool is nullptr"); + // Test with nullptr file system + ASSERT_NOK_WITH_MSG( + BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, nullptr, pool_), + "blob format writer create failed. fs is nullptr"); + // Test with invalid field count (more than 1 field) auto multi_field_type = arrow::struct_( {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())}); @@ -454,8 +535,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnMissingFile) { ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); ASSERT_OK(AddBatchOnce(writer, missing_array)); - // A fetch failure is not converted to NULL by write_null_on_missing_file alone (aligned - // with Java); the rejected row leaves the writer usable. + // A fetch failure is not converted to NULL by write_null_on_missing_file alone; the + // rejected row leaves the writer usable. std::string file = paimon::test::GetDataDir() + "/xxhash.data"; ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, Blob::FromPath(file, /*offset=*/1 << 20, /*length=*/10)); @@ -492,20 +573,31 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnFetchFailure) { ASSERT_OK_AND_ASSIGN(auto bad_offset_array, PrepareDescriptorArray(bad_offset_blob)); ASSERT_OK(AddBatchOnce(writer, bad_offset_array)); - // A missing file is not converted to NULL by write_null_on_fetch_failure alone (aligned - // with Java); the rejected row leaves the writer usable. + // Without write_null_on_missing_file no existence check runs, so a missing file is not told + // apart from any other failed open and is converted by this option. ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, /*length=*/10)); ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); - ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, missing_array), "not exists"); + ASSERT_OK(AddBatchOnce(writer, missing_array)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); + // Both rows count as fetch failures. + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 0); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 2); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); - ASSERT_EQ(result_struct->length(), 1); + ASSERT_EQ(result_struct->length(), 2); ASSERT_TRUE(result_struct->field(0)->IsNull(0)); + ASSERT_TRUE(result_struct->field(0)->IsNull(1)); } TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) { @@ -536,6 +628,16 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) { ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); + // The two NULL rows had different causes, counted separately. + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 1); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 1); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); ASSERT_EQ(result_struct->length(), 3); ASSERT_TRUE(result_struct->field(0)->IsNull(0)); @@ -548,42 +650,251 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) { std::string_view(expected_data->data(), expected_data->size())); } -TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByStatusCode) { - // The missing-file vs fetch-failure split keys on the open status code (NotExist <-> missing - // file), independent of the file system implementation. - ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, - Blob::FromPath(dir_->Str() + "/any_file", /*offset=*/0, /*length=*/10)); - ASSERT_OK_AND_ASSIGN(auto array, PrepareDescriptorArray(blob)); - auto not_exist_fs = std::make_shared(Status::NotExist("mock not exist")); +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { + // Each case needs its own option pair and therefore its own writer, so none of them finishes + // the shared output stream: this test only asserts how a failure is classified. That the + // resulting NULL element is written correctly is covered by the TestWriteNullOn* tests. auto io_error_fs = std::make_shared(Status::IOError("mock io error")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, + Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, + /*length=*/10)); + ASSERT_OK_AND_ASSIGN(auto missing_array, PrepareDescriptorArray(missing_blob)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr existing_blob, + Blob::FromPath(paimon::test::GetDataDir() + "/xxhash.data")); + ASSERT_OK_AND_ASSIGN(auto existing_array, PrepareDescriptorArray(existing_blob)); + + // Missing file: classified without opening it, so what Open would return is irrelevant. { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, not_exist_fs, pool_)); - ASSERT_OK(AddBatchOnce(writer, array)); + /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, missing_array)); + ASSERT_EQ(io_error_fs->OpenCallCount(), 0); } + // Existing file that cannot be opened: a fetch failure, which this writer does not convert. { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); - ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "mock io error"); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); } + // The same fetch failure, now converted to NULL. { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); - ASSERT_OK(AddBatchOnce(writer, array)); + ASSERT_OK(AddBatchOnce(writer, existing_array)); } + // Missing file with only fetch-failure enabled: no existence check runs, so the file is + // opened and the failure is converted like any other fetch failure. The mock's count + // accumulates across cases, so compare against it. { + const int64_t open_calls_before = io_error_fs->OpenCallCount(); ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, not_exist_fs, pool_)); - ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "mock not exist"); + /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, missing_array)); + ASSERT_EQ(io_error_fs->OpenCallCount(), open_calls_before + 1); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 1); + } + + // A file deleted between the check and the open is still a missing file: the failed open + // triggers one more check rather than being classified by its status. Without it the deletion + // would defeat write_null_on_missing_file, which does not convert a fetch failure. Each case + // needs its own file system, since the mock reports the file as present only on the first call. + { + auto vanishing_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, existing_array)); + // One check before the open and one after it. + ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 1); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 0); + } + // The same deletion with both options enabled: classified as missing rather than swallowed + // by fetch-failure. + { + auto vanishing_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, existing_array)); + ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 1); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 0); + } + // The same deletion with only fetch-failure enabled: existence is never consulted, and the + // failed open is converted like any other fetch failure. + { + auto vanishing_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, existing_array)); + ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); + } + // Neither option: existence is never consulted, and the open failure propagates as it is. + { + auto vanishing_fs = std::make_shared(); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); + ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); + } +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnInvalidDescriptor) { + // Descriptor detection only inspects version and magic, so a descriptor truncated after those + // passes detection and then fails to deserialize: a fetch failure, not a missing file. + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, + Blob::FromPath(paimon::test::GetDataDir() + "/xxhash.data")); + PAIMON_UNIQUE_PTR descriptor = blob->ToDescriptor(pool_); + ASSERT_GT(descriptor->size(), 8); + std::string truncated(descriptor->data(), descriptor->size() - 8); + ASSERT_OK_AND_ASSIGN(bool is_descriptor, + BlobDescriptor::IsBlobDescriptor(truncated.data(), truncated.size())); + ASSERT_TRUE(is_descriptor); + ASSERT_OK_AND_ASSIGN(auto array, MakeBlobArrayFromBytes(truncated)); + + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "invalid blob descriptor"); + } + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + } + + // Only the second case reaches the stream; the first fails before writing any byte. + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 1); + ASSERT_TRUE(result_struct->field(0)->IsNull(0)); +} + +TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { + // An existence check that cannot answer leaves it unknown whether the file is there. With no + // fetch-failure handling to defer to, the write fails; otherwise the failed check is deferred + // to the open, whose own outcome decides. + ASSERT_OK_AND_ASSIGN(std::shared_ptr blob, + Blob::FromPath(paimon::test::GetDataDir() + "/xxhash.data")); + ASSERT_OK_AND_ASSIGN(auto array, PrepareDescriptorArray(blob)); + + // Deferred check failure whose open succeeds: the blob is written as data, not as NULL. + // This case finishes the shared output stream, so it runs first and is read back below. + { + auto exists_fail_fs = + std::make_shared(Status::IOError("mock exists error")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 1); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + ASSERT_OK_AND_ASSIGN( + uint64_t missing_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_MISSING_FILE_COUNT)); + ASSERT_EQ(missing_nulls, 0); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 0); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr result_struct, ReadBackAsData()); + ASSERT_EQ(result_struct->length(), 1); + ASSERT_FALSE(result_struct->field(0)->IsNull(0)); + auto binary_array = + arrow::internal::checked_pointer_cast(result_struct->field(0)); + ASSERT_OK_AND_ASSIGN(auto expected_data, blob->ToData(file_system_, pool_)); + ASSERT_EQ(binary_array->GetView(0), + std::string_view(expected_data->data(), expected_data->size())); + + // With no fetch-failure handling to defer to, the check failure fails the write. + { + auto exists_fail_fs = + std::make_shared(Status::IOError("mock exists error")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/false, exists_fail_fs, pool_)); + // The reported failure names the check and keeps the underlying status message. + Status check_status = AddBatchOnce(writer, array); + ASSERT_NOK_WITH_MSG(check_status, "failed to check existence of blob file"); + ASSERT_NOK_WITH_MSG(check_status, "mock exists error"); + } + // Deferred check failure whose open then fails: a fetch failure. The re-check after the + // failed open cannot answer either, so it falls through to the open failure. + { + auto exists_fail_fs = std::make_shared( + Status::IOError("mock exists error"), Status::IOError("mock open error")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + output_stream_, struct_type_, /*write_null_on_missing_file=*/true, + /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + // One check before the open and one after it failed. + ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 2); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 1); + } + // With only write_null_on_fetch_failure, no existence check runs at all; the open succeeds + // and the blob is written as data. A separate output stream keeps the data bytes out of the + // already finished shared stream. + { + auto exists_fail_fs = + std::make_shared(Status::IOError("mock exists error")); + ASSERT_OK_AND_ASSIGN(std::shared_ptr side_stream, + file_system_->Create(dir_->Str() + "/side.blob", /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, + BlobFormatWriter::Create( + side_stream, struct_type_, /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + ASSERT_OK(AddBatchOnce(writer, array)); + ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 0); + ASSERT_OK_AND_ASSIGN( + uint64_t fetch_failure_nulls, + writer->GetWriterMetrics()->GetCounter(BlobMetrics::WRITE_NULL_ON_FETCH_FAILURE_COUNT)); + ASSERT_EQ(fetch_failure_nulls, 0); + ASSERT_OK(side_stream->Flush()); + ASSERT_OK(side_stream->Close()); } } diff --git a/src/paimon/format/orc/CMakeLists.txt b/src/paimon/format/orc/CMakeLists.txt index 8bb2e7dc..d86750ea 100644 --- a/src/paimon/format/orc/CMakeLists.txt +++ b/src/paimon/format/orc/CMakeLists.txt @@ -40,7 +40,6 @@ if(PAIMON_ENABLE_ORC) fmt orc::orc tbb - ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 8f78fd6c..8129902c 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -40,7 +40,6 @@ add_paimon_lib(paimon_parquet_file_format arrow glog fmt - ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index 3e7b0525..43d85b3a 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -38,7 +38,6 @@ if(PAIMON_ENABLE_LUCENE) lucene arrow fmt - ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS paimon_shared diff --git a/src/paimon/global_index/lumina/CMakeLists.txt b/src/paimon/global_index/lumina/CMakeLists.txt index 326f6660..b0496df6 100644 --- a/src/paimon/global_index/lumina/CMakeLists.txt +++ b/src/paimon/global_index/lumina/CMakeLists.txt @@ -27,7 +27,6 @@ if(PAIMON_ENABLE_LUMINA) arrow glog fmt - ${CMAKE_DL_LIBS} Threads::Threads SHARED_LINK_LIBS lumina::interface diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 5f878849..9c7406cd 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -765,9 +765,9 @@ TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailure) { << "expected:" << expected_with_rk->ToString(); } -TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailureKeepsMissingFileFailing) { - // blob-write-null-on-fetch-failure only converts non-NotExist open failures; a missing - // descriptor file still fails the write when only this option is enabled. +TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailureCoversMissingFile) { + // The existence check runs only under blob-write-null-on-missing-file, so with only + // blob-write-null-on-fetch-failure a missing file is converted as an ordinary fetch failure. arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), BlobUtils::ToArrowField("blob", true)}; @@ -787,7 +787,8 @@ TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailureKeepsMissingFileFailing) { std::string raw_json = R"([ ["str_0", 0, "blob_data_0"], - ["str_1", 1, "blob_data_1"] + ["str_1", 1, "blob_data_1"], + ["str_2", 2, "blob_data_2"] ])"; auto raw_array = std::dynamic_pointer_cast( arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), raw_json).ValueOrDie()); @@ -795,8 +796,35 @@ TEST_P(BlobTableInteTest, TestWriteNullOnFetchFailureKeepsMissingFileFailing) { ASSERT_OK(DeleteDescriptorTarget(desc_array, "blob", /*row=*/1)); auto schema = arrow::schema(fields); - ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, schema->field_names(), {desc_array}), - "not exists"); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, {}, schema->field_names(), {desc_array})); + ASSERT_OK(Commit(table_path, commit_msgs)); + + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + VerifyDataFileMetas(plan, /*expected_file_count=*/2, /*expected_row_counts=*/{3, 3}, + /*expected_min_seqs=*/{1, 1}, /*expected_max_seqs=*/{1, 1}, + /*expected_first_row_ids=*/{0, 0}, + /*expected_write_cols=*/ + {std::vector{"f0", "f1"}, std::vector{"blob"}}); + + ASSERT_OK_AND_ASSIGN(auto result, ReadTable(table_path, schema->field_names(), plan)); + ASSERT_TRUE(result.chunked_array); + auto read_concat = arrow::Concatenate(result.chunked_array->chunks()).ValueOrDie(); + auto read_struct = std::dynamic_pointer_cast(read_concat); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(read_struct, {"blob"})); + + std::string expected_json = R"([ + ["str_0", 0, "blob_data_0"], + ["str_1", 1, null], + ["str_2", 2, "blob_data_2"] + ])"; + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_json) + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << std::endl + << "expected:" << expected_with_rk->ToString(); } TEST_P(BlobTableInteTest, TestBasic) { From cf50b55943f4cd018a84c6b42f165e9d54a8426b Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:43:00 +0800 Subject: [PATCH 108/138] fix(btree): preserve empty keys in index metadata & disable global bitmap index --- README.md | 12 +- .../paimon/global_index/global_index_reader.h | 2 +- .../global_index/global_index_write_task.h | 2 +- .../global_index/global_indexer_factory.h | 6 +- .../btree/btree_file_meta_selector.cpp | 58 +++++-- .../btree/btree_file_meta_selector.h | 7 + .../btree/btree_file_meta_selector_test.cpp | 39 +++++ .../btree_global_index_integration_test.cpp | 45 +++++ .../global_index/btree/btree_index_meta.cpp | 59 +++++-- .../global_index/btree/btree_index_meta.h | 14 +- .../btree/btree_index_meta_test.cpp | 103 ++++++++++-- .../global_index/global_indexer_factory.cpp | 11 ++ .../global_indexer_factory_test.cpp | 12 +- test/inte/global_index_test.cpp | 159 +++++++++++++++--- 14 files changed, 444 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 81d3e82f..d1140f51 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,16 @@ Paimon-cpp currently provides: - **File systems**: file system abstraction with built-in local and Jindo file system support. - **File formats**: file format abstraction with built-in ORC, Parquet, and Avro support. - **Runtime utilities**: memory pool and thread pool abstractions with default implementations. -- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index capabilities including bitmap index, B-tree index, DiskANN-based vector search with Lumina, and Lucene-based full-text search. -- **Compatibility**: compatibility with Apache Paimon Java format and communication protocols, including commit messages, data splits, and manifests. +- **AI-Oriented Features**: supports RowTracking and DataEvolution mode and provides Global Index + capabilities including B-tree index, DiskANN-based vector search with Lumina, and Lucene-based + full-text search. +- **Compatibility**: compatibility with Apache Paimon Java format and communication protocols, + including commit messages, data splits, and manifests. + +> **Bitmap global index compatibility:** Java Paimon now uses a dedicated bitmap global index +> format instead of the previously shared wrapped bitmap file index format. +> Paimon C++ therefore currently treats the `bitmap` global index type as unsupported. The legacy +> implementation remains in the codebase pending migration to the Java-compatible format. Note: Linux x86_64 and macOS arm64 builds are currently verified. diff --git a/include/paimon/global_index/global_index_reader.h b/include/paimon/global_index/global_index_reader.h index 36bb201a..ca3f4c9b 100644 --- a/include/paimon/global_index/global_index_reader.h +++ b/include/paimon/global_index/global_index_reader.h @@ -50,7 +50,7 @@ class PAIMON_EXPORT GlobalIndexReader : public FunctionVisitor> Get( const std::string& identifier, const std::map& options); diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp index 07537949..cedfa4a7 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector.cpp @@ -49,11 +49,7 @@ Result> BTreeFileMetaSelector::VisitEqual(const L if (meta.OnlyNulls()) { return false; } - MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey()); - MemorySlice max_key_slice = WrapKeySlice(meta.LastKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp_min, comparator_(literal_slice, min_key_slice)); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp_max, comparator_(literal_slice, max_key_slice)); - return cmp_min >= 0 && cmp_max <= 0; + return Overlaps(meta, literal_slice, literal_slice); }); } @@ -70,8 +66,7 @@ Result> BTreeFileMetaSelector::VisitLessThan( if (meta.OnlyNulls()) { return false; } - MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(min_key_slice, literal_slice)); + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareFirstKey(meta, literal_slice)); return cmp < 0; }); } @@ -84,8 +79,7 @@ Result> BTreeFileMetaSelector::VisitLessOrEqual( if (meta.OnlyNulls()) { return false; } - MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(min_key_slice, literal_slice)); + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareFirstKey(meta, literal_slice)); return cmp <= 0; }); } @@ -98,8 +92,7 @@ Result> BTreeFileMetaSelector::VisitGreaterThan( if (meta.OnlyNulls()) { return false; } - MemorySlice max_key_slice = WrapKeySlice(meta.LastKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(max_key_slice, literal_slice)); + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareLastKey(meta, literal_slice)); return cmp > 0; }); } @@ -112,8 +105,7 @@ Result> BTreeFileMetaSelector::VisitGreaterOrEqua if (meta.OnlyNulls()) { return false; } - MemorySlice max_key_slice = WrapKeySlice(meta.LastKey()); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(max_key_slice, literal_slice)); + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, CompareLastKey(meta, literal_slice)); return cmp >= 0; }); } @@ -130,12 +122,9 @@ Result> BTreeFileMetaSelector::VisitIn( if (meta.OnlyNulls()) { return false; } - MemorySlice min_key_slice = WrapKeySlice(meta.FirstKey()); - MemorySlice max_key_slice = WrapKeySlice(meta.LastKey()); for (const auto& literal_slice : literal_slices) { - PAIMON_ASSIGN_OR_RAISE(int32_t cmp_min, comparator_(literal_slice, min_key_slice)); - PAIMON_ASSIGN_OR_RAISE(int32_t cmp_max, comparator_(literal_slice, max_key_slice)); - if (cmp_min >= 0 && cmp_max <= 0) { + PAIMON_ASSIGN_OR_RAISE(bool overlaps, Overlaps(meta, literal_slice, literal_slice)); + if (overlaps) { return true; } } @@ -179,6 +168,39 @@ Result> BTreeFileMetaSelector::Filter( return result; } +Result BTreeFileMetaSelector::Overlaps(const BTreeIndexMeta& meta, const MemorySlice& from, + const MemorySlice& to) const { + if (meta.FirstKey()) { + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(to, WrapKeySlice(meta.FirstKey()))); + if (cmp < 0) { + return false; + } + } + if (meta.LastKey()) { + PAIMON_ASSIGN_OR_RAISE(int32_t cmp, comparator_(from, WrapKeySlice(meta.LastKey()))); + if (cmp > 0) { + return false; + } + } + return true; +} + +Result BTreeFileMetaSelector::CompareFirstKey(const BTreeIndexMeta& meta, + const MemorySlice& literal) const { + if (!meta.FirstKey()) { + return -1; + } + return comparator_(WrapKeySlice(meta.FirstKey()), literal); +} + +Result BTreeFileMetaSelector::CompareLastKey(const BTreeIndexMeta& meta, + const MemorySlice& literal) const { + if (!meta.LastKey()) { + return 1; + } + return comparator_(WrapKeySlice(meta.LastKey()), literal); +} + MemorySlice BTreeFileMetaSelector::WrapKeySlice(const std::shared_ptr& key) { return MemorySlice::Wrap(MemorySegment::WrapView(key->data(), key->size())); } diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector.h b/src/paimon/common/global_index/btree/btree_file_meta_selector.h index 405efc90..b59fc7ed 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector.h +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector.h @@ -59,6 +59,13 @@ class BTreeFileMetaSelector : public FunctionVisitor> Filter(const MetaPredicate& predicate) const; + Result Overlaps(const BTreeIndexMeta& meta, const MemorySlice& from, + const MemorySlice& to) const; + + Result CompareFirstKey(const BTreeIndexMeta& meta, const MemorySlice& literal) const; + + Result CompareLastKey(const BTreeIndexMeta& meta, const MemorySlice& literal) const; + Result SerializeLiteral(const Literal& literal) const; /// Create a non-owning MemorySlice view over the raw bytes of a key, diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp index 4e532fe1..ce77207b 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp @@ -228,4 +228,43 @@ TEST_F(BTreeFileMetaSelectorTest, TestOnlyNullsFileExcludedFromRangeQueries) { ASSERT_EQ(names.count("file6"), 1u); } +TEST_F(BTreeFileMetaSelectorTest, TestEmptyStringKeyDoesNotCrash) { + std::shared_ptr pool = GetDefaultPool(); + std::shared_ptr key_type = arrow::utf8(); + auto serialize = [&](const char* value, size_t size) { + Literal literal(FieldType::STRING, value, size); + EXPECT_OK_AND_ASSIGN(std::shared_ptr result, + KeySerializer::SerializeKey(literal, key_type, pool.get())); + return result; + }; + + auto empty_meta = + std::make_shared(serialize("", 0), serialize("www.example.com", 15), false); + auto normal_meta = + std::make_shared(serialize("aaa.com", 7), serialize("zzz.com", 7), false); + auto null_meta = std::make_shared(nullptr, nullptr, true); + std::vector files = { + GlobalIndexIOMeta("file_empty", 1, empty_meta->Serialize(pool.get())), + GlobalIndexIOMeta("file_normal", 1, normal_meta->Serialize(pool.get())), + GlobalIndexIOMeta("file_nulls", 1, null_meta->Serialize(pool.get())), + }; + + BTreeFileMetaSelector selector(files, key_type, pool); + + ASSERT_OK_AND_ASSIGN(std::vector result, + selector.VisitEqual(Literal(FieldType::STRING, "www.example.com", 15))); + CheckResult(result, {"file_empty", "file_normal"}); + + ASSERT_OK_AND_ASSIGN(result, selector.VisitLessThan(Literal(FieldType::STRING, "bbb.com", 7))); + CheckResult(result, {"file_empty", "file_normal"}); + + ASSERT_OK_AND_ASSIGN( + result, selector.VisitGreaterThan(Literal(FieldType::STRING, "www.example.com", 15))); + CheckResult(result, {"file_normal"}); + + ASSERT_OK_AND_ASSIGN(result, selector.VisitIn({Literal(FieldType::STRING, "", 0), + Literal(FieldType::STRING, "zzz.com", 7)})); + CheckResult(result, {"file_empty", "file_normal"}); +} + } // namespace paimon::test diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index 041394bc..e5aeaac2 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -23,6 +23,7 @@ #include "paimon/common/factories/io_hook.h" #include "paimon/common/global_index/btree/btree_global_index_writer.h" #include "paimon/common/global_index/btree/btree_global_indexer.h" +#include "paimon/common/global_index/btree/btree_index_meta.h" #include "paimon/common/global_index/btree/lazy_filtered_btree_reader.h" #include "paimon/common/options/memory_size.h" #include "paimon/common/utils/scope_guard.h" @@ -460,6 +461,50 @@ TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadStringData) { } } +TEST_P(BTreeGlobalIndexIntegrationTest, WriteEmptyStringKeyMetadata) { + auto file_writer = std::make_shared(fs_, base_path_); + auto field = arrow::field("str_field", arrow::utf8()); + auto c_schema = CreateArrowSchema(field); + + std::map options = {{BtreeDefs::kBtreeIndexBlockSize, "128"}, + {BtreeDefs::kBtreeIndexCompression, GetParam()}}; + ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options)); + ASSERT_OK_AND_ASSIGN(auto writer, + indexer->CreateWriter("str_field", c_schema.get(), file_writer, pool_)); + auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({field}), R"([ + [null], + [""], + ["abc"] + ])") + .ValueOrDie(); + + ArrowArray c_array; + ASSERT_TRUE(arrow::ExportArray(*array, &c_array).ok()); + std::vector row_ids(array->length()); + std::iota(row_ids.begin(), row_ids.end(), 0); + ASSERT_OK(writer->AddBatch(&c_array, std::move(row_ids))); + ASSERT_OK_AND_ASSIGN(auto metas, writer->Finish()); + ASSERT_EQ(metas.size(), 1); + + std::shared_ptr meta = + BTreeIndexMeta::Deserialize(metas[0].metadata, pool_.get()); + ASSERT_TRUE(meta->FirstKey()); + ASSERT_EQ(meta->FirstKey()->size(), 0); + ASSERT_TRUE(meta->LastKey()); + ASSERT_EQ(std::string(meta->LastKey()->data(), meta->LastKey()->size()), "abc"); + ASSERT_TRUE(meta->HasNulls()); + ASSERT_FALSE(meta->OnlyNulls()); + + auto file_reader = std::make_shared(fs_, base_path_); + c_schema = CreateArrowSchema(field); + ASSERT_OK_AND_ASSIGN(auto reader, + indexer->CreateReader(c_schema.get(), file_reader, metas, pool_)); + + Literal empty(FieldType::STRING, "", 0); + ASSERT_OK_AND_ASSIGN(auto result, reader->VisitEqual(empty)); + CheckResult(result, {1}); +} + TEST_P(BTreeGlobalIndexIntegrationTest, WriteAndReadBigIntData) { auto file_writer = std::make_shared(fs_, base_path_); auto field = arrow::field("bigint_field", arrow::int64()); diff --git a/src/paimon/common/global_index/btree/btree_index_meta.cpp b/src/paimon/common/global_index/btree/btree_index_meta.cpp index 94d7b44c..9ba88e53 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.cpp +++ b/src/paimon/common/global_index/btree/btree_index_meta.cpp @@ -22,48 +22,71 @@ #include "paimon/common/memory/memory_slice_output.h" namespace paimon { +namespace { + +std::shared_ptr ReadKey(MemorySliceInput* input, int32_t key_length, MemoryPool* pool) { + if (key_length == 0) { + return std::make_shared(0, pool); + } + return input->ReadSliceView(key_length).CopyBytes(pool); +} + +} // namespace std::shared_ptr BTreeIndexMeta::Deserialize(const std::shared_ptr& meta, paimon::MemoryPool* pool) { - auto slice = MemorySlice::Wrap(meta); - auto input = slice.ToInput(); - auto first_key_len = input.ReadInt(); - std::shared_ptr first_key; - if (first_key_len) { - first_key = input.ReadSliceView(first_key_len).CopyBytes(pool); - } - auto last_key_len = input.ReadInt(); - std::shared_ptr last_key; - if (last_key_len) { - last_key = input.ReadSliceView(last_key_len).CopyBytes(pool); + MemorySlice slice = MemorySlice::Wrap(meta); + MemorySliceInput input = slice.ToInput(); + int32_t first_key_len = input.ReadInt(); + std::shared_ptr first_key = ReadKey(&input, first_key_len, pool); + int32_t last_key_len = input.ReadInt(); + std::shared_ptr last_key = ReadKey(&input, last_key_len, pool); + bool has_nulls = input.ReadByte() == static_cast(1); + + if (input.Available() >= 2) { + int8_t format_version = input.ReadByte(); + if (format_version == kFormatVersionWithNullFlags) { + int8_t null_key_flags = input.ReadByte(); + if ((null_key_flags & kFirstKeyIsNull) != 0) { + first_key.reset(); + } + if ((null_key_flags & kLastKeyIsNull) != 0) { + last_key.reset(); + } + } + } else if (first_key_len == 0 && last_key_len == 0 && has_nulls) { + // Legacy metadata used zero length for null keys. Both empty boundaries plus a null bitmap + // identify an all-null file; a single empty boundary remains a valid serialized key. + first_key.reset(); + last_key.reset(); } - auto has_nulls = input.ReadByte() == static_cast(1); return std::make_shared(first_key, last_key, has_nulls); } std::shared_ptr BTreeIndexMeta::Serialize(paimon::MemoryPool* pool) const { - // Calculate total size: first_key_len(4) + first_key + last_key_len(4) + last_key + - // has_nulls(1) int32_t first_key_size = first_key_ ? first_key_->size() : 0; int32_t last_key_size = last_key_ ? last_key_->size() : 0; int32_t total_size = Size(); - MemorySliceOutput output(total_size, pool); + int8_t null_key_flags = 0; - // Write first_key_len and first_key output.WriteValue(first_key_size); if (first_key_) { output.WriteBytes(first_key_); + } else { + null_key_flags |= kFirstKeyIsNull; } - // Write last_key_len and last_key output.WriteValue(last_key_size); if (last_key_) { output.WriteBytes(last_key_); + } else { + null_key_flags |= kLastKeyIsNull; } - // Write has_nulls output.WriteValue(static_cast(has_nulls_ ? 1 : 0)); + output.WriteValue(kFormatVersionWithNullFlags); + output.WriteValue(null_key_flags); return output.ToSlice().GetOrCreateHeapMemory(pool); } diff --git a/src/paimon/common/global_index/btree/btree_index_meta.h b/src/paimon/common/global_index/btree/btree_index_meta.h index 05f08f91..85df5ff3 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta.h +++ b/src/paimon/common/global_index/btree/btree_index_meta.h @@ -19,14 +19,16 @@ #pragma once +#include #include #include "paimon/common/memory/memory_slice_input.h" #include "paimon/memory/bytes.h" namespace paimon { -/// Index Meta of each BTree index file. The first key and last key of this meta could be null if -/// the entire btree index file only contains nulls. +/// Index metadata for each BTree index file. +/// +/// Empty serialized keys are valid, so null boundary keys are encoded separately with flags. class BTreeIndexMeta { public: static std::shared_ptr Deserialize(const std::shared_ptr& meta, @@ -56,11 +58,15 @@ class BTreeIndexMeta { private: int32_t Size() const { - // 9 bytes => first_key_len(4 byte) + last_key_len(4 byte) + has_null(1 byte) - return (first_key_ ? first_key_->size() : 0) + (last_key_ ? last_key_->size() : 0) + 9; + // 11 bytes => key lengths (8) + has_nulls (1) + format version (1) + null flags (1). + return (first_key_ ? first_key_->size() : 0) + (last_key_ ? last_key_->size() : 0) + 11; } private: + static constexpr int8_t kFormatVersionWithNullFlags = 1; + static constexpr int8_t kFirstKeyIsNull = 1; + static constexpr int8_t kLastKeyIsNull = 1 << 1; + std::shared_ptr first_key_; std::shared_ptr last_key_; bool has_nulls_; diff --git a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp b/src/paimon/common/global_index/btree/btree_index_meta_test.cpp index c29ec6be..70c4dfcb 100644 --- a/src/paimon/common/global_index/btree/btree_index_meta_test.cpp +++ b/src/paimon/common/global_index/btree/btree_index_meta_test.cpp @@ -20,6 +20,7 @@ #include "paimon/common/global_index/btree/btree_index_meta.h" #include "gtest/gtest.h" +#include "paimon/common/memory/memory_slice_output.h" #include "paimon/memory/memory_pool.h" namespace paimon::test { @@ -29,6 +30,24 @@ class BTreeIndexMetaTest : public ::testing::Test { pool_ = GetDefaultPool(); } + std::shared_ptr LegacyMetaBytes(const std::shared_ptr& first_key, + const std::shared_ptr& last_key, + bool has_nulls) const { + int32_t first_key_size = first_key ? first_key->size() : 0; + int32_t last_key_size = last_key ? last_key->size() : 0; + MemorySliceOutput output(first_key_size + last_key_size + 9, pool_.get()); + output.WriteValue(first_key_size); + if (first_key) { + output.WriteBytes(first_key); + } + output.WriteValue(last_key_size); + if (last_key) { + output.WriteBytes(last_key); + } + output.WriteValue(static_cast(has_nulls ? 1 : 0)); + return output.ToSlice().CopyBytes(pool_.get()); + } + std::shared_ptr pool_; }; @@ -61,29 +80,93 @@ TEST_F(BTreeIndexMetaTest, SerializeDeserializeNormalKeys) { ASSERT_TRUE(deserialized->HasNulls()); } -TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyKeys) { - // Create a BTreeIndexMeta with empty keys (OnlyNulls case) +TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstKey) { + auto empty_key = std::make_shared(0, pool_.get()); + auto last_key = std::make_shared("last_key_data", pool_.get()); + auto meta = std::make_shared(empty_key, last_key, false); + + auto serialized = meta->Serialize(pool_.get()); + ASSERT_EQ(serialized->size(), 11 + last_key->size()); + + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_TRUE(deserialized->FirstKey()); + ASSERT_EQ(deserialized->FirstKey()->size(), 0); + ASSERT_TRUE(deserialized->LastKey()); + ASSERT_EQ(std::string(deserialized->LastKey()->data(), deserialized->LastKey()->size()), + "last_key_data"); + ASSERT_FALSE(deserialized->HasNulls()); + ASSERT_FALSE(deserialized->OnlyNulls()); +} + +TEST_F(BTreeIndexMetaTest, SerializeDeserializeEmptyFirstAndLastKeysWithNulls) { + auto empty_key = std::make_shared(0, pool_.get()); + auto meta = std::make_shared(empty_key, empty_key, true); + + auto serialized = meta->Serialize(pool_.get()); + ASSERT_EQ(serialized->size(), 11); + + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_TRUE(deserialized->FirstKey()); + ASSERT_EQ(deserialized->FirstKey()->size(), 0); + ASSERT_TRUE(deserialized->LastKey()); + ASSERT_EQ(deserialized->LastKey()->size(), 0); + ASSERT_TRUE(deserialized->HasNulls()); + ASSERT_FALSE(deserialized->OnlyNulls()); +} + +TEST_F(BTreeIndexMetaTest, SerializeDeserializeOnlyNulls) { auto meta = std::make_shared(nullptr, nullptr, true); - // Serialize auto serialized = meta->Serialize(pool_.get()); - ASSERT_TRUE(serialized); + ASSERT_EQ(serialized->size(), 11); - // Deserialize auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); ASSERT_TRUE(deserialized); - - // Verify keys are null ASSERT_FALSE(deserialized->FirstKey()); ASSERT_FALSE(deserialized->LastKey()); - - // Verify has_nulls ASSERT_TRUE(deserialized->HasNulls()); + ASSERT_TRUE(deserialized->OnlyNulls()); +} + +TEST_F(BTreeIndexMetaTest, DeserializeLegacyOnlyNulls) { + auto empty_key = std::make_shared(0, pool_.get()); + auto serialized = LegacyMetaBytes(empty_key, empty_key, true); - // Verify OnlyNulls + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_FALSE(deserialized->FirstKey()); + ASSERT_FALSE(deserialized->LastKey()); + ASSERT_TRUE(deserialized->HasNulls()); ASSERT_TRUE(deserialized->OnlyNulls()); } +TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstKey) { + auto empty_key = std::make_shared(0, pool_.get()); + auto last_key = std::make_shared("last_key_data", pool_.get()); + auto serialized = LegacyMetaBytes(empty_key, last_key, false); + + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_TRUE(deserialized->FirstKey()); + ASSERT_EQ(deserialized->FirstKey()->size(), 0); + ASSERT_TRUE(deserialized->LastKey()); + ASSERT_EQ(std::string(deserialized->LastKey()->data(), deserialized->LastKey()->size()), + "last_key_data"); + ASSERT_FALSE(deserialized->HasNulls()); + ASSERT_FALSE(deserialized->OnlyNulls()); +} + +TEST_F(BTreeIndexMetaTest, DeserializeLegacyEmptyFirstAndLastKeysWithoutNulls) { + auto empty_key = std::make_shared(0, pool_.get()); + auto serialized = LegacyMetaBytes(empty_key, empty_key, false); + + auto deserialized = BTreeIndexMeta::Deserialize(serialized, pool_.get()); + ASSERT_TRUE(deserialized->FirstKey()); + ASSERT_EQ(deserialized->FirstKey()->size(), 0); + ASSERT_TRUE(deserialized->LastKey()); + ASSERT_EQ(deserialized->LastKey()->size(), 0); + ASSERT_FALSE(deserialized->HasNulls()); + ASSERT_FALSE(deserialized->OnlyNulls()); +} + TEST_F(BTreeIndexMetaTest, HasNullsAndOnlyNulls) { // Case 1: Has nulls with keys auto meta1 = diff --git a/src/paimon/common/global_index/global_indexer_factory.cpp b/src/paimon/common/global_index/global_indexer_factory.cpp index 11dbc981..f40a42d1 100644 --- a/src/paimon/common/global_index/global_indexer_factory.cpp +++ b/src/paimon/common/global_index/global_indexer_factory.cpp @@ -31,6 +31,17 @@ const char GlobalIndexerFactory::GLOBAL_INDEX_IDENTIFIER_SUFFIX[] = "-global"; Result> GlobalIndexerFactory::Get( const std::string& identifier, const std::map& options) { + // Java now uses a dedicated bitmap global index format instead of the previously shared + // wrapped file index format. Keep the legacy implementation registered for a future migration, + // but do not expose it as a compatible global index. + static constexpr const char* kEnableLegacyBitmapForTesting = + "bitmap-global-index.legacy-format.enabled-for-testing"; + auto enable_legacy_bitmap = options.find(kEnableLegacyBitmapForTesting); + if (identifier == "bitmap" && + (enable_legacy_bitmap == options.end() || enable_legacy_bitmap->second != "true")) { + return std::unique_ptr(); + } + // Compatibility: "lumina-vector-ann" was the old identifier for lumina global index. std::string final_identifier = (identifier == "lumina-vector-ann" ? "lumina" : identifier); std::string global_index_identifier = final_identifier + GLOBAL_INDEX_IDENTIFIER_SUFFIX; diff --git a/src/paimon/common/global_index/global_indexer_factory_test.cpp b/src/paimon/common/global_index/global_indexer_factory_test.cpp index 77e819ce..c03fec58 100644 --- a/src/paimon/common/global_index/global_indexer_factory_test.cpp +++ b/src/paimon/common/global_index/global_indexer_factory_test.cpp @@ -26,13 +26,19 @@ #include "paimon/testing/utils/testharness.h" namespace paimon::test { -TEST(GlobalIndexerFactoryTest, TestSimple) { +TEST(GlobalIndexerFactoryTest, TestBitmapUnsupported) { std::map options; ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, GlobalIndexerFactory::Get("bitmap", options)); + ASSERT_FALSE(indexer); +} - auto bitmap_global_index = dynamic_cast(indexer.get()); - ASSERT_TRUE(bitmap_global_index); +TEST(GlobalIndexerFactoryTest, TestLegacyBitmapEnabledForTesting) { + std::map options = { + {"bitmap-global-index.legacy-format.enabled-for-testing", "true"}}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, + GlobalIndexerFactory::Get("bitmap", options)); + ASSERT_TRUE(dynamic_cast(indexer.get())); } TEST(GlobalIndexerFactoryTest, TestNonExist) { diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index e1b76b68..3cb84874 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -47,6 +47,12 @@ namespace paimon::test { using ParamType = std::tuple; /// This is a sdk end-to-end test for global index. class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInterface { + static const std::map& LegacyBitmapTestOptions() { + static const std::map options = { + {"bitmap-global-index.legacy-format.enabled-for-testing", "true"}}; + return options; + } + void SetUp() override { file_format_ = std::get<0>(GetParam()); dir_ = UniqueTestDirectory::Create("local"); @@ -63,13 +69,17 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter void CreateTable(const std::vector& partition_keys, const std::shared_ptr& schema, const std::map& options) const { + std::map test_options = options; + for (const auto& [key, value] : LegacyBitmapTestOptions()) { + test_options[key] = value; + } ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(dir_->Str(), {}, fs_)); ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &c_schema, partition_keys, - /*primary_keys=*/{}, options, + /*primary_keys=*/{}, test_options, /*ignore_if_exists=*/false)); } @@ -471,10 +481,11 @@ TEST_P(GlobalIndexTest, TestScanIndex) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); // test index reader // test f0 field ASSERT_OK_AND_ASSIGN(auto index_readers, global_index_scan->CreateReaders("f0", std::nullopt)); @@ -641,10 +652,11 @@ TEST_P(GlobalIndexTest, TestScanIndexWithSpecificSnapshot) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; // snapshot 2 has f0 index - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/2l, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/2l, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); // test index reader // test f0 field ASSERT_OK_AND_ASSIGN(auto index_readers, global_index_scan->CreateReaders("f0", std::nullopt)); @@ -690,10 +702,11 @@ TEST_P(GlobalIndexTest, TestScanIndexWithSpecificSnapshotWithNoIndex) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; // snapshot 1 has no index - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/1l, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/1l, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); // test index reader ASSERT_OK_AND_ASSIGN(auto index_readers, global_index_scan->CreateReaders("f0", std::nullopt)); ASSERT_EQ(index_readers.size(), 0u); @@ -714,10 +727,11 @@ TEST_P(GlobalIndexTest, TestScanIndexWithRange) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); auto global_index_scan_impl = std::dynamic_pointer_cast(global_index_scan); { // test index reader @@ -754,10 +768,10 @@ TEST_P(GlobalIndexTest, TestScanIndexWithPartition) { "/append_with_global_index_with_partition.db/append_with_global_index_with_partition"; auto check_result = [&](const std::optional>>& partitions) { - ASSERT_OK_AND_ASSIGN( - std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, partitions, - /*options=*/{}, fs_, /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + partitions, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); // test index reader ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 4)})); @@ -810,10 +824,11 @@ TEST_P(GlobalIndexTest, TestScanUnregisteredIndex) { std::string table_path = paimon::test::GetDataDir() + "/" + file_format_ + "/append_with_global_index.db/append_with_global_index"; - ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, - GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, - /*partitions=*/std::nullopt, /*options=*/{}, fs_, - /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, LegacyBitmapTestOptions(), fs_, + /*executor=*/nullptr, pool_)); ASSERT_OK_AND_ASSIGN(auto index_readers, global_index_scan->CreateReaders("f0", std::nullopt)); ASSERT_EQ(index_readers.size(), 0u); @@ -2412,6 +2427,100 @@ TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndex) { } } +TEST_P(GlobalIndexTest, TestBTreeEmptyStringKeyPredicates) { + CreateTable(); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields_); + std::vector write_cols = schema->field_names(); + + // Null keys are tracked separately. The non-null keys remain monotonically increasing. + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([ +[null, 10, 0, 10.0], +["", 20, 1, 20.0], +["abc", 30, 2, 30.0] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols, src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", "btree", + /*options=*/{}, Range(0, 2))); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, /*options=*/{}, fs_, + /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(auto index_readers, + global_index_scan->CreateReaders("f0", /*row_range_index=*/std::nullopt)); + ASSERT_EQ(index_readers.size(), 1u); + auto index_reader = index_readers[0]; + + Literal empty(FieldType::STRING, "", 0); + Literal abc(FieldType::STRING, "abc", 3); + Literal missing(FieldType::STRING, "missing", 7); + + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitEqual(empty)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitEqual(abc)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitEqual(missing)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitNotEqual(empty)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitIsNull()); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{0}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitIsNotNull()); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1,2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitIn({empty, abc, missing})); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1,2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitNotIn({empty, missing})); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitLessThan(abc)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitLessOrEqual(empty)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{1}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitGreaterThan(empty)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } + { + ASSERT_OK_AND_ASSIGN(auto result, index_reader->VisitGreaterOrEqual(abc)); + ASSERT_TRUE(result); + ASSERT_EQ(result->ToString(), "{2}"); + } +} + TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndexWithPartition) { // BTree index with partitioned table. Each partition's data is sorted by f0 independently. auto schema = arrow::schema(fields_); From 7149439e679cadf9485c07e850aa4ed6244668b2 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Wed, 22 Jul 2026 09:10:08 +0800 Subject: [PATCH 109/138] feat(variant): support variant data type --- docs/source/user_guide/data_types.rst | 41 + include/paimon/data/variant.h | 200 ++++ include/paimon/defs.h | 23 + src/paimon/CMakeLists.txt | 29 + .../map_shared_shredding_batch_converter.h | 7 +- ...ap_shared_shredding_write_plan_factory.cpp | 72 ++ .../map_shared_shredding_write_plan_factory.h | 70 ++ .../shredding/shredding_batch_converter.h | 47 + .../data/shredding/shredding_file_reader.cpp | 135 +++ .../data/shredding/shredding_file_reader.h | 68 ++ .../data/shredding/shredding_read_plan.h | 53 + .../shredding_write_plan_factories.cpp | 48 + .../shredding_write_plan_factories.h | 50 + .../shredding/shredding_write_plan_factory.h | 70 ++ .../common/data/variant/generic_variant.cpp | 427 +++++++ .../common/data/variant/generic_variant.h | 147 +++ .../data/variant/generic_variant_test.cpp | 351 ++++++ .../infer_variant_shredding_schema.cpp | 359 ++++++ .../variant/infer_variant_shredding_schema.h | 71 ++ .../infer_variant_shredding_schema_test.cpp | 215 ++++ src/paimon/common/data/variant/variant.cpp | 190 +++ .../data/variant/variant_access_utils.cpp | 181 +++ .../data/variant/variant_access_utils.h | 80 ++ .../data/variant/variant_binary_util.cpp | 599 ++++++++++ .../common/data/variant/variant_binary_util.h | 203 ++++ .../common/data/variant/variant_builder.cpp | 654 ++++++++++ .../common/data/variant/variant_builder.h | 147 +++ src/paimon/common/data/variant/variant_defs.h | 164 +++ .../common/data/variant/variant_get.cpp | 422 +++++++ src/paimon/common/data/variant/variant_get.h | 88 ++ .../common/data/variant/variant_get_test.cpp | 338 ++++++ .../data/variant/variant_json_utils.cpp | 309 +++++ .../common/data/variant/variant_json_utils.h | 62 + .../data/variant/variant_json_utils_test.cpp | 75 ++ .../data/variant/variant_path_segment.cpp | 118 ++ .../data/variant/variant_path_segment.h | 60 + .../data/variant/variant_reassembler.cpp | 273 +++++ .../common/data/variant/variant_reassembler.h | 73 ++ .../common/data/variant/variant_schema.h | 98 ++ .../variant_shredding_batch_converter.cpp | 188 +++ .../variant_shredding_batch_converter.h | 78 ++ .../variant_shredding_read_plan_factory.cpp | 598 ++++++++++ .../variant_shredding_read_plan_factory.h | 54 + .../data/variant/variant_shredding_test.cpp | 245 ++++ .../data/variant/variant_shredding_utils.cpp | 312 +++++ .../data/variant/variant_shredding_utils.h | 64 + .../variant/variant_shredding_write_plan.cpp | 147 +++ .../variant/variant_shredding_write_plan.h | 104 ++ .../variant_shredding_write_plan_factory.cpp | 202 ++++ .../variant_shredding_write_plan_factory.h | 92 ++ ...iant_shredding_write_plan_factory_test.cpp | 278 +++++ .../data/variant/variant_shredding_writer.cpp | 473 ++++++++ .../data/variant/variant_shredding_writer.h | 106 ++ .../common/data/variant/variant_test.cpp | 110 ++ .../data/variant/variant_type_utils.cpp | 128 ++ .../common/data/variant/variant_type_utils.h | 78 ++ .../data/variant/variant_type_utils_test.cpp | 105 ++ src/paimon/common/defs.cpp | 9 + src/paimon/common/types/array_type.h | 4 +- src/paimon/common/types/data_type.cpp | 17 +- .../common/types/data_type_json_parser.cpp | 54 +- .../types/data_type_json_parser_test.cpp | 12 + src/paimon/common/types/data_type_test.cpp | 50 + src/paimon/common/types/map_type.h | 6 +- src/paimon/common/utils/field_type_utils.h | 12 + src/paimon/core/append/append_only_writer.cpp | 6 +- src/paimon/core/core_options.cpp | 82 ++ src/paimon/core/core_options.h | 9 + src/paimon/core/core_options_test.cpp | 50 + .../core/io/infer_shredding_file_writer.h | 178 +++ .../io/infer_shredding_file_writer_test.cpp | 214 ++++ src/paimon/core/io/rolling_file_writer.h | 4 +- ...edding_append_data_file_writer_factory.cpp | 47 +- ...hredding_append_data_file_writer_factory.h | 12 +- ...ing_key_value_data_file_writer_factory.cpp | 50 +- ...dding_key_value_data_file_writer_factory.h | 12 +- src/paimon/core/io/single_file_writer.h | 6 +- .../compact/merge_tree_compact_rewriter.cpp | 6 +- .../core/mergetree/merge_tree_writer.cpp | 8 +- .../core/operation/abstract_split_read.cpp | 34 + .../core/operation/abstract_split_read.h | 7 + .../append_only_file_store_write.cpp | 6 +- .../operation/data_evolution_split_read.h | 3 +- .../core/operation/internal_read_context.cpp | 10 + .../core/operation/merge_file_split_read.h | 3 +- .../core/operation/raw_file_split_read.h | 3 +- .../core/postpone/postpone_bucket_writer.cpp | 8 +- .../core/schema/arrow_schema_validator.cpp | 17 + .../schema/arrow_schema_validator_test.cpp | 44 + src/paimon/core/schema/schema_validation.cpp | 7 + src/paimon/core/schema/table_schema.cpp | 8 +- src/paimon/core/schema/table_schema_test.cpp | 15 + .../core/utils/nested_projection_utils.cpp | 54 + .../utils/nested_projection_utils_test.cpp | 63 + .../format/avro/avro_schema_converter.cpp | 4 + .../avro/avro_schema_converter_test.cpp | 8 + src/paimon/format/orc/orc_format_writer.cpp | 4 + .../format/orc/orc_format_writer_test.cpp | 18 + src/paimon/format/parquet/CMakeLists.txt | 1 + .../format/parquet/variant_parquet_test.cpp | 1053 +++++++++++++++++ src/paimon/testing/utils/test_helper.h | 35 + src/paimon/testing/utils/variant_test_data.h | 79 ++ test/inte/CMakeLists.txt | 7 + test/inte/variant_table_inte_test.cpp | 484 ++++++++ 104 files changed, 12720 insertions(+), 82 deletions(-) create mode 100644 include/paimon/data/variant.h create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp create mode 100644 src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h create mode 100644 src/paimon/common/data/shredding/shredding_batch_converter.h create mode 100644 src/paimon/common/data/shredding/shredding_file_reader.cpp create mode 100644 src/paimon/common/data/shredding/shredding_file_reader.h create mode 100644 src/paimon/common/data/shredding/shredding_read_plan.h create mode 100644 src/paimon/common/data/shredding/shredding_write_plan_factories.cpp create mode 100644 src/paimon/common/data/shredding/shredding_write_plan_factories.h create mode 100644 src/paimon/common/data/shredding/shredding_write_plan_factory.h create mode 100644 src/paimon/common/data/variant/generic_variant.cpp create mode 100644 src/paimon/common/data/variant/generic_variant.h create mode 100644 src/paimon/common/data/variant/generic_variant_test.cpp create mode 100644 src/paimon/common/data/variant/infer_variant_shredding_schema.cpp create mode 100644 src/paimon/common/data/variant/infer_variant_shredding_schema.h create mode 100644 src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp create mode 100644 src/paimon/common/data/variant/variant.cpp create mode 100644 src/paimon/common/data/variant/variant_access_utils.cpp create mode 100644 src/paimon/common/data/variant/variant_access_utils.h create mode 100644 src/paimon/common/data/variant/variant_binary_util.cpp create mode 100644 src/paimon/common/data/variant/variant_binary_util.h create mode 100644 src/paimon/common/data/variant/variant_builder.cpp create mode 100644 src/paimon/common/data/variant/variant_builder.h create mode 100644 src/paimon/common/data/variant/variant_defs.h create mode 100644 src/paimon/common/data/variant/variant_get.cpp create mode 100644 src/paimon/common/data/variant/variant_get.h create mode 100644 src/paimon/common/data/variant/variant_get_test.cpp create mode 100644 src/paimon/common/data/variant/variant_json_utils.cpp create mode 100644 src/paimon/common/data/variant/variant_json_utils.h create mode 100644 src/paimon/common/data/variant/variant_json_utils_test.cpp create mode 100644 src/paimon/common/data/variant/variant_path_segment.cpp create mode 100644 src/paimon/common/data/variant/variant_path_segment.h create mode 100644 src/paimon/common/data/variant/variant_reassembler.cpp create mode 100644 src/paimon/common/data/variant/variant_reassembler.h create mode 100644 src/paimon/common/data/variant/variant_schema.h create mode 100644 src/paimon/common/data/variant/variant_shredding_batch_converter.cpp create mode 100644 src/paimon/common/data/variant/variant_shredding_batch_converter.h create mode 100644 src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp create mode 100644 src/paimon/common/data/variant/variant_shredding_read_plan_factory.h create mode 100644 src/paimon/common/data/variant/variant_shredding_test.cpp create mode 100644 src/paimon/common/data/variant/variant_shredding_utils.cpp create mode 100644 src/paimon/common/data/variant/variant_shredding_utils.h create mode 100644 src/paimon/common/data/variant/variant_shredding_write_plan.cpp create mode 100644 src/paimon/common/data/variant/variant_shredding_write_plan.h create mode 100644 src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp create mode 100644 src/paimon/common/data/variant/variant_shredding_write_plan_factory.h create mode 100644 src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp create mode 100644 src/paimon/common/data/variant/variant_shredding_writer.cpp create mode 100644 src/paimon/common/data/variant/variant_shredding_writer.h create mode 100644 src/paimon/common/data/variant/variant_test.cpp create mode 100644 src/paimon/common/data/variant/variant_type_utils.cpp create mode 100644 src/paimon/common/data/variant/variant_type_utils.h create mode 100644 src/paimon/common/data/variant/variant_type_utils_test.cpp create mode 100644 src/paimon/core/io/infer_shredding_file_writer.h create mode 100644 src/paimon/core/io/infer_shredding_file_writer_test.cpp create mode 100644 src/paimon/format/parquet/variant_parquet_test.cpp create mode 100644 src/paimon/testing/utils/variant_test_data.h create mode 100644 test/inte/variant_table_inte_test.cpp diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index 0c60e41e..60597dee 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -224,3 +224,44 @@ and `Arrow DataTypes `` where n is the unique name of a field, t is the logical type of a field, d is the description of a field. + + * - ``VARIANT`` + - Struct + - Data type of semi-structured data (e.g. JSON). A variant value contains one of: + a primitive (e.g. integer, string), an array of variant values, or an object + mapping string keys to variant values. + + Variant values are encoded with two binaries following the parquet-format + `Variant Binary Encoding `_ + specification (compatible with the Java Paimon / Spark implementation). In + C++ Paimon, a variant field is represented in an Arrow schema as + ``Struct{value: Binary NOT NULL, metadata: Binary NOT NULL}`` marked with + Paimon-specific field metadata; use ``paimon::Variant::ArrowField`` to + construct such a field, and ``paimon::Variant`` (``FromJson``/``ToJson``/ + ``VariantGet``) to build and inspect values. + + Only the parquet file format supports VARIANT columns. VARIANT cannot be + used as a primary key, partition key or bucket key, and no predicate + pushdown applies to it. + + When writing, variant columns can optionally be *shredded* into typed + parquet columns per the parquet-format + `Variant Shredding `_ + specification by setting ``variant.shreddingSchema`` to a ROW type JSON + whose fields map top-level variant column names to their shredding + types. Alternatively, setting ``variant.inferShreddingSchema`` to + ``true`` infers a shredding schema per file from the first written rows + (tuned by ``variant.shredding.maxSchemaWidth``, which bounds the total + number of shredded fields across all variant columns of the schema, + ``variant.shredding.maxSchemaDepth``, + ``variant.shredding.minFieldCardinalityRatio`` and + ``variant.shredding.maxInferBufferRow``). Inference also covers variant + columns nested inside ROW columns (variants inside arrays or maps stay + unshredded, as in Java Paimon). Readers reassemble shredded columns + transparently. + + When reading, instead of the full variant, specific paths can be + extracted by replacing the variant column in the read schema with a + projection built by ``paimon::VariantAccessBuilder`` (e.g. ``$.a.b`` as + BIGINT). For shredded files only the required typed sub-columns are + read. diff --git a/include/paimon/data/variant.h b/include/paimon/data/variant.h new file mode 100644 index 00000000..1958d09d --- /dev/null +++ b/include/paimon/data/variant.h @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" +#include "paimon/visibility.h" + +struct ArrowArray; +struct ArrowSchema; + +namespace paimon { + +/// Arguments controlling how a variant value is cast to a target type in `Variant::VariantGet`. +struct PAIMON_EXPORT VariantCastArgs { + /// Whether an invalid cast fails the call (true) or yields SQL NULL (false). + bool fail_on_error = true; + /// The time zone used when rendering TIMESTAMP values. Supported forms are `UTC`/`Z`/`GMT`, + /// fixed offsets such as `+08:00`, and IANA region ids such as `Asia/Shanghai`. + std::string zone_id = "UTC"; +}; + +/// A Variant represents a type that contains one of: 1) Primitive: A type and corresponding +/// value (e.g. INT, STRING); 2) Array: An ordered list of Variant values; 3) Object: An +/// unordered collection of string/Variant pairs (i.e. key/value pairs). An object may not +/// contain duplicate keys. +/// +/// A Variant is encoded with 2 binaries: the value and the metadata, following the parquet +/// Variant Binary Encoding specification (compatible with the Java / Spark implementation). The +/// encoding allows representation of semi-structured data (e.g. JSON) in a form that can be +/// efficiently queried by path. +/// +/// In an Arrow schema, a Variant field is represented as +/// `struct` marked with Paimon-specific field +/// metadata; use `Variant::ArrowField` to construct such a field. +class PAIMON_EXPORT Variant { + public: + ~Variant(); + + /// Parses a JSON string as a Variant (duplicate object keys are rejected). + /// + /// @param json The JSON document text. + /// @param pool The memory pool used for the variant buffers. + /// @return A result containing the created variant or an error. + static Result> FromJson(const std::string& json, + const std::shared_ptr& pool); + + /// Creates a Variant from already-encoded value and metadata binaries (copied into `pool`). + /// + /// @param value The variant value binary. + /// @param value_length The length of the value binary. + /// @param metadata The variant metadata binary. + /// @param metadata_length The length of the metadata binary. + /// @param pool The memory pool used for the variant buffers. + /// @return A result containing the created variant or an error. + static Result> Create(const char* value, uint64_t value_length, + const char* metadata, uint64_t metadata_length, + const std::shared_ptr& pool); + + /// The variant value binary. The view remains valid as long as this variant exists. + std::string_view Value() const; + + /// The variant metadata binary. The view remains valid as long as this variant exists. + std::string_view Metadata() const; + + /// The size of the variant in bytes (value size + metadata size). + int64_t SizeInBytes() const; + + /// Stringifies the variant in JSON format. + /// + /// @param zone_id The time zone used when rendering TIMESTAMP values. + /// @return A result containing the JSON text or an error. + Result ToJson(const std::string& zone_id = "UTC") const; + + /// Extracts a sub-variant value according to a path which starts with a `$`, e.g. `$.key`, + /// `$['key']`, `$["key"]`, `$.array[0]`, and casts the value to the target type. + /// + /// @param path The extraction path. + /// @param target_type The target Arrow type (C data interface, consumed by the call); only + /// scalar types are supported. Use `VariantGetArrow` for nested targets. + /// @param cast_args Cast behavior arguments. + /// @return A result containing the extracted literal, or nullopt for SQL NULL (unmatched + /// path, variant null, or an invalid cast with `fail_on_error == false`). + Result> VariantGet(const std::string& path, + struct ArrowSchema* target_type, + const VariantCastArgs& cast_args) const; + + /// Extracts a sub-variant value according to a path which starts with a `$` and casts the + /// value to the (possibly nested) target field type. + /// + /// In addition to scalar types, the target may be a STRUCT (cast from a variant object, + /// children matched by field name; unmatched children are null), a MAP with string keys + /// (cast from a variant object), a LIST (cast from a variant array), or a variant-marked + /// field created by `Variant::ArrowField` (the sub-variant is deeply re-encoded). + /// + /// @param path The extraction path. + /// @param target_field The target Arrow field (C data interface, consumed by the call). + /// @param cast_args Cast behavior arguments. + /// @return A result containing a length-1 Arrow array of the target type whose single slot + /// holds the result; a null slot represents SQL NULL (unmatched path, variant null, + /// or an invalid cast with `fail_on_error == false`). + Result> VariantGetArrow( + const std::string& path, struct ArrowSchema* target_field, + const VariantCastArgs& cast_args) const; + + /// Extracts a sub-variant value according to a path which starts with a `$` and renders it + /// as JSON text. + /// + /// @param path The extraction path. + /// @param zone_id The time zone used when rendering TIMESTAMP values. + /// @return A result containing the JSON text, or nullopt if the path does not match. + Result> VariantGetJson(const std::string& path, + const std::string& zone_id = "UTC") const; + + /// Creates an Arrow field definition for the Variant type. + /// + /// This function constructs an Arrow Field (internally + /// `struct`) and exports it to the C data + /// interface structure `::ArrowSchema`. It automatically injects Paimon-specific metadata to + /// identify the field as a VARIANT. + /// + /// @param field_name The name of the Arrow field. + /// @param nullable Whether the field is nullable. + /// @param metadata A map of key-value metadata to be attached to the field. + /// @return A result containing a unique pointer to the generated `::ArrowSchema` or an error. + static Result> ArrowField( + const std::string& field_name, bool nullable = true, + std::unordered_map metadata = {}); + + private: + class Impl; + + explicit Variant(std::unique_ptr&& impl); + + std::unique_ptr impl_; +}; + +/// Builds a variant-access projection field: a struct field that replaces a VARIANT column in +/// the read schema so that, instead of the full variant, only the described paths are extracted +/// at read time (reading only the required shredded sub-columns from shredded files). +/// +/// Example: read `$.age` as INT64 and `$.city` as STRING from variant column `v`: +/// +/// VariantAccessBuilder builder; +/// builder.AddField(age_type, "$.age"); +/// builder.AddField(city_type, "$.city"); +/// auto field = builder.Build("v"); // use in ReadContextBuilder::SetReadSchema +/// +/// The resulting struct has one child per added field, named by its position ("0", "1", ...). +class PAIMON_EXPORT VariantAccessBuilder { + public: + VariantAccessBuilder(); + ~VariantAccessBuilder(); + + /// Adds an extracted field. + /// + /// @param target_type The Arrow type the extracted value is cast to (C data interface, + /// consumed by the call). Nested targets follow `Variant::VariantGetArrow` semantics. + /// @param path The extraction path, e.g. `$.a.b` or `$.array[0]`. + /// @param fail_on_error Whether an invalid cast fails the read (true) or yields SQL NULL. + /// @param zone_id The time zone used when rendering TIMESTAMP values. + Status AddField(struct ArrowSchema* target_type, const std::string& path, + bool fail_on_error = true, const std::string& zone_id = "UTC"); + + /// Builds the projection field named `field_name`, to be used in the read schema in place + /// of the variant column with the same name. + Result> Build(const std::string& field_name) const; + + private: + class Impl; + + std::unique_ptr impl_; +}; + +} // namespace paimon diff --git a/include/paimon/defs.h b/include/paimon/defs.h index ddc157ba..2f137539 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -49,6 +49,7 @@ enum class FieldType { MAP = 14, STRUCT = 15, BLOB = 16, + VARIANT = 17, UNKNOWN = 128, }; @@ -418,6 +419,28 @@ struct PAIMON_EXPORT Options { /// Only effective when map.storage-layout = shared-shredding. static const char MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY[]; + /// "variant.shreddingSchema" - The Variant shredding schema for writing: a ROW type JSON + /// whose fields map variant column names to their shredding types. No default value. + static const char VARIANT_SHREDDING_SCHEMA[]; + /// "parquet.variant.shreddingSchema" - Fallback key of "variant.shreddingSchema". + static const char PARQUET_VARIANT_SHREDDING_SCHEMA[]; + /// "variant.inferShreddingSchema" - Whether to automatically infer the shredding schema when + /// writing Variant columns. Default value is "false". + static const char VARIANT_INFER_SHREDDING_SCHEMA[]; + /// "variant.shredding.maxSchemaWidth" - Maximum number of shredded fields allowed in an + /// inferred schema. Default value is 300. + static const char VARIANT_SHREDDING_MAX_SCHEMA_WIDTH[]; + /// "variant.shredding.maxSchemaDepth" - Maximum traversal depth in Variant values during + /// schema inference. Default value is 50. + static const char VARIANT_SHREDDING_MAX_SCHEMA_DEPTH[]; + /// "variant.shredding.minFieldCardinalityRatio" - Minimum fraction of rows that must contain + /// a field for it to be shredded. Fields below this threshold stay in the un-shredded + /// Variant binary. Default value is 0.1. + static const char VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO[]; + /// "variant.shredding.maxInferBufferRow" - Maximum number of rows to buffer for schema + /// inference. Default value is 4096. + static const char VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW[]; + /// "blob-as-descriptor" - Read blob field using blob descriptor rather than blob /// bytes. Default value is "false". static const char BLOB_AS_DESCRIPTOR[]; diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 3250babf..0551cabc 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -40,6 +40,23 @@ set(PAIMON_COMMON_SRCS common/data/serializer/row_compacted_serializer.cpp common/data/serializer/binary_serializer_utils.cpp common/data/timestamp.cpp + common/data/variant/generic_variant.cpp + common/data/variant/infer_variant_shredding_schema.cpp + common/data/variant/variant.cpp + common/data/variant/variant_binary_util.cpp + common/data/variant/variant_builder.cpp + common/data/variant/variant_get.cpp + common/data/variant/variant_json_utils.cpp + common/data/variant/variant_path_segment.cpp + common/data/variant/variant_reassembler.cpp + common/data/variant/variant_shredding_batch_converter.cpp + common/data/variant/variant_access_utils.cpp + common/data/variant/variant_shredding_read_plan_factory.cpp + common/data/variant/variant_shredding_utils.cpp + common/data/variant/variant_shredding_write_plan.cpp + common/data/variant/variant_shredding_write_plan_factory.cpp + common/data/variant/variant_shredding_writer.cpp + common/data/variant/variant_type_utils.cpp common/defs.cpp common/executor/executor.cpp common/factories/singleton.cpp @@ -142,11 +159,14 @@ set(PAIMON_COMMON_SRCS common/utils/decimal_utils.cpp common/data/shredding/map_shared_shredding_utils.cpp common/data/shredding/map_shared_shredding_schema_utils.cpp + common/data/shredding/map_shared_shredding_write_plan_factory.cpp + common/data/shredding/shredding_write_plan_factories.cpp common/data/shredding/map_shared_shredding_context.cpp common/data/shredding/map_shared_shredding_batch_converter.cpp common/data/shredding/map_shared_shredding_column_allocator.cpp common/data/shredding/lru_map_shared_shredding_column_allocator.cpp common/data/shredding/map_shared_shredding_file_reader.cpp + common/data/shredding/shredding_file_reader.cpp common/utils/delta_varint_compressor.cpp common/utils/fields_comparator.cpp common/utils/path_util.cpp @@ -443,6 +463,14 @@ if(PAIMON_BUILD_TESTS) common/data/blob_descriptor_test.cpp common/data/blob_view_struct_test.cpp common/data/blob_utils_test.cpp + common/data/variant/generic_variant_test.cpp + common/data/variant/infer_variant_shredding_schema_test.cpp + common/data/variant/variant_shredding_write_plan_factory_test.cpp + common/data/variant/variant_get_test.cpp + common/data/variant/variant_json_utils_test.cpp + common/data/variant/variant_shredding_test.cpp + common/data/variant/variant_test.cpp + common/data/variant/variant_type_utils_test.cpp common/executor/default_executor_test.cpp common/format/column_stats_test.cpp common/fs/external_path_provider_test.cpp @@ -636,6 +664,7 @@ if(PAIMON_BUILD_TESTS) core/index/index_file_meta_serializer_test.cpp core/index/index_file_handler_test.cpp core/io/compact_increment_test.cpp + core/io/infer_shredding_file_writer_test.cpp core/io/concat_key_value_record_reader_test.cpp core/io/data_file_meta_serializer_test.cpp core/io/data_file_path_factory_test.cpp diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h index ea49f2ec..8bea8120 100644 --- a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h +++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h @@ -30,6 +30,7 @@ #include "paimon/common/data/shredding/map_shared_shredding_column_allocator.h" #include "paimon/common/data/shredding/map_shared_shredding_field_dict.h" #include "paimon/common/data/shredding/map_shredding_defs.h" +#include "paimon/common/data/shredding/shredding_batch_converter.h" #include "paimon/memory/memory_pool.h" #include "paimon/result.h" #include "paimon/status.h" @@ -47,7 +48,7 @@ class MapSharedShreddingContext; /// /// Non-shared-shredding columns are passed through unchanged. /// Each shared-shredding column has its own FieldDict and ColumnAllocator. -class MapSharedShreddingBatchConverter { +class MapSharedShreddingBatchConverter : public ShreddingBatchConverter { public: /// Creates a converter for one file write cycle. /// Computes per-file K from context, builds physical schema, and constructs the converter. @@ -62,12 +63,12 @@ class MapSharedShreddingBatchConverter { const std::shared_ptr& pool); /// Returns the physical schema produced for this converter. - const std::shared_ptr& GetPhysicalSchema() const; + const std::shared_ptr& GetPhysicalSchema() const override; /// Converts a logical batch to a physical batch. /// @param logical_batch Input ArrowArray (C ABI) with logical schema. Consumed on success. /// @return Owned physical ArrowArray (C ABI) with physical schema. - Result> Convert(ArrowArray* logical_batch); + Result> Convert(ArrowArray* logical_batch) override; /// Builds MapSharedShreddingFieldMeta for one shredding column (by field name). /// Called at file close to serialize metadata. diff --git a/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp new file mode 100644 index 00000000..3ee94e0f --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h" + +#include + +#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" +#include "paimon/common/data/shredding/map_shared_shredding_context.h" +#include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/map_shredding_defs.h" + +namespace paimon { + +MapSharedShreddingWritePlanFactory::MapSharedShreddingWritePlanFactory( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& context, + const std::shared_ptr& pool) + : options_(options), write_schema_(write_schema), context_(context), pool_(pool) {} + +bool MapSharedShreddingWritePlanFactory::ShouldCreateWritePlan() const { + return context_ != nullptr; +} + +bool MapSharedShreddingWritePlanFactory::ShouldInferWritePlan() const { + return false; +} + +int32_t MapSharedShreddingWritePlanFactory::InferBufferRowCount() const { + return 0; +} + +Result> +MapSharedShreddingWritePlanFactory::CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) const { + if (context_ == nullptr) { + return Status::Invalid("Shared-shredding write plan requires a shredding context."); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr converter, + MapSharedShreddingBatchConverter::Create(write_schema_, context_, options_, pool_)); + return std::shared_ptr(std::move(converter)); +} + +ShreddingWritePlanFactory::MetadataFinalizer +MapSharedShreddingWritePlanFactory::CreateMetadataFinalizer( + const std::shared_ptr& converter) const { + // The converter is created by CreateConverter above; the concrete type is guaranteed. + auto map_converter = std::static_pointer_cast(converter); + return MapSharedShreddingUtils::BuildMetadataFinalizer( + map_converter, MapSharedShreddingDefine::kDefaultDictCompression, context_, + map_converter->GetPhysicalSchema()); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h new file mode 100644 index 00000000..f4911fa8 --- /dev/null +++ b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" +#include "paimon/core/core_options.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class Schema; +} // namespace arrow + +namespace paimon { + +class MapSharedShreddingContext; + +/// Creates MAP shared-shredding batch converters driven by the cross-file adaptive-K context. +/// The write plan is never inferred from samples; per-file field metadata is persisted into the +/// file footer by the metadata finalizer. +class MapSharedShreddingWritePlanFactory : public ShreddingWritePlanFactory { + public: + MapSharedShreddingWritePlanFactory(const CoreOptions& options, + const std::shared_ptr& write_schema, + const std::shared_ptr& context, + const std::shared_ptr& pool); + + bool ShouldCreateWritePlan() const override; + + bool ShouldInferWritePlan() const override; + + int32_t InferBufferRowCount() const override; + + Result> CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) const override; + + MetadataFinalizer CreateMetadataFinalizer( + const std::shared_ptr& converter) const override; + + private: + CoreOptions options_; + std::shared_ptr write_schema_; + std::shared_ptr context_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_batch_converter.h b/src/paimon/common/data/shredding/shredding_batch_converter.h new file mode 100644 index 00000000..736ea4e2 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_batch_converter.h @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "arrow/c/abi.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +/// Converts logical write batches into a physical (shredded) file layout. Implemented by both +/// the VARIANT shredding and the MAP shared-shredding batch converters so that shredded file +/// writers can be composed generically. +class ShreddingBatchConverter { + public: + virtual ~ShreddingBatchConverter() = default; + + /// The physical file schema that converted batches conform to. + virtual const std::shared_ptr& GetPhysicalSchema() const = 0; + + /// Converts a logical batch into the physical layout. Consumes `logical_batch`. + virtual Result> Convert(::ArrowArray* logical_batch) = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_file_reader.cpp b/src/paimon/common/data/shredding/shredding_file_reader.cpp new file mode 100644 index 00000000..cb0eaa28 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_file_reader.cpp @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/shredding_file_reader.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +ShreddingFileReader::ShreddingFileReader( + std::unique_ptr&& reader, + std::map>&& plans, + const std::shared_ptr& pool) + : arrow_pool_(GetArrowPool(pool)), reader_(std::move(reader)), plans_(std::move(plans)) {} + +Result> ShreddingFileReader::GetFileSchema() const { + return reader_->GetFileSchema(); +} + +Status ShreddingFileReader::SetReadSchema(::ArrowSchema* read_schema, + const std::shared_ptr& predicate, + const std::optional& selection_bitmap) { + if (!read_schema) { + return Status::Invalid("invalid read schema in ShreddingFileReader, cannot be null"); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_read_schema, + arrow::ImportSchema(read_schema)); + arrow::FieldVector resolved_fields = logical_read_schema->fields(); + bool any_resolved = false; + for (auto& resolved_field : resolved_fields) { + auto it = plans_.find(resolved_field->name()); + if (it == plans_.end()) { + continue; + } + // Push the physical (possibly pruned) subtree down so the inner reader materializes it. + resolved_field = it->second->PhysicalField(); + any_resolved = true; + } + if (!any_resolved) { + return Status::Invalid("no planned shredded columns exist in the read schema"); + } + auto resolved_schema = arrow::schema(resolved_fields, logical_read_schema->metadata()); + auto c_resolved_schema = std::make_unique<::ArrowSchema>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*resolved_schema, c_resolved_schema.get())); + return reader_->SetReadSchema(c_resolved_schema.get(), predicate, selection_bitmap); +} + +Result ShreddingFileReader::NextBatch() { + return Status::Invalid( + "paimon inner reader ShreddingFileReader should use NextBatchWithBitmap"); +} + +Result ShreddingFileReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap batch_with_bitmap, + reader_->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + return batch_with_bitmap; + } + + auto& [batch, bitmap] = batch_with_bitmap; + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + if (arrow_array->type_id() != arrow::Type::STRUCT) { + return Status::Invalid("cannot cast batch to StructArray in ShreddingFileReader"); + } + auto struct_array = std::static_pointer_cast(arrow_array); + + arrow::ArrayVector resolved_arrays = struct_array->fields(); + arrow::FieldVector resolved_fields = struct_array->struct_type()->fields(); + for (int32_t field_idx = 0; field_idx < struct_array->num_fields(); ++field_idx) { + const auto& physical_field = struct_array->struct_type()->field(field_idx); + auto it = plans_.find(physical_field->name()); + if (it == plans_.end()) { + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr logical_array, + it->second->Assemble(struct_array->field(field_idx), arrow_pool_.get())); + resolved_arrays[field_idx] = logical_array; + resolved_fields[field_idx] = it->second->LogicalField(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr new_struct_array, + arrow::StructArray::Make(resolved_arrays, resolved_fields)); + auto new_c_array = std::make_unique(); + auto new_c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*new_struct_array, new_c_array.get(), new_c_schema.get())); + batch = std::make_pair(std::move(new_c_array), std::move(new_c_schema)); + return batch_with_bitmap; +} + +std::shared_ptr ShreddingFileReader::GetReaderMetrics() const { + return reader_->GetReaderMetrics(); +} + +void ShreddingFileReader::Close() { + reader_->Close(); +} + +Result ShreddingFileReader::GetPreviousBatchFileRowId(uint64_t batch_row_id) const { + return reader_->GetPreviousBatchFileRowId(batch_row_id); +} + +Result ShreddingFileReader::GetNumberOfRows() const { + return reader_->GetNumberOfRows(); +} + +bool ShreddingFileReader::SupportPreciseBitmapSelection() const { + return reader_->SupportPreciseBitmapSelection(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_file_reader.h b/src/paimon/common/data/shredding/shredding_file_reader.h new file mode 100644 index 00000000..f6ab3575 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_file_reader.h @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/common/data/shredding/shredding_read_plan.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/file_batch_reader.h" + +namespace paimon { + +/// A file batch reader wrapper that pushes per-column physical (shredded, possibly pruned) +/// subtrees down to the inner reader and assembles the physical batches back into the logical +/// columns as planned by `ShreddingColumnReadPlan`s. +class ShreddingFileReader : public FileBatchReader { + public: + ShreddingFileReader(std::unique_ptr&& reader, + std::map>&& plans, + const std::shared_ptr& pool); + + Result> GetFileSchema() const override; + + Status SetReadSchema(::ArrowSchema* read_schema, const std::shared_ptr& predicate, + const std::optional& selection_bitmap) override; + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + std::shared_ptr GetReaderMetrics() const override; + + void Close() override; + + Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override; + + Result GetNumberOfRows() const override; + + bool SupportPreciseBitmapSelection() const override; + + private: + std::shared_ptr arrow_pool_; + std::unique_ptr reader_; + std::map> plans_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_read_plan.h b/src/paimon/common/data/shredding/shredding_read_plan.h new file mode 100644 index 00000000..a9338f0b --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_read_plan.h @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class Field; +class MemoryPool; +} // namespace arrow + +namespace paimon { + +/// A per-column read plan translating between a column's logical shape and the physical +/// (shredded, possibly pruned) shape stored in one file: the physical field is pushed down to +/// the format reader and the physical batches are assembled back into logical arrays. +class ShreddingColumnReadPlan { + public: + virtual ~ShreddingColumnReadPlan() = default; + + /// The logical field restored on the assembled output. + virtual const std::shared_ptr& LogicalField() const = 0; + + /// The physical file field (possibly a pruned subtree) to read from the file. + virtual const std::shared_ptr& PhysicalField() const = 0; + + /// Assembles the physical column array back into the logical column array. + virtual Result> Assemble( + const std::shared_ptr& physical, arrow::MemoryPool* pool) const = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp b/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp new file mode 100644 index 00000000..b7992871 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" + +#include "paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h" +#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h" +#include "paimon/core/core_options.h" + +namespace paimon { + +std::shared_ptr ShreddingWritePlanFactories::SelectActive( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& shredding_context, + const std::shared_ptr& pool) { + // MAP shared-shredding is active exactly when a context exists; constructing its factory + // copies the options, so skip it otherwise. + if (shredding_context != nullptr) { + auto map_factory = std::make_shared( + options, write_schema, shredding_context, pool); + if (map_factory->ShouldCreateWritePlan()) { + return map_factory; + } + } + auto variant_factory = VariantShreddingWritePlanFactory::Create(options, write_schema, pool); + if (variant_factory->ShouldCreateWritePlan()) { + return variant_factory; + } + return std::shared_ptr(nullptr); +} + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_write_plan_factories.h b/src/paimon/common/data/shredding/shredding_write_plan_factories.h new file mode 100644 index 00000000..cfad5f97 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_write_plan_factories.h @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class CoreOptions; +class MapSharedShreddingContext; + +/// Composes the known shredding write-plan factories (MAP shared-shredding and VARIANT +/// shredding) and selects the one active for a write schema. +class ShreddingWritePlanFactories { + public: + /// Returns the single active write-plan factory for the write, or nullptr when no shredding + /// applies. MAP shared-shredding takes precedence over VARIANT shredding, preserving the + /// selection order of the writer call sites. + static std::shared_ptr SelectActive( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& shredding_context, + const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/shredding/shredding_write_plan_factory.h b/src/paimon/common/data/shredding/shredding_write_plan_factory.h new file mode 100644 index 00000000..601e5114 --- /dev/null +++ b/src/paimon/common/data/shredding/shredding_write_plan_factory.h @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/shredding/shredding_batch_converter.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class Schema; +} // namespace arrow + +namespace paimon { + +/// Decides whether write batches must be rewritten into a physical (shredded) layout before +/// they reach the format writer, and creates the per-file batch converter -- either immediately +/// from configuration or, for inference, from sampled logical batches buffered by the writer. +class ShreddingWritePlanFactory { + public: + /// Finalizes per-file shredding metadata; the returned schema (or nullptr) is persisted into + /// the file footer right before the file is finished. + using MetadataFinalizer = std::function>()>; + + virtual ~ShreddingWritePlanFactory() = default; + + /// Whether a write plan (immediate or inferred) applies to the write schema. + virtual bool ShouldCreateWritePlan() const = 0; + + /// Whether the write plan must be inferred from sampled rows instead of configuration. + virtual bool ShouldInferWritePlan() const = 0; + + /// The number of rows buffered per file to sample the inferred write plan from. + virtual int32_t InferBufferRowCount() const = 0; + + /// Creates the per-file batch converter. `sample_batches` holds the logical batches sampled + /// for inference and is empty when the plan comes from configuration. Returns nullptr when + /// no conversion is useful for this file (the file is written with the logical schema). + virtual Result> CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) const = 0; + + /// The per-file metadata finalizer persisted into the file footer, or nullptr when the + /// physical schema is self-describing (as it is for VARIANT shredding). + virtual MetadataFinalizer CreateMetadataFinalizer( + const std::shared_ptr& converter) const = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/generic_variant.cpp b/src/paimon/common/data/variant/generic_variant.cpp new file mode 100644 index 00000000..34b74c0b --- /dev/null +++ b/src/paimon/common/data/variant/generic_variant.cpp @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/generic_variant.h" + +#include +#include +#include + +#include "arrow/util/base64.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_json_utils.h" + +namespace paimon { + +namespace { + +Status ToJsonImpl(std::string_view value, std::string_view metadata, int32_t pos, + const std::string& zone_id, std::string* out) { + PAIMON_ASSIGN_OR_RAISE(VariantValueType type, VariantBinaryUtil::GetType(value, pos)); + switch (type) { + case VariantValueType::kObject: { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(value, pos)); + out->push_back('{'); + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t id, + VariantBinaryUtil::ReadUnsigned( + value, info.id_start + info.id_size * i, info.id_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(value, info.offset_start + info.offset_size * i, + info.offset_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t element_pos, + VariantBinaryUtil::CheckedElementPos(info.data_start, offset, value.size())); + if (i != 0) { + out->push_back(','); + } + PAIMON_ASSIGN_OR_RAISE(std::string_view key, + VariantBinaryUtil::GetMetadataKey(metadata, id)); + VariantJsonUtils::AppendEscapedJson(key, out); + out->push_back(':'); + PAIMON_RETURN_NOT_OK(ToJsonImpl(value, metadata, element_pos, zone_id, out)); + } + out->push_back('}'); + return Status::OK(); + } + case VariantValueType::kArray: { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ArrayInfo info, + VariantBinaryUtil::GetArrayInfo(value, pos)); + out->push_back('['); + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(value, info.offset_start + info.offset_size * i, + info.offset_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t element_pos, + VariantBinaryUtil::CheckedElementPos(info.data_start, offset, value.size())); + if (i != 0) { + out->push_back(','); + } + PAIMON_RETURN_NOT_OK(ToJsonImpl(value, metadata, element_pos, zone_id, out)); + } + out->push_back(']'); + return Status::OK(); + } + case VariantValueType::kNull: + out->append("null"); + return Status::OK(); + case VariantValueType::kBoolean: { + PAIMON_ASSIGN_OR_RAISE(bool b, VariantBinaryUtil::GetBoolean(value, pos)); + out->append(b ? "true" : "false"); + return Status::OK(); + } + case VariantValueType::kLong: { + PAIMON_ASSIGN_OR_RAISE(int64_t l, VariantBinaryUtil::GetLong(value, pos)); + out->append(std::to_string(l)); + return Status::OK(); + } + case VariantValueType::kString: { + PAIMON_ASSIGN_OR_RAISE(std::string_view s, VariantBinaryUtil::GetString(value, pos)); + VariantJsonUtils::AppendEscapedJson(s, out); + return Status::OK(); + } + case VariantValueType::kDouble: { + PAIMON_ASSIGN_OR_RAISE(double d, VariantBinaryUtil::GetDouble(value, pos)); + std::string repr = VariantJsonUtils::JavaDoubleToString(d); + if (std::isfinite(d)) { + out->append(repr); + } else { + out->push_back('"'); + out->append(repr); + out->push_back('"'); + } + return Status::OK(); + } + case VariantValueType::kDecimal: { + PAIMON_ASSIGN_OR_RAISE(VariantDecimal d, VariantBinaryUtil::GetDecimal(value, pos)); + out->append(d.ToPlainString()); + return Status::OK(); + } + case VariantValueType::kDate: { + PAIMON_ASSIGN_OR_RAISE(int64_t days, VariantBinaryUtil::GetLong(value, pos)); + out->push_back('"'); + out->append(VariantJsonUtils::DateToString(static_cast(days))); + out->push_back('"'); + return Status::OK(); + } + case VariantValueType::kTimestamp: { + PAIMON_ASSIGN_OR_RAISE(int64_t micros, VariantBinaryUtil::GetLong(value, pos)); + PAIMON_ASSIGN_OR_RAISE(int32_t offset_seconds, + VariantJsonUtils::GetZoneOffsetSeconds(zone_id, micros)); + out->push_back('"'); + out->append(VariantJsonUtils::TimestampToString(micros, offset_seconds, true)); + out->push_back('"'); + return Status::OK(); + } + case VariantValueType::kTimestampNtz: { + PAIMON_ASSIGN_OR_RAISE(int64_t micros, VariantBinaryUtil::GetLong(value, pos)); + out->push_back('"'); + out->append(VariantJsonUtils::TimestampToString(micros, 0, false)); + out->push_back('"'); + return Status::OK(); + } + case VariantValueType::kFloat: { + PAIMON_ASSIGN_OR_RAISE(float f, VariantBinaryUtil::GetFloat(value, pos)); + std::string repr = VariantJsonUtils::JavaFloatToString(f); + if (std::isfinite(f)) { + out->append(repr); + } else { + out->push_back('"'); + out->append(repr); + out->push_back('"'); + } + return Status::OK(); + } + case VariantValueType::kBinary: { + PAIMON_ASSIGN_OR_RAISE(std::string_view binary, + VariantBinaryUtil::GetBinary(value, pos)); + out->push_back('"'); + out->append(arrow::util::base64_encode(binary)); + out->push_back('"'); + return Status::OK(); + } + case VariantValueType::kUuid: { + PAIMON_ASSIGN_OR_RAISE(std::string_view uuid, VariantBinaryUtil::GetUuid(value, pos)); + out->push_back('"'); + out->append(VariantBinaryUtil::UuidToString(uuid)); + out->push_back('"'); + return Status::OK(); + } + } + return VariantBinaryUtil::MalformedVariant("unknown variant value type in JSON rendering"); +} + +} // namespace + +GenericVariant::GenericVariant(std::shared_ptr value, std::shared_ptr metadata, + int32_t pos) + : value_(std::move(value)), metadata_(std::move(metadata)), pos_(pos) {} + +Result> GenericVariant::Create(std::shared_ptr value, + std::shared_ptr metadata) { + if (!value || !metadata) { + return Status::Invalid("variant value and metadata must not be null"); + } + // There is currently only one allowed version. + if (metadata->size() < 1 || (static_cast((*metadata)[0]) & + VariantDefs::kVersionMask) != VariantDefs::kVersion) { + return VariantBinaryUtil::MalformedVariant("unsupported variant metadata version"); + } + // Don't attempt to use a Variant larger than 128 MiB. We'll never produce one, and it risks + // memory instability. + if (metadata->size() > static_cast(VariantDefs::kSizeLimit) || + value->size() > static_cast(VariantDefs::kSizeLimit)) { + return VariantBinaryUtil::VariantConstructorSizeLimit(); + } + return std::shared_ptr( + new GenericVariant(std::move(value), std::move(metadata), 0)); +} + +Result> GenericVariant::Create( + std::string_view value, std::string_view metadata, const std::shared_ptr& pool) { + // Reject over-limit inputs before allocating and copying them. + if (value.size() > static_cast(VariantDefs::kSizeLimit) || + metadata.size() > static_cast(VariantDefs::kSizeLimit)) { + return VariantBinaryUtil::VariantConstructorSizeLimit(); + } + std::shared_ptr value_bytes = Bytes::AllocateBytes(value.size(), pool.get()); + if (!value.empty()) { + std::memcpy(value_bytes->data(), value.data(), value.size()); + } + std::shared_ptr metadata_bytes = Bytes::AllocateBytes(metadata.size(), pool.get()); + if (!metadata.empty()) { + // An empty (malformed) metadata view may carry a null data pointer; the size check + // keeps memcpy away from it. + std::memcpy(metadata_bytes->data(), metadata.data(), metadata.size()); + } + return Create(std::move(value_bytes), std::move(metadata_bytes)); +} + +Result> GenericVariant::FromJson( + std::string_view json, const std::shared_ptr& pool) { + return VariantBuilder::ParseJson(json, /*allow_duplicate_keys=*/false, pool); +} + +Result GenericVariant::Value() const { + std::string_view raw = RawValue(); + if (pos_ == 0) { + return raw; + } + PAIMON_ASSIGN_OR_RAISE(int32_t size, VariantBinaryUtil::ValueSize(raw, pos_)); + PAIMON_RETURN_NOT_OK( + VariantBinaryUtil::CheckIndex(pos_ + size - 1, static_cast(raw.size()))); + return raw.substr(pos_, size); +} + +std::string_view GenericVariant::RawValue() const { + return {value_->data(), value_->size()}; +} + +std::string_view GenericVariant::Metadata() const { + return {metadata_->data(), metadata_->size()}; +} + +int64_t GenericVariant::SizeInBytes() const { + return static_cast(value_->size()) + static_cast(metadata_->size()); +} + +Result GenericVariant::ToJson(const std::string& zone_id) const { + std::string result; + PAIMON_RETURN_NOT_OK(ToJsonImpl(RawValue(), Metadata(), pos_, zone_id, &result)); + return result; +} + +Result GenericVariant::GetType() const { + return VariantBinaryUtil::GetType(RawValue(), pos_); +} + +Result GenericVariant::GetTypeInfo() const { + return VariantBinaryUtil::GetTypeInfo(RawValue(), pos_); +} + +Result GenericVariant::GetBoolean() const { + return VariantBinaryUtil::GetBoolean(RawValue(), pos_); +} + +Result GenericVariant::GetLong() const { + return VariantBinaryUtil::GetLong(RawValue(), pos_); +} + +Result GenericVariant::GetDouble() const { + return VariantBinaryUtil::GetDouble(RawValue(), pos_); +} + +Result GenericVariant::GetDecimal() const { + return VariantBinaryUtil::GetDecimal(RawValue(), pos_); +} + +Result GenericVariant::GetFloat() const { + return VariantBinaryUtil::GetFloat(RawValue(), pos_); +} + +Result GenericVariant::GetBinary() const { + return VariantBinaryUtil::GetBinary(RawValue(), pos_); +} + +Result GenericVariant::GetString() const { + return VariantBinaryUtil::GetString(RawValue(), pos_); +} + +Result GenericVariant::GetUuid() const { + return VariantBinaryUtil::GetUuid(RawValue(), pos_); +} + +Result GenericVariant::ObjectSize() const { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(RawValue(), pos_)); + return info.num_elements; +} + +std::shared_ptr GenericVariant::SubVariant(int32_t pos) const { + return std::shared_ptr(new GenericVariant(value_, metadata_, pos)); +} + +Result> GenericVariant::GetFieldByKey(std::string_view key) const { + std::string_view raw = RawValue(); + std::string_view metadata = Metadata(); + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(raw, pos_)); + // Use linear search for a short list. Switch to binary search when the length reaches + // `kBinarySearchThreshold`. + if (info.num_elements < VariantDefs::kBinarySearchThreshold) { + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE( + int32_t id, VariantBinaryUtil::ReadUnsigned(raw, info.id_start + info.id_size * i, + info.id_size)); + PAIMON_ASSIGN_OR_RAISE(std::string_view field_key, + VariantBinaryUtil::GetMetadataKey(metadata, id)); + if (field_key == key) { + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(raw, info.offset_start + info.offset_size * i, + info.offset_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t element_pos, + VariantBinaryUtil::CheckedElementPos(info.data_start, offset, raw.size())); + return SubVariant(element_pos); + } + } + } else { + int32_t low = 0; + int32_t high = info.num_elements - 1; + while (low <= high) { + // Use an unsigned shift to compute the middle of `low` and `high`, which properly + // handles the case where `low + high` overflows int32. + auto mid = static_cast( + (static_cast(low) + static_cast(high)) >> 1); + PAIMON_ASSIGN_OR_RAISE( + int32_t id, VariantBinaryUtil::ReadUnsigned(raw, info.id_start + info.id_size * mid, + info.id_size)); + PAIMON_ASSIGN_OR_RAISE(std::string_view field_key, + VariantBinaryUtil::GetMetadataKey(metadata, id)); + int32_t cmp = field_key.compare(key); + if (cmp < 0) { + low = mid + 1; + } else if (cmp > 0) { + high = mid - 1; + } else { + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(raw, info.offset_start + info.offset_size * mid, + info.offset_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t element_pos, + VariantBinaryUtil::CheckedElementPos(info.data_start, offset, raw.size())); + return SubVariant(element_pos); + } + } + } + return std::shared_ptr(nullptr); +} + +Result> GenericVariant::GetFieldAtIndex( + int32_t index) const { + std::string_view raw = RawValue(); + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(raw, pos_)); + if (index < 0 || index >= info.num_elements) { + return std::optional(std::nullopt); + } + PAIMON_ASSIGN_OR_RAISE( + int32_t id, + VariantBinaryUtil::ReadUnsigned(raw, info.id_start + info.id_size * index, info.id_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, VariantBinaryUtil::ReadUnsigned( + raw, info.offset_start + info.offset_size * index, info.offset_size)); + PAIMON_ASSIGN_OR_RAISE(std::string_view key, VariantBinaryUtil::GetMetadataKey(Metadata(), id)); + ObjectField field; + field.key = std::string(key); + PAIMON_ASSIGN_OR_RAISE(int32_t field_pos, VariantBinaryUtil::CheckedElementPos( + info.data_start, offset, raw.size())); + field.value = SubVariant(field_pos); + return std::optional(std::move(field)); +} + +Result GenericVariant::GetDictionaryIdAtIndex(int32_t index) const { + std::string_view raw = RawValue(); + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(raw, pos_)); + if (index < 0 || index >= info.num_elements) { + return VariantBinaryUtil::MalformedVariant(fmt::format( + "object field index {} is out of bounds for {} fields", index, info.num_elements)); + } + return VariantBinaryUtil::ReadUnsigned(raw, info.id_start + info.id_size * index, info.id_size); +} + +Result GenericVariant::ArraySize() const { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ArrayInfo info, + VariantBinaryUtil::GetArrayInfo(RawValue(), pos_)); + return info.num_elements; +} + +Result> GenericVariant::GetElementAtIndex(int32_t index) const { + std::string_view raw = RawValue(); + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ArrayInfo info, + VariantBinaryUtil::GetArrayInfo(raw, pos_)); + if (index < 0 || index >= info.num_elements) { + return std::shared_ptr(nullptr); + } + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, VariantBinaryUtil::ReadUnsigned( + raw, info.offset_start + info.offset_size * index, info.offset_size)); + PAIMON_ASSIGN_OR_RAISE(int32_t element_pos, VariantBinaryUtil::CheckedElementPos( + info.data_start, offset, raw.size())); + return SubVariant(element_pos); +} + +bool GenericVariant::operator==(const GenericVariant& other) const { + return pos_ == other.pos_ && RawValue() == other.RawValue() && Metadata() == other.Metadata(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/generic_variant.h b/src/paimon/common/data/variant/generic_variant.h new file mode 100644 index 00000000..f95d12f0 --- /dev/null +++ b/src/paimon/common/data/variant/generic_variant.h @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { + +/// A Variant represents a type that contains one of: 1) Primitive: A type and corresponding value +/// (e.g. INT, STRING); 2) Array: An ordered list of Variant values; 3) Object: An unordered +/// collection of string/Variant pairs (i.e. key/value pairs). An object may not contain duplicate +/// keys. +/// +/// A Variant is encoded with 2 binaries: the value and the metadata. The Variant Binary Encoding +/// allows representation of semi-structured data (e.g. JSON) in a form that can be efficiently +/// queried by path. The design is intended to allow efficient access to nested data even in the +/// presence of very wide or deep structures. +class GenericVariant { + public: + /// A field of a variant object. + struct ObjectField { + std::string key; + std::shared_ptr value; + }; + + /// Creates a variant taking ownership of the given buffers. Fails with `MALFORMED_VARIANT` + /// if the metadata version is unsupported, or `VARIANT_CONSTRUCTOR_SIZE_LIMIT` if either + /// buffer exceeds the 128MiB size limit. + static Result> Create(std::shared_ptr value, + std::shared_ptr metadata); + + /// Creates a variant by copying the given buffers into `pool`. + static Result> Create(std::string_view value, + std::string_view metadata, + const std::shared_ptr& pool); + + /// Parses a JSON string as a variant (duplicate object keys are rejected). + static Result> FromJson( + std::string_view json, const std::shared_ptr& pool); + + /// The variant value binary. For a sub-variant (`Pos() != 0`), the view covers only the + /// sub-variant slice of the underlying buffer. + Result Value() const; + + /// The whole underlying value buffer, regardless of `Pos()`. + std::string_view RawValue() const; + + /// The variant metadata binary. + std::string_view Metadata() const; + + /// The variant value doesn't use the whole value binary, but starts from the `Pos()` index + /// and spans a size of `VariantBinaryUtil::ValueSize`. This design avoids frequent copies of + /// the value binary when reading a sub-variant in an array/object element. + int32_t Pos() const { + return pos_; + } + + /// The size of the variant in bytes (value size + metadata size). + int64_t SizeInBytes() const; + + /// Stringifies the variant in JSON format. `zone_id` controls the rendering of TIMESTAMP + /// values; supported forms are "UTC"/"Z"/"GMT" and fixed offsets such as "+08:00". + Result ToJson(const std::string& zone_id = "UTC") const; + + /// The value type of the variant. + Result GetType() const; + + /// The type info bits of the variant value header. + Result GetTypeInfo() const; + + Result GetBoolean() const; + Result GetLong() const; + Result GetDouble() const; + Result GetDecimal() const; + Result GetFloat() const; + Result GetBinary() const; + Result GetString() const; + /// The 16-byte big-endian UUID value. + Result GetUuid() const; + + /// The number of object fields in the variant. It is only legal to call it when `GetType()` + /// is `kObject`. + Result ObjectSize() const; + + /// Finds the field value whose key is equal to `key`. Returns nullptr if the key is not + /// found. It is only legal to call it when `GetType()` is `kObject`. The returned sub-variant + /// shares the underlying buffers with this variant. + Result> GetFieldByKey(std::string_view key) const; + + /// Gets the object field at the `index` slot. Returns nullopt if `index` is out of the bound + /// of `[0, ObjectSize())`. It is only legal to call it when `GetType()` is `kObject`. + Result> GetFieldAtIndex(int32_t index) const; + + /// Gets the metadata dictionary id for the object field at the `index` slot. It is only + /// legal to call it when `GetType()` is `kObject`. + Result GetDictionaryIdAtIndex(int32_t index) const; + + /// The number of array elements in the variant. It is only legal to call it when `GetType()` + /// is `kArray`. + Result ArraySize() const; + + /// Gets the array element at the `index` slot. Returns nullptr if `index` is out of the + /// bound of `[0, ArraySize())`. It is only legal to call it when `GetType()` is `kArray`. + Result> GetElementAtIndex(int32_t index) const; + + bool operator==(const GenericVariant& other) const; + + private: + GenericVariant(std::shared_ptr value, std::shared_ptr metadata, int32_t pos); + + std::shared_ptr SubVariant(int32_t pos) const; + + std::shared_ptr value_; + std::shared_ptr metadata_; + int32_t pos_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/generic_variant_test.cpp b/src/paimon/common/data/variant/generic_variant_test.cpp new file mode 100644 index 00000000..d628b3e7 --- /dev/null +++ b/src/paimon/common/data/variant/generic_variant_test.cpp @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/generic_variant.h" + +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class GenericVariantTest : public ::testing::Test { + public: + static std::string ToHex(std::string_view data) { + constexpr char kHexDigits[] = "0123456789abcdef"; + std::string result; + result.reserve(data.size() * 2); + for (char c : data) { + auto byte = static_cast(c); + result.push_back(kHexDigits[byte >> 4]); + result.push_back(kHexDigits[byte & 0xF]); + } + return result; + } + + std::shared_ptr FromJson(const std::string& json) { + auto result = GenericVariant::FromJson(json, pool_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); + } + + // Asserts that parsing `json` produces exactly the value/metadata binaries produced by the + // Java implementation (`GenericVariantBuilder`), and that rendering back to JSON matches the + // Java `toJson` output. + void CheckGolden(const std::string& json, const std::string& expected_value_hex, + const std::string& expected_metadata_hex, + const std::string& expected_to_json) { + auto variant = FromJson(json); + ASSERT_OK_AND_ASSIGN(std::string_view value, variant->Value()); + ASSERT_EQ(ToHex(value), expected_value_hex) << "value bytes mismatch for: " << json; + ASSERT_EQ(ToHex(variant->Metadata()), expected_metadata_hex) + << "metadata bytes mismatch for: " << json; + ASSERT_OK_AND_ASSIGN(std::string to_json, variant->ToJson()); + ASSERT_EQ(to_json, expected_to_json); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +// Golden binaries generated by the Java implementation (org.apache.paimon.data.variant); these +// pin the cross-implementation byte compatibility of the variant encoding. +TEST_F(GenericVariantTest, GoldenPrimitives) { + CheckGolden("null", "00", "010000", "null"); + CheckGolden("true", "04", "010000", "true"); + CheckGolden("false", "08", "010000", "false"); + CheckGolden("1", "0c01", "010000", "1"); + CheckGolden("-1", "0cff", "010000", "-1"); + CheckGolden("300", "102c01", "010000", "300"); + CheckGolden("100000", "14a0860100", "010000", "100000"); + CheckGolden("12345678901234", "18f22fce733a0b0000", "010000", "12345678901234"); + CheckGolden("1e40", "1ca55cc3f129633d48", "010000", "1.0E40"); + CheckGolden("1.0123456789012345678901234567890123456789", "1c240bf2619132f03f", "010000", + "1.0123456789012346"); + CheckGolden("2.5e-3", "1c7b14ae47e17a643f", "010000", "0.0025"); + CheckGolden("100.99", "200273270000", "010000", "100.99"); + CheckGolden("-0.5", "2001fbffffff", "010000", "-0.5"); + CheckGolden("0.0", "200100000000", "010000", "0"); + CheckGolden("12345678.90123", "2405cb04fb711f010000", "010000", "12345678.90123"); + CheckGolden("1234567890123456789.0123456789", "280a1581396eb1c9be46321be42700000000", "010000", + "1234567890123456789.0123456789"); + // An integer that overflows int64 is parsed as an exact decimal. + CheckGolden("123456789012345678901234567890", "2800d20a3f4eeee073c3f60fe98e01000000", "010000", + "123456789012345678901234567890"); +} + +TEST_F(GenericVariantTest, GoldenStrings) { + CheckGolden("\"Hello, World!\"", "3548656c6c6f2c20576f726c6421", "010000", "\"Hello, World!\""); + CheckGolden("\"\"", "01", "010000", "\"\""); + CheckGolden( + "\"This is a long string that definitely exceeds the sixty-three byte short string limit " + "...!\"", + "405a000000546869732069732061206c6f6e6720737472696e67207468617420646566696e6974656c792065" + "786365656473207468652073697874792d746872656520627974652073686f727420737472696e67206c696d" + "6974202e2e2e21", + "010000", + "\"This is a long string that definitely exceeds the sixty-three byte short string limit " + "...!\""); +} + +TEST_F(GenericVariantTest, GoldenContainers) { + CheckGolden("{}", "020000", "010000", "{}"); + CheckGolden("[]", "030000", "010000", "[]"); + CheckGolden(R"({"a": 1, "b": "hello"})", "020200010002080c011568656c6c6f", "01020001026162", + R"({"a":1,"b":"hello"})"); + CheckGolden(R"([1, "two", 3.5, null, true, {"k":[]}])", + "03060002060c0d0e160c010d74776f20012300000000040201000003030000", "010100016b", + R"([1,"two",3.5,null,true,{"k":[]}])"); +} + +TEST_F(GenericVariantTest, GoldenNested) { + CheckGolden( + "{\"object\":{\"name\":\"Apache Paimon\",\"age\":2,\"address\":{\"street\":\"Main " + "St\",\"city\":\"Hangzhou\"}},\"array\":[1,2,3,4,5],\"string\":\"Hello, " + "World!\",\"long\":12345678901234,\"double\":1." + "0123456789012345678901234567890123456789,\"decimal\":100.99,\"boolean1\":true," + "\"boolean2\":false,\"nullField\":null}", + "0209060b0c0a09080d000731696a635a516b00436c0203030201100e0028354170616368652050" + "61696d6f6e0c02020205040800111d4d61696e2053742148616e677a686f75030500020406080a" + "0c010c020c030c040c053548656c6c6f2c20576f726c642118f22fce733a0b00001c240bf26191" + "32f03f200273270000040800", + "010e00060a0d141a1e23292d333a424a536f626a6563746e616d6561676561646472657373737472656574" + "636974796172726179737472696e676c6f6e67646f75626c65646563696d616c626f6f6c65616e31626f6f" + "6c65616e326e756c6c4669656c64", + "{\"array\":[1,2,3,4,5],\"boolean1\":true,\"boolean2\":false,\"decimal\":100.99," + "\"double\":1.0123456789012346,\"long\":12345678901234,\"nullField\":null,\"object\":{" + "\"address\":{\"city\":\"Hangzhou\",\"street\":\"Main St\"},\"age\":2,\"name\":\"Apache " + "Paimon\"},\"string\":\"Hello, World!\"}"); +} + +TEST_F(GenericVariantTest, GoldenUnicodeEscape) { + CheckGolden(R"({"\u4e2d\u6587": "\u4f60\u597d\n\t\"quoted\""})", + "020100001141e4bda0e5a5bd0a092271756f74656422", "01010006e4b8ade69687", + "{\"中文\":\"你好\\n\\t\\\"quoted\\\"\"}"); +} + +TEST_F(GenericVariantTest, TypedAccessors) { + auto variant = FromJson(R"({"a": 1, "b": "hello"})"); + ASSERT_OK_AND_ASSIGN(VariantValueType type, variant->GetType()); + ASSERT_EQ(type, VariantValueType::kObject); + ASSERT_OK_AND_ASSIGN(int32_t object_size, variant->ObjectSize()); + ASSERT_EQ(object_size, 2); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr a, variant->GetFieldByKey("a")); + ASSERT_NE(a, nullptr); + ASSERT_OK_AND_ASSIGN(int64_t a_value, a->GetLong()); + ASSERT_EQ(a_value, 1); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr b, variant->GetFieldByKey("b")); + ASSERT_NE(b, nullptr); + ASSERT_OK_AND_ASSIGN(std::string_view b_value, b->GetString()); + ASSERT_EQ(b_value, "hello"); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing, variant->GetFieldByKey("c")); + ASSERT_EQ(missing, nullptr); + + ASSERT_OK_AND_ASSIGN(auto field0, variant->GetFieldAtIndex(0)); + ASSERT_TRUE(field0.has_value()); + ASSERT_EQ(field0->key, "a"); + ASSERT_OK_AND_ASSIGN(auto field_oob, variant->GetFieldAtIndex(2)); + ASSERT_FALSE(field_oob.has_value()); + + auto array = FromJson("[10, 20, 30]"); + ASSERT_OK_AND_ASSIGN(int32_t array_size, array->ArraySize()); + ASSERT_EQ(array_size, 3); + ASSERT_OK_AND_ASSIGN(std::shared_ptr elem, array->GetElementAtIndex(1)); + ASSERT_NE(elem, nullptr); + ASSERT_OK_AND_ASSIGN(int64_t elem_value, elem->GetLong()); + ASSERT_EQ(elem_value, 20); + ASSERT_OK_AND_ASSIGN(std::shared_ptr elem_oob, array->GetElementAtIndex(3)); + ASSERT_EQ(elem_oob, nullptr); +} + +TEST_F(GenericVariantTest, ObjectBinarySearch) { + // More fields than kBinarySearchThreshold exercises the binary-search lookup. + std::string json = "{"; + for (int32_t i = 0; i < 40; ++i) { + if (i != 0) { + json += ","; + } + json += "\"key" + std::to_string(i) + "\":" + std::to_string(i); + } + json += "}"; + auto variant = FromJson(json); + for (int32_t i = 0; i < 40; ++i) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr field, + variant->GetFieldByKey("key" + std::to_string(i))); + ASSERT_NE(field, nullptr); + ASSERT_OK_AND_ASSIGN(int64_t value, field->GetLong()); + ASSERT_EQ(value, i); + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing, variant->GetFieldByKey("key40")); + ASSERT_EQ(missing, nullptr); +} + +TEST_F(GenericVariantTest, DuplicateKeys) { + ASSERT_NOK(GenericVariant::FromJson("{\"a\": 1, \"a\": 2}", pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variant, + VariantBuilder::ParseJson("{\"a\": 1, \"a\": 2}", /*allow_duplicate_keys=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(std::string to_json, variant->ToJson()); + ASSERT_EQ(to_json, "{\"a\":2}"); +} + +TEST_F(GenericVariantTest, CorruptedLayoutRejected) { + // Headers claiming a near-INT32_MAX element count must be rejected by the 64-bit layout + // bound instead of overflowing the 32-bit offset arithmetic. + std::string metadata; + metadata.push_back(static_cast(0x01)); + metadata.push_back(static_cast(0x00)); + { + // Object, large size, 4-byte ids and offsets, num_elements = INT32_MAX. + std::string value; + value.push_back(static_cast((0x1F << 2) | 0x02)); + value.append({static_cast(0xFF), static_cast(0xFF), static_cast(0xFF), + static_cast(0x7F)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value, metadata, pool_)); + ASSERT_NOK(variant->ToJson()); + } + { + // Array, large size, 4-byte offsets, num_elements = INT32_MAX. + std::string value; + value.push_back(static_cast((0x07 << 2) | 0x03)); + value.append({static_cast(0xFF), static_cast(0xFF), static_cast(0xFF), + static_cast(0x7F)}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value, metadata, pool_)); + ASSERT_NOK(variant->ToJson()); + } +} + +TEST_F(GenericVariantTest, OverLimitInputRejectedBeforeAllocation) { + // The size check must run before the buffers are allocated/copied; the fake-length view is + // never dereferenced. + char byte = 0; + std::string_view huge(&byte, static_cast(VariantDefs::kSizeLimit) + 1); + ASSERT_NOK(GenericVariant::Create(huge, std::string_view(&byte, 1), pool_)); + ASSERT_NOK(GenericVariant::Create(std::string_view(&byte, 1), huge, pool_)); +} + +TEST_F(GenericVariantTest, MalformedInput) { + ASSERT_NOK(GenericVariant::FromJson("", pool_)); + ASSERT_NOK(GenericVariant::FromJson("{", pool_)); + ASSERT_NOK(GenericVariant::FromJson("{\"a\":}", pool_)); + ASSERT_NOK(GenericVariant::FromJson("NaN", pool_)); + + // Unsupported metadata version. + std::string bad_metadata = std::string("\x02\x00\x00", 3); + std::string value = std::string("\x00", 1); + ASSERT_NOK(GenericVariant::Create(value, bad_metadata, pool_)); + // Empty metadata. + ASSERT_NOK(GenericVariant::Create(value, std::string(), pool_)); +} + +TEST_F(GenericVariantTest, SizeInBytesAndViews) { + auto variant = FromJson(R"({"a": 1, "b": "hello"})"); + ASSERT_EQ(variant->SizeInBytes(), + static_cast(variant->RawValue().size() + variant->Metadata().size())); + // A sub-variant shares buffers with its parent and reports a positive position. + ASSERT_OK_AND_ASSIGN(std::shared_ptr b, variant->GetFieldByKey("b")); + ASSERT_GT(b->Pos(), 0); + ASSERT_OK_AND_ASSIGN(std::string_view b_slice, b->Value()); + ASSERT_EQ(ToHex(b_slice), "1568656c6c6f"); +} + +TEST_F(GenericVariantTest, AppendVariantRebuild) { + // Rebuilding a sub-variant through a fresh builder produces a self-contained variant. + auto variant = FromJson(R"({"outer": {"x": [1, 2], "y": "z"}})"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr outer, variant->GetFieldByKey("outer")); + VariantBuilder builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(builder.AppendVariant(*outer)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr rebuilt, builder.Build(pool_)); + ASSERT_EQ(rebuilt->Pos(), 0); + ASSERT_OK_AND_ASSIGN(std::string to_json, rebuilt->ToJson()); + ASSERT_EQ(to_json, "{\"x\":[1,2],\"y\":\"z\"}"); +} + +TEST_F(GenericVariantTest, TimestampAndSpecialTypesToJson) { + // JSON can't produce date/timestamp/binary/uuid variants; build them directly. + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendDate(19737)); // 2024-01-15 + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"2024-01-15\""); + } + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendTimestamp(1705312496123456LL)); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"2024-01-15 09:54:56.123456+00:00\""); + ASSERT_OK_AND_ASSIGN(std::string json_shanghai, v->ToJson("Asia/Shanghai")); + ASSERT_EQ(json_shanghai, "\"2024-01-15 17:54:56.123456+08:00\""); + ASSERT_OK_AND_ASSIGN(std::string json_offset, v->ToJson("+08:00")); + ASSERT_EQ(json_offset, "\"2024-01-15 17:54:56.123456+08:00\""); + ASSERT_NOK(v->ToJson("Not/AZone")); + } + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendTimestampNtz(1705312496000000LL)); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"2024-01-15 09:54:56\""); + } + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendBinary(std::string_view("\x01\x02\x03", 3))); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"AQID\""); + } + { + VariantBuilder builder(false); + std::string uuid_bytes = + std::string("\x12\x3e\x45\x67\xe8\x9b\x12\xd3\xa4\x56\x42\x66\x14\x17\x40\x00", 16); + ASSERT_OK(builder.AppendUuid(uuid_bytes)); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"123e4567-e89b-12d3-a456-426614174000\""); + } + { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendFloat(1.5f)); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "1.5"); + } +} + +TEST_F(GenericVariantTest, NonFiniteDoubleToJson) { + VariantBuilder builder(false); + ASSERT_OK(builder.AppendDouble(std::numeric_limits::infinity())); + ASSERT_OK_AND_ASSIGN(auto v, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::string json, v->ToJson()); + ASSERT_EQ(json, "\"Infinity\""); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp b/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp new file mode 100644 index 00000000..4f4bc967 --- /dev/null +++ b/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/infer_variant_shredding_schema.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/common/data/variant/variant_defs.h" + +namespace paimon { + +namespace { + +constexpr int32_t kMaxRowFieldSize = 1000; + +// The inference type lattice. A scalar node holds an arrow type (`arrow::null()` is the untyped +// VARIANT sentinel); object nodes track per-field occurrence counts so that rare fields can be +// dropped in the final schema. +struct SimpleSchema { + struct Field { + std::string name; + std::shared_ptr schema; + int64_t count; + }; + + bool is_object = false; + bool is_array = false; + std::vector fields; + std::shared_ptr element; + std::shared_ptr scalar; + + static std::shared_ptr Variant() { + auto schema = std::make_shared(); + schema->scalar = arrow::null(); + return schema; + } + + static std::shared_ptr Scalar(std::shared_ptr type) { + auto schema = std::make_shared(); + schema->scalar = std::move(type); + return schema; + } +}; + +std::shared_ptr MergeSchema(const std::shared_ptr& s1, + const std::shared_ptr& s2); + +// Merges two decimals with possibly different scales. +std::shared_ptr MergeDecimal(const arrow::Decimal128Type& d1, + const arrow::Decimal128Type& d2) { + int32_t scale = std::max(d1.scale(), d2.scale()); + int32_t range = std::max(d1.precision() - d1.scale(), d2.precision() - d2.scale()); + if (range + scale > VariantDefs::kMaxDecimal16Precision) { + // Decimal cannot support precision > 38. + return SimpleSchema::Variant(); + } + return SimpleSchema::Scalar(arrow::decimal128(range + scale, scale)); +} + +std::shared_ptr MergeDecimalWithLong(const arrow::Decimal128Type& d) { + if (d.scale() == 0 && d.precision() <= 18) { + return SimpleSchema::Scalar(arrow::int64()); + } + // A long can always fit in a decimal(19, 0). + auto long_decimal = std::static_pointer_cast(arrow::decimal128(19, 0)); + return MergeDecimal(d, *long_decimal); +} + +std::shared_ptr MergeObjects(const std::shared_ptr& s1, + const std::shared_ptr& s2) { + auto result = std::make_shared(); + result->is_object = true; + size_t f1_idx = 0; + size_t f2_idx = 0; + while (f1_idx < s1->fields.size() && f2_idx < s2->fields.size() && + result->fields.size() < kMaxRowFieldSize) { + const auto& field1 = s1->fields[f1_idx]; + const auto& field2 = s2->fields[f2_idx]; + int32_t comp = field1.name.compare(field2.name); + if (comp == 0) { + result->fields.push_back(SimpleSchema::Field{field1.name, + MergeSchema(field1.schema, field2.schema), + field1.count + field2.count}); + ++f1_idx; + ++f2_idx; + } else if (comp < 0) { + result->fields.push_back(field1); + ++f1_idx; + } else { + result->fields.push_back(field2); + ++f2_idx; + } + } + while (f1_idx < s1->fields.size() && result->fields.size() < kMaxRowFieldSize) { + result->fields.push_back(s1->fields[f1_idx++]); + } + while (f2_idx < s2->fields.size() && result->fields.size() < kMaxRowFieldSize) { + result->fields.push_back(s2->fields[f2_idx++]); + } + return result; +} + +std::shared_ptr MergeSchema(const std::shared_ptr& s1, + const std::shared_ptr& s2) { + // Allow null (missing) to merge into any typed schema. + if (s1 == nullptr) { + return s2; + } + if (s2 == nullptr) { + return s1; + } + if (s1->is_object && s2->is_object) { + return MergeObjects(s1, s2); + } + if (s1->is_array && s2->is_array) { + auto result = std::make_shared(); + result->is_array = true; + result->element = MergeSchema(s1->element, s2->element); + if (result->element == nullptr) { + result->element = SimpleSchema::Variant(); + } + return result; + } + if (s1->scalar != nullptr && s2->scalar != nullptr) { + bool s1_decimal = s1->scalar->id() == arrow::Type::DECIMAL128; + bool s2_decimal = s2->scalar->id() == arrow::Type::DECIMAL128; + bool s1_long = s1->scalar->id() == arrow::Type::INT64; + bool s2_long = s2->scalar->id() == arrow::Type::INT64; + if (s1_decimal && s2_decimal) { + return MergeDecimal(static_cast(*s1->scalar), + static_cast(*s2->scalar)); + } + if (s1_decimal && s2_long) { + return MergeDecimalWithLong(static_cast(*s1->scalar)); + } + if (s1_long && s2_decimal) { + return MergeDecimalWithLong(static_cast(*s2->scalar)); + } + if (s1->scalar->Equals(*s2->scalar)) { + return s1; + } + } + return SimpleSchema::Variant(); +} + +// Returns an appropriate schema for shredding a variant value. Unlike a generic schema-of +// expression, the merged types stay consistent with what shredding allows (e.g. an integer and a +// double merge to VARIANT, not double). +Result> SchemaOf(const GenericVariant& variant, int32_t max_depth) { + PAIMON_ASSIGN_OR_RAISE(VariantValueType type, variant.GetType()); + switch (type) { + case VariantValueType::kObject: { + if (max_depth <= 0) { + return SimpleSchema::Variant(); + } + PAIMON_ASSIGN_OR_RAISE(int32_t size, variant.ObjectSize()); + auto result = std::make_shared(); + result->is_object = true; + result->fields.reserve(size); + for (int32_t i = 0; i < size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::optional field, + variant.GetFieldAtIndex(i)); + if (!field.has_value()) { + return VariantBinaryUtil::MalformedVariant("an object field is missing"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr field_schema, + SchemaOf(*field->value, max_depth - 1)); + if (field_schema == nullptr) { + field_schema = SimpleSchema::Variant(); + } + result->fields.push_back(SimpleSchema::Field{field->key, field_schema, 1}); + } + // According to the variant spec, object fields must be sorted alphabetically. + for (size_t i = 1; i < result->fields.size(); ++i) { + if (result->fields[i - 1].name.compare(result->fields[i].name) >= 0) { + return Status::Invalid("Variant object fields must be sorted alphabetically"); + } + } + return result; + } + case VariantValueType::kArray: { + if (max_depth <= 0) { + return SimpleSchema::Variant(); + } + PAIMON_ASSIGN_OR_RAISE(int32_t size, variant.ArraySize()); + std::shared_ptr element_type; + for (int32_t i = 0; i < size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element, + variant.GetElementAtIndex(i)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element_schema, + SchemaOf(*element, max_depth - 1)); + element_type = MergeSchema(element_type, element_schema); + } + auto result = std::make_shared(); + result->is_array = true; + result->element = element_type == nullptr ? SimpleSchema::Variant() : element_type; + return result; + } + case VariantValueType::kNull: + return std::shared_ptr(nullptr); + case VariantValueType::kBoolean: + return SimpleSchema::Scalar(arrow::boolean()); + case VariantValueType::kLong: { + // Compute the smallest decimal that can contain this value. + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant.GetLong()); + VariantDecimal decimal{value, 0}; + int32_t precision = decimal.Precision(); + if (precision <= 18) { + return SimpleSchema::Scalar(arrow::decimal128(precision, 0)); + } + return SimpleSchema::Scalar(arrow::int64()); + } + case VariantValueType::kString: + return SimpleSchema::Scalar(arrow::utf8()); + case VariantValueType::kDouble: + return SimpleSchema::Scalar(arrow::float64()); + case VariantValueType::kDecimal: { + PAIMON_ASSIGN_OR_RAISE(VariantDecimal decimal, variant.GetDecimal()); + if (decimal.scale < 0) { + // GetDecimal strips trailing zeros and can return a negative scale (100.00 -> + // 1E+2), which neither the shredded parquet type nor reassembly's AppendDecimal + // can represent; scale the value back up. Bounded by construction: the encoded + // decimal had at most 38 digits before the point. + for (; decimal.scale < 0; ++decimal.scale) { + decimal.unscaled *= 10; + } + } + int32_t precision = decimal.Precision(); + int32_t scale = decimal.scale; + // Ensure precision is at least scale (and at least 1) to be valid. + if (precision < scale) { + precision = scale; + } + if (precision == 0) { + precision = 1; + } + return SimpleSchema::Scalar(arrow::decimal128(precision, scale)); + } + case VariantValueType::kDate: + case VariantValueType::kTimestamp: + case VariantValueType::kTimestampNtz: + // The shredding schema builder rejects temporal leaf types (as the Java one does), + // so inferring them would abort the whole write; keep such values in the untyped + // column instead. + return SimpleSchema::Scalar(arrow::null()); + case VariantValueType::kFloat: + return SimpleSchema::Scalar(arrow::float32()); + case VariantValueType::kBinary: + return SimpleSchema::Scalar(arrow::binary()); + default: + return SimpleSchema::Variant(); + } +} + +// Finalizes the inferred schema: 1) widen integer types to int64, 2) replace empty objects with +// VARIANT, 3) limit the total number of shredded fields in the schema. +std::shared_ptr FinalizeSimpleSchema( + const std::shared_ptr& schema, int64_t min_cardinality, + InferVariantShreddingSchema::MaxFields* max_fields) { + // Every field uses a value column. + --max_fields->remaining; + if (max_fields->remaining <= 0) { + return arrow::null(); + } + if (schema == nullptr || + (schema->scalar != nullptr && schema->scalar->id() == arrow::Type::NA)) { + return arrow::null(); + } + if (schema->is_object) { + arrow::FieldVector new_fields; + for (const auto& field : schema->fields) { + if (field.count >= min_cardinality && max_fields->remaining > 0) { + auto new_type = FinalizeSimpleSchema(field.schema, min_cardinality, max_fields); + new_fields.push_back(arrow::field(field.name, new_type)); + } + } + if (!new_fields.empty()) { + return arrow::struct_(new_fields); + } + return arrow::null(); + } + if (schema->is_array) { + auto new_element = FinalizeSimpleSchema(schema->element, min_cardinality, max_fields); + return arrow::list(new_element); + } + switch (schema->scalar->id()) { + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::INT64: + --max_fields->remaining; + return arrow::int64(); + case arrow::Type::DECIMAL128: { + const auto& decimal_type = static_cast(*schema->scalar); + --max_fields->remaining; + if (decimal_type.precision() <= 18 && decimal_type.scale() == 0) { + return arrow::int64(); + } + if (decimal_type.precision() <= 18) { + return arrow::decimal128(18, decimal_type.scale()); + } + return arrow::decimal128(VariantDefs::kMaxDecimal16Precision, decimal_type.scale()); + } + default: + // All other scalar types use typed_value. + --max_fields->remaining; + return schema->scalar; + } +} + +} // namespace + +Result> InferVariantShreddingSchema::InferColumnShreddingType( + const std::vector>& samples, MaxFields* max_fields) const { + int64_t num_non_null_values = 0; + std::shared_ptr simple_schema; + for (const auto& sample : samples) { + if (sample == nullptr) { + continue; + } + ++num_non_null_values; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_schema, + SchemaOf(*sample, max_schema_depth_)); + simple_schema = MergeSchema(simple_schema, row_schema); + } + // Don't infer a schema for fields that appear in less than min_field_cardinality_ratio of + // the rows. + auto min_cardinality = static_cast( + std::ceil(static_cast(num_non_null_values) * min_field_cardinality_ratio_)); + std::shared_ptr finalized = + FinalizeSimpleSchema(simple_schema, min_cardinality, max_fields); + if (finalized->id() == arrow::Type::NA) { + // The whole column stays unshredded. + return std::shared_ptr(nullptr); + } + return finalized; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema.h b/src/paimon/common/data/variant/infer_variant_shredding_schema.h new file mode 100644 index 00000000..7b8d3d1a --- /dev/null +++ b/src/paimon/common/data/variant/infer_variant_shredding_schema.h @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +} // namespace arrow + +namespace paimon { + +/// Infers a shredding type for a variant column from sampled values (mirroring the Java +/// `InferVariantShreddingSchema`). Rare fields (below the cardinality ratio) stay in the +/// un-shredded variant binary, integer types widen to int64, and the total number of shredded +/// fields is limited. +class InferVariantShreddingSchema { + public: + /// The mutable budget of shredded fields remaining. One instance is shared across all + /// variant columns of a schema so that the total inferred width stays within + /// `variant.shredding.maxSchemaWidth` (mirroring the Java `MaxFields`). + struct MaxFields { + int32_t remaining; + }; + + InferVariantShreddingSchema(int32_t max_schema_width, int32_t max_schema_depth, + double min_field_cardinality_ratio) + : max_schema_width_(max_schema_width), + max_schema_depth_(max_schema_depth), + min_field_cardinality_ratio_(min_field_cardinality_ratio) {} + + /// Creates the shared shredded-field budget for one schema inference. + MaxFields CreateMaxFieldsBudget() const { + return MaxFields{max_schema_width_}; + } + + /// Infers the shredding type of one variant column from its sampled non-null values, e.g. + /// `struct{a: int64, b: string}`. `arrow::null()` leaves denote untyped variant sub-values. + /// `max_fields` is the budget shared across all columns of the schema. Returns nullptr when + /// no useful shredding schema was found (the column should stay unshredded). + Result> InferColumnShreddingType( + const std::vector>& samples, MaxFields* max_fields) const; + + private: + int32_t max_schema_width_; + int32_t max_schema_depth_; + double min_field_cardinality_ratio_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp b/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp new file mode 100644 index 00000000..e473782f --- /dev/null +++ b/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/infer_variant_shredding_schema.h" + +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class InferVariantShreddingSchemaTest : public ::testing::Test { + public: + // Infers one column with a fresh shared-width budget. + static Result> InferColumn( + const InferVariantShreddingSchema& infer, + const std::vector>& samples) { + InferVariantShreddingSchema::MaxFields max_fields = infer.CreateMaxFieldsBudget(); + return infer.InferColumnShreddingType(samples, &max_fields); + } + + std::vector> Samples(const std::vector& jsons) { + std::vector> samples; + for (const char* json : jsons) { + if (json == nullptr) { + samples.push_back(nullptr); + continue; + } + auto variant = GenericVariant::FromJson(json, pool_); + EXPECT_TRUE(variant.ok()) << variant.status().ToString(); + samples.push_back(variant.value()); + } + return samples; + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + InferVariantShreddingSchema infer_{/*max_schema_width=*/300, /*max_schema_depth=*/50, + /*min_field_cardinality_ratio=*/0.1}; +}; + +TEST_F(InferVariantShreddingSchemaTest, InferObjectSchema) { + auto samples = Samples({ + R"({"age": 35, "city": "Hangzhou"})", + R"({"age": 20, "city": "Beijing", "tags": [1, 2]})", + nullptr, + R"({"age": 120000000000, "city": "Shanghai"})", + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + // Integers widen to int64, strings stay, arrays of small ints infer as list. + auto expected = + arrow::struct_({arrow::field("age", arrow::int64()), arrow::field("city", arrow::utf8()), + arrow::field("tags", arrow::list(arrow::int64()))}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); + ASSERT_OK(VariantShreddingUtils::VariantShreddingSchema(inferred)); +} + +TEST_F(InferVariantShreddingSchemaTest, MixedTypesFallToVariant) { + auto samples = Samples({ + R"({"x": 1, "y": 1.5e0})", + R"({"x": "string now", "y": 2.5e0})", + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + // x saw both int and string: untyped variant leaf; y stays double (exponent notation + // parses as double, plain decimals parse as DECIMAL). + auto expected = + arrow::struct_({arrow::field("x", arrow::null()), arrow::field("y", arrow::float64())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); + ASSERT_OK(VariantShreddingUtils::VariantShreddingSchema(inferred)); +} + +TEST_F(InferVariantShreddingSchemaTest, RareFieldsDropped) { + std::vector jsons; + for (int i = 0; i < 19; ++i) { + jsons.push_back("{\"common\": 1}"); + } + jsons.push_back(R"({"common": 2, "rare": true})"); + auto samples = Samples(jsons); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + // "rare" appears in 1/20 rows (< 0.1 ratio): dropped from the typed schema. + auto expected = arrow::struct_({arrow::field("common", arrow::int64())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, DecimalMerging) { + auto samples = Samples({ + "{\"d\": 100.99}", + "{\"d\": 1.5}", + "{\"d\": 42}", + }); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + // Decimals merge to a widened decimal (scale 2, enough integer digits), capped at 18 digits. + auto expected = arrow::struct_({arrow::field("d", arrow::decimal128(18, 2))}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, NoUsefulSchema) { + auto scalar_samples = Samples({"1", "2"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scalar_inferred, + InferColumn(infer_, scalar_samples)); + ASSERT_NE(scalar_inferred, nullptr); + ASSERT_TRUE(scalar_inferred->Equals(*arrow::int64())) << scalar_inferred->ToString(); + + // Conflicting top-level types stay unshredded. + auto mixed_samples = Samples({"1", "\"a string\""}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr mixed_inferred, + InferColumn(infer_, mixed_samples)); + ASSERT_EQ(mixed_inferred, nullptr); + + // All-null columns stay unshredded. + auto null_samples = Samples({nullptr, nullptr}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr null_inferred, + InferColumn(infer_, null_samples)); + ASSERT_EQ(null_inferred, nullptr); +} + +TEST_F(InferVariantShreddingSchemaTest, MaxSchemaWidthLimit) { + InferVariantShreddingSchema narrow_infer{/*max_schema_width=*/3, /*max_schema_depth=*/50, + /*min_field_cardinality_ratio=*/0.1}; + auto samples = Samples({R"({"a": 1, "b": 2, "c": 3, "d": 4, "e": 5})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(narrow_infer, samples)); + ASSERT_NE(inferred, nullptr); + // Budget of 3: the root object costs 1, "a" costs 2 (value + typed_value); the remaining + // fields exceed the budget and are dropped from the typed schema. + auto expected = arrow::struct_({arrow::field("a", arrow::int64())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); + + // One budget serves all variant columns of a schema: after the first column consumes it, a + // second column cannot shred anymore. + InferVariantShreddingSchema::MaxFields max_fields = narrow_infer.CreateMaxFieldsBudget(); + auto first_samples = Samples({R"({"a": 1, "b": 2})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first, + narrow_infer.InferColumnShreddingType(first_samples, &max_fields)); + ASSERT_NE(first, nullptr); + ASSERT_TRUE(first->Equals(*expected)) << first->ToString(); + auto second_samples = Samples({R"({"c": 1})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr second, + narrow_infer.InferColumnShreddingType(second_samples, &max_fields)); + ASSERT_EQ(second, nullptr); +} + +TEST_F(InferVariantShreddingSchemaTest, MaxSchemaDepthLimit) { + InferVariantShreddingSchema shallow_infer{/*max_schema_width=*/300, /*max_schema_depth=*/1, + /*min_field_cardinality_ratio=*/0.1}; + auto samples = Samples({R"({"outer": {"inner": 1}})"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, + InferColumn(shallow_infer, samples)); + ASSERT_NE(inferred, nullptr); + // Depth 1: the nested object stays an untyped variant leaf. + auto expected = arrow::struct_({arrow::field("outer", arrow::null())}); + ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, TrailingZeroDecimalNormalized) { + // GetDecimal strips 100.00 to 1E+2 (scale -2); the inferred type must carry a non-negative + // scale or reassembling the shredded file would be rejected. After normalization the value + // is an integral decimal, which finalization widens to int64. + auto samples = Samples({"100.00"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples)); + ASSERT_NE(inferred, nullptr); + ASSERT_TRUE(inferred->Equals(*arrow::int64())) << inferred->ToString(); + + auto mixed = Samples({"100.00", "1.5"}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr mixed_inferred, + InferColumn(infer_, mixed)); + ASSERT_NE(mixed_inferred, nullptr); + ASSERT_TRUE(mixed_inferred->Equals(*arrow::decimal128(18, 1))) << mixed_inferred->ToString(); +} + +TEST_F(InferVariantShreddingSchemaTest, TemporalValuesStayUnshredded) { + // The shredding schema builder rejects temporal leaf types, so inferring them would abort + // the write; date/timestamp samples must leave the column unshredded. + VariantBuilder date_builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(date_builder.AppendDate(19000)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr date_variant, date_builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr date_inferred, + InferColumn(infer_, {date_variant})); + ASSERT_EQ(date_inferred, nullptr); + + VariantBuilder ts_builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(ts_builder.AppendTimestamp(1700000000000000)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ts_variant, ts_builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ts_inferred, + InferColumn(infer_, {ts_variant})); + ASSERT_EQ(ts_inferred, nullptr); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant.cpp b/src/paimon/common/data/variant/variant.cpp new file mode 100644 index 00000000..455fa266 --- /dev/null +++ b/src/paimon/common/data/variant/variant.cpp @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/data/variant.h" + +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_get.h" +#include "paimon/common/data/variant/variant_path_segment.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +class Variant::Impl { + public: + Impl(std::shared_ptr variant, std::shared_ptr pool) + : variant_(std::move(variant)), pool_(std::move(pool)), arrow_pool_(GetArrowPool(pool_)) {} + + const std::shared_ptr& GetVariant() const { + return variant_; + } + + const std::shared_ptr& GetPool() const { + return pool_; + } + + const std::shared_ptr& GetArrowMemoryPool() const { + return arrow_pool_; + } + + private: + std::shared_ptr variant_; + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; +}; + +Variant::Variant(std::unique_ptr&& impl) : impl_(std::move(impl)) {} +Variant::~Variant() = default; + +Result> Variant::FromJson(const std::string& json, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + GenericVariant::FromJson(json, pool)); + auto impl = std::make_unique(std::move(variant), pool); + return std::unique_ptr(new Variant(std::move(impl))); +} + +Result> Variant::Create(const char* value, uint64_t value_length, + const char* metadata, uint64_t metadata_length, + const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(std::string_view(value, value_length), + std::string_view(metadata, metadata_length), pool)); + auto impl = std::make_unique(std::move(variant), pool); + return std::unique_ptr(new Variant(std::move(impl))); +} + +std::string_view Variant::Value() const { + return impl_->GetVariant()->RawValue(); +} + +std::string_view Variant::Metadata() const { + return impl_->GetVariant()->Metadata(); +} + +int64_t Variant::SizeInBytes() const { + return impl_->GetVariant()->SizeInBytes(); +} + +Result Variant::ToJson(const std::string& zone_id) const { + return impl_->GetVariant()->ToJson(zone_id); +} + +Result> Variant::VariantGet(const std::string& path, + struct ArrowSchema* target_type, + const VariantCastArgs& cast_args) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_field, + arrow::ImportField(target_type)); + return VariantGetExecutor::Get(impl_->GetVariant(), path, target_field->type(), cast_args); +} + +Result> Variant::VariantGetArrow( + const std::string& path, struct ArrowSchema* target_field, + const VariantCastArgs& cast_args) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr field, + arrow::ImportField(target_field)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr array, + VariantGetExecutor::GetAsArrow(impl_->GetVariant(), path, field, cast_args, + impl_->GetPool(), impl_->GetArrowMemoryPool())); + auto result = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, result.get())); + return result; +} + +Result> Variant::VariantGetJson(const std::string& path, + const std::string& zone_id) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr extracted, + VariantGetExecutor::ExtractByPath(impl_->GetVariant(), path)); + if (extracted == nullptr) { + return std::optional(std::nullopt); + } + PAIMON_ASSIGN_OR_RAISE(std::string json, extracted->ToJson(zone_id)); + return std::optional(std::move(json)); +} + +Result> Variant::ArrowField( + const std::string& field_name, bool nullable, + std::unordered_map metadata) { + auto variant_field = VariantTypeUtils::ToArrowField(field_name, nullable, std::move(metadata)); + auto field = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportField(*variant_field, field.get())); + return field; +} + +class VariantAccessBuilder::Impl { + public: + arrow::FieldVector fields; +}; + +VariantAccessBuilder::VariantAccessBuilder() : impl_(std::make_unique()) {} +VariantAccessBuilder::~VariantAccessBuilder() = default; + +Status VariantAccessBuilder::AddField(struct ArrowSchema* target_type, const std::string& path, + bool fail_on_error, const std::string& zone_id) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target, + arrow::ImportField(target_type)); + // Validate the path eagerly so mistakes fail at build time, not at read time. + PAIMON_RETURN_NOT_OK(VariantPathSegment::Parse(path)); + // Keep the target field's own metadata (e.g. the variant extension marker of a + // `Variant::ArrowField` target, which drives the deep re-encode cast) and add the access + // description to it. + std::vector keys = {DataField::DESCRIPTION}; + std::vector values = { + VariantAccessUtils::BuildVariantMetadata(path, fail_on_error, zone_id)}; + if (target->metadata() != nullptr) { + for (int64_t i = 0; i < target->metadata()->size(); ++i) { + if (target->metadata()->key(i) == DataField::DESCRIPTION) { + continue; + } + keys.push_back(target->metadata()->key(i)); + values.push_back(target->metadata()->value(i)); + } + } + impl_->fields.push_back(arrow::field(std::to_string(impl_->fields.size()), target->type(), + /*nullable=*/true, + arrow::KeyValueMetadata::Make(keys, values))); + return Status::OK(); +} + +Result> VariantAccessBuilder::Build( + const std::string& field_name) const { + if (impl_->fields.empty()) { + return Status::Invalid("a variant-access projection needs at least one field"); + } + auto access_field = + arrow::field(field_name, arrow::struct_(impl_->fields), /*nullable=*/true, + arrow::KeyValueMetadata::Make({VariantDefs::kExtensionTypeKey}, + {VariantDefs::kExtensionTypeValue})); + auto field = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportField(*access_field, field.get())); + return field; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_access_utils.cpp b/src/paimon/common/data/variant/variant_access_utils.cpp new file mode 100644 index 00000000..78180d1c --- /dev/null +++ b/src/paimon/common/data/variant/variant_access_utils.cpp @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_access_utils.h" + +#include +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/types/data_field.h" + +namespace paimon { + +namespace { + +std::string GetDescription(const std::shared_ptr& field) { + if (field->metadata() == nullptr) { + return std::string(); + } + auto result = field->metadata()->Get(DataField::DESCRIPTION); + if (!result.ok()) { + return std::string(); + } + return result.ValueOrDie(); +} + +// Parses the description body from the right: the last two delimited tokens are `failOnError` +// and `timeZoneId` (neither contains the delimiter), and everything before them is the path, +// which may itself contain the delimiter inside object keys (e.g. `$['a;b']`). +std::vector SplitDescription(const std::string& description) { + std::string body = description.substr(sizeof(VariantAccessUtils::kMetadataKey) - 1); + size_t tz_sep = body.rfind(VariantAccessUtils::kDelimiter); + if (tz_sep == std::string::npos) { + return {body}; + } + size_t fail_sep = + tz_sep == 0 ? std::string::npos : body.rfind(VariantAccessUtils::kDelimiter, tz_sep - 1); + if (fail_sep == std::string::npos) { + return {body.substr(0, tz_sep), body.substr(tz_sep + 1)}; + } + return {body.substr(0, fail_sep), body.substr(fail_sep + 1, tz_sep - fail_sep - 1), + body.substr(tz_sep + 1)}; +} + +bool HasAccessDescription(const std::shared_ptr& field) { + return GetDescription(field).rfind(VariantAccessUtils::kMetadataKey, 0) == 0; +} + +} // namespace + +constexpr char VariantAccessUtils::kMetadataKey[]; +constexpr char VariantAccessUtils::kDelimiter; + +std::string VariantAccessUtils::BuildVariantMetadata(const std::string& path, bool fail_on_error, + const std::string& zone_id) { + return fmt::format("{}{}{}{}{}{}", kMetadataKey, path, kDelimiter, + fail_on_error ? "true" : "false", kDelimiter, zone_id); +} + +bool VariantAccessUtils::IsVariantAccessType(const std::shared_ptr& type) { + if (type == nullptr || type->id() != arrow::Type::STRUCT || type->num_fields() == 0) { + return false; + } + for (const auto& child : type->fields()) { + if (!HasAccessDescription(child)) { + return false; + } + } + return true; +} + +Result> VariantAccessUtils::ParseAccessSpecs( + const std::shared_ptr& access_field) { + if (!IsVariantAccessType(access_field->type())) { + return Status::Invalid( + fmt::format("field '{}' is not a variant-access projection", access_field->name())); + } + std::vector specs; + specs.reserve(access_field->type()->num_fields()); + for (const auto& child : access_field->type()->fields()) { + std::string description = GetDescription(child); + std::vector parts = SplitDescription(description); + if (parts.size() != 3) { + return Status::Invalid( + fmt::format("malformed variant access description '{}' on field '{}'", description, + child->name())); + } + VariantAccessSpec spec; + spec.path = parts[0]; + PAIMON_ASSIGN_OR_RAISE(spec.segments, VariantPathSegment::Parse(spec.path)); + spec.cast_args.fail_on_error = parts[1] == "true"; + spec.cast_args.zone_id = parts[2]; + spec.target_field = child; + specs.push_back(std::move(spec)); + } + return specs; +} + +Result> VariantAccessUtils::ClipShreddedFileField( + const std::vector& specs, const std::shared_ptr& file_field) { + if (file_field->type()->id() != arrow::Type::STRUCT) { + return Status::Invalid( + fmt::format("variant file field '{}' is not a struct", file_field->name())); + } + const auto& file_struct = static_cast(*file_field->type()); + std::shared_ptr typed_value = + file_struct.GetFieldByName(VariantDefs::kTypedValueFieldName); + if (typed_value == nullptr) { + // The file stores the column unshredded; there is nothing to prune. + return file_field; + } + + bool can_clip = true; + std::set fields_to_read; + for (const auto& spec : specs) { + if (spec.segments.empty()) { + // A root path needs the whole variant. + can_clip = false; + break; + } + if (spec.segments[0].kind == VariantPathSegment::Kind::kObjectExtraction) { + // Only top-level object keys are pruned; nested paths still narrow to their + // top-level key. + fields_to_read.insert(spec.segments[0].key); + } else { + can_clip = false; + break; + } + } + if (!can_clip) { + return file_field; + } + + std::shared_ptr metadata_field = + file_struct.GetFieldByName(VariantDefs::kMetadataFieldName); + std::shared_ptr value_field = + file_struct.GetFieldByName(VariantDefs::kValueFieldName); + if (metadata_field == nullptr) { + return Status::Invalid( + fmt::format("shredded variant field '{}' misses metadata", file_field->name())); + } + + arrow::FieldVector typed_fields; + if (typed_value->type()->id() == arrow::Type::STRUCT) { + for (const auto& typed_child : typed_value->type()->fields()) { + if (fields_to_read.erase(typed_child->name()) > 0) { + typed_fields.push_back(typed_child); + } + } + } + + arrow::FieldVector clipped_fields = {metadata_field}; + if (!fields_to_read.empty() && value_field != nullptr) { + // Some requested key is not shredded; keep `value` for the binary fallback. + clipped_fields.push_back(value_field); + } + if (!typed_fields.empty()) { + clipped_fields.push_back(typed_value->WithType(arrow::struct_(typed_fields))); + } + return file_field->WithType(arrow::struct_(clipped_fields)); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_access_utils.h b/src/paimon/common/data/variant/variant_access_utils.h new file mode 100644 index 00000000..2cd79e9b --- /dev/null +++ b/src/paimon/common/data/variant/variant_access_utils.h @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/variant/variant_path_segment.h" +#include "paimon/data/variant.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class Field; +} // namespace arrow + +namespace paimon { + +/// One extracted field of a variant-access projection: the extraction path, cast behavior, and +/// the target field the extracted value is cast to. +struct VariantAccessSpec { + std::string path; + std::vector segments; + VariantCastArgs cast_args; + std::shared_ptr target_field; +}; + +/// Utilities for variant-access projections: a variant column read as a struct whose children +/// each carry a `__VARIANT_METADATA;;` description (mirroring the +/// Java `VariantMetadataUtils`). Such a projection extracts the described paths from the variant +/// column at read time, reading only the required shredded sub-columns. +class VariantAccessUtils { + public: + static constexpr char kMetadataKey[] = "__VARIANT_METADATA"; + static constexpr char kDelimiter = ';'; + + VariantAccessUtils() = delete; + ~VariantAccessUtils() = delete; + + /// Builds the description string encoding one access spec. + static std::string BuildVariantMetadata(const std::string& path, bool fail_on_error, + const std::string& zone_id); + + /// Whether `type` is a variant-access projection: a struct with at least one field whose + /// children all carry a `__VARIANT_METADATA` description. + static bool IsVariantAccessType(const std::shared_ptr& type); + + /// Parses the access specs of a variant-access projection field. + static Result> ParseAccessSpecs( + const std::shared_ptr& access_field); + + /// Prunes a shredded file field down to the sub-columns required by the access specs: + /// `metadata` is always kept, `typed_value` is narrowed to the requested top-level keys, and + /// `value` is kept only when some requested key is not shredded. Returns `file_field` + /// unchanged when the file is unshredded or the paths cannot be pruned (root or array-first + /// paths). + static Result> ClipShreddedFileField( + const std::vector& specs, + const std::shared_ptr& file_field); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_binary_util.cpp b/src/paimon/common/data/variant/variant_binary_util.cpp new file mode 100644 index 00000000..596d129c --- /dev/null +++ b/src/paimon/common/data/variant/variant_binary_util.cpp @@ -0,0 +1,599 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_binary_util.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" + +namespace paimon { + +int32_t VariantDecimal::Precision() const { + __uint128_t abs_value = + unscaled < 0 ? -static_cast<__uint128_t>(unscaled) : static_cast<__uint128_t>(unscaled); + int32_t digits = 1; + while (abs_value >= 10) { + abs_value /= 10; + ++digits; + } + return digits; +} + +VariantDecimal VariantDecimal::StripTrailingZeros() const { + VariantDecimal result = *this; + if (result.unscaled == 0) { + result.scale = 0; + return result; + } + while (result.unscaled % 10 == 0) { + result.unscaled /= 10; + --result.scale; + } + return result; +} + +std::string VariantDecimal::ToPlainString() const { + __uint128_t abs_value = + unscaled < 0 ? -static_cast<__uint128_t>(unscaled) : static_cast<__uint128_t>(unscaled); + std::string digits; + if (abs_value == 0) { + digits = "0"; + } else { + while (abs_value > 0) { + digits.push_back(static_cast('0' + static_cast(abs_value % 10))); + abs_value /= 10; + } + std::reverse(digits.begin(), digits.end()); + } + std::string result; + if (unscaled < 0) { + result.push_back('-'); + } + if (scale <= 0) { + result.append(digits); + result.append(static_cast(-scale), '0'); + } else if (static_cast(scale) < digits.size()) { + result.append(digits, 0, digits.size() - scale); + result.push_back('.'); + result.append(digits, digits.size() - scale, scale); + } else { + result.append("0."); + result.append(static_cast(scale) - digits.size(), '0'); + result.append(digits); + } + return result; +} + +Status VariantBinaryUtil::MalformedVariant(const std::string& message) { + if (message.empty()) { + return Status::Invalid("MALFORMED_VARIANT"); + } + return Status::Invalid(fmt::format("MALFORMED_VARIANT: {}", message)); +} + +Status VariantBinaryUtil::UnknownPrimitiveTypeInVariant(int32_t id) { + return Status::Invalid(fmt::format("UNKNOWN_PRIMITIVE_TYPE_IN_VARIANT, id: {}", id)); +} + +Status VariantBinaryUtil::VariantConstructorSizeLimit() { + return Status::Invalid("VARIANT_CONSTRUCTOR_SIZE_LIMIT"); +} + +Status VariantBinaryUtil::UnexpectedType(VariantValueType type) { + static constexpr const char* kTypeNames[] = { + "OBJECT", "ARRAY", "NULL", "BOOLEAN", "LONG", "STRING", "DOUBLE", + "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP_NTZ", "FLOAT", "BINARY", "UUID"}; + return Status::Invalid( + fmt::format("Expect type to be {}", kTypeNames[static_cast(type)])); +} + +Status VariantBinaryUtil::CheckIndex(int32_t pos, int32_t length) { + if (pos < 0 || pos >= length) { + return MalformedVariant( + fmt::format("index {} is out of bounds for a buffer of {} bytes", pos, length)); + } + return Status::OK(); +} + +void VariantBinaryUtil::WriteLong(int64_t value, int32_t num_bytes, uint8_t* bytes, int32_t pos) { + for (int32_t i = 0; i < num_bytes; ++i) { + bytes[pos + i] = static_cast((static_cast(value) >> (8 * i)) & 0xFF); + } +} + +Result VariantBinaryUtil::ReadLong(std::string_view bytes, int32_t pos, + int32_t num_bytes) { + auto length = static_cast(bytes.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + PAIMON_RETURN_NOT_OK(CheckIndex(pos + num_bytes - 1, length)); + uint64_t result = 0; + // All bytes except the most significant byte should be unsign-extended and shifted. The most + // significant byte should be sign-extended. + for (int32_t i = 0; i < num_bytes - 1; ++i) { + uint64_t unsigned_byte_value = static_cast(bytes[pos + i]); + result |= unsigned_byte_value << (8 * i); + } + int64_t signed_byte_value = static_cast(bytes[pos + num_bytes - 1]); + result |= static_cast(signed_byte_value) << (8 * (num_bytes - 1)); + return static_cast(result); +} + +Result VariantBinaryUtil::ReadUnsigned(std::string_view bytes, int32_t pos, + int32_t num_bytes) { + auto length = static_cast(bytes.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + PAIMON_RETURN_NOT_OK(CheckIndex(pos + num_bytes - 1, length)); + int64_t result = 0; + // Similar to the `ReadLong` loop, but all bytes should be unsign-extended. + for (int32_t i = 0; i < num_bytes; ++i) { + int64_t unsigned_byte_value = static_cast(bytes[pos + i]); + result |= unsigned_byte_value << (8 * i); + } + if (result < 0 || result > std::numeric_limits::max()) { + return MalformedVariant(fmt::format("unsigned value {} does not fit into int32", result)); + } + return static_cast(result); +} + +uint8_t VariantBinaryUtil::PrimitiveHeader(int32_t type) { + return static_cast(type << 2 | VariantDefs::kPrimitive); +} + +uint8_t VariantBinaryUtil::ShortStrHeader(int32_t size) { + return static_cast(size << 2 | VariantDefs::kShortStr); +} + +uint8_t VariantBinaryUtil::ObjectHeader(bool large_size, int32_t id_size, int32_t offset_size) { + return static_cast(((large_size ? 1 : 0) << (VariantDefs::kBasicTypeBits + 4)) | + ((id_size - 1) << (VariantDefs::kBasicTypeBits + 2)) | + ((offset_size - 1) << VariantDefs::kBasicTypeBits) | + VariantDefs::kObject); +} + +uint8_t VariantBinaryUtil::ArrayHeader(bool large_size, int32_t offset_size) { + return static_cast(((large_size ? 1 : 0) << (VariantDefs::kBasicTypeBits + 2)) | + ((offset_size - 1) << VariantDefs::kBasicTypeBits) | + VariantDefs::kArray); +} + +Result VariantBinaryUtil::GetTypeInfo(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + return (static_cast(value[pos]) >> VariantDefs::kBasicTypeBits) & + VariantDefs::kTypeInfoMask; +} + +Result VariantBinaryUtil::GetType(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + switch (basic_type) { + case VariantDefs::kShortStr: + return VariantValueType::kString; + case VariantDefs::kObject: + return VariantValueType::kObject; + case VariantDefs::kArray: + return VariantValueType::kArray; + default: + switch (type_info) { + case VariantDefs::kNull: + return VariantValueType::kNull; + case VariantDefs::kTrue: + case VariantDefs::kFalse: + return VariantValueType::kBoolean; + case VariantDefs::kInt1: + case VariantDefs::kInt2: + case VariantDefs::kInt4: + case VariantDefs::kInt8: + return VariantValueType::kLong; + case VariantDefs::kDouble: + return VariantValueType::kDouble; + case VariantDefs::kDecimal4: + case VariantDefs::kDecimal8: + case VariantDefs::kDecimal16: + return VariantValueType::kDecimal; + case VariantDefs::kDate: + return VariantValueType::kDate; + case VariantDefs::kTimestamp: + return VariantValueType::kTimestamp; + case VariantDefs::kTimestampNtz: + return VariantValueType::kTimestampNtz; + case VariantDefs::kFloat: + return VariantValueType::kFloat; + case VariantDefs::kBinary: + return VariantValueType::kBinary; + case VariantDefs::kLongStr: + return VariantValueType::kString; + case VariantDefs::kUuid: + return VariantValueType::kUuid; + default: + return UnknownPrimitiveTypeInVariant(type_info); + } + } +} + +Result VariantBinaryUtil::ValueSize(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + switch (basic_type) { + case VariantDefs::kShortStr: + return 1 + type_info; + case VariantDefs::kObject: { + PAIMON_ASSIGN_OR_RAISE(ObjectInfo info, GetObjectInfo(value, pos)); + PAIMON_ASSIGN_OR_RAISE( + int32_t data_size, + ReadUnsigned(value, info.offset_start + info.num_elements * info.offset_size, + info.offset_size)); + return info.data_start - pos + data_size; + } + case VariantDefs::kArray: { + PAIMON_ASSIGN_OR_RAISE(ArrayInfo info, GetArrayInfo(value, pos)); + PAIMON_ASSIGN_OR_RAISE( + int32_t data_size, + ReadUnsigned(value, info.offset_start + info.num_elements * info.offset_size, + info.offset_size)); + return info.data_start - pos + data_size; + } + default: + switch (type_info) { + case VariantDefs::kNull: + case VariantDefs::kTrue: + case VariantDefs::kFalse: + return 1; + case VariantDefs::kInt1: + return 2; + case VariantDefs::kInt2: + return 3; + case VariantDefs::kInt4: + case VariantDefs::kDate: + case VariantDefs::kFloat: + return 5; + case VariantDefs::kInt8: + case VariantDefs::kDouble: + case VariantDefs::kTimestamp: + case VariantDefs::kTimestampNtz: + return 9; + case VariantDefs::kDecimal4: + return 6; + case VariantDefs::kDecimal8: + return 10; + case VariantDefs::kDecimal16: + return 18; + case VariantDefs::kBinary: + case VariantDefs::kLongStr: { + PAIMON_ASSIGN_OR_RAISE(int32_t data_size, + ReadUnsigned(value, pos + 1, VariantDefs::kU32Size)); + return 1 + VariantDefs::kU32Size + data_size; + } + case VariantDefs::kUuid: + return 17; + default: + return UnknownPrimitiveTypeInVariant(type_info); + } + } +} + +Result VariantBinaryUtil::GetBoolean(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || + (type_info != VariantDefs::kTrue && type_info != VariantDefs::kFalse)) { + return UnexpectedType(VariantValueType::kBoolean); + } + return type_info == VariantDefs::kTrue; +} + +Result VariantBinaryUtil::GetLong(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + constexpr const char* kExceptionMessage = "Expect type to be LONG/DATE/TIMESTAMP/TIMESTAMP_NTZ"; + if (basic_type != VariantDefs::kPrimitive) { + return Status::Invalid(kExceptionMessage); + } + switch (type_info) { + case VariantDefs::kInt1: + return ReadLong(value, pos + 1, 1); + case VariantDefs::kInt2: + return ReadLong(value, pos + 1, 2); + case VariantDefs::kInt4: + case VariantDefs::kDate: + return ReadLong(value, pos + 1, 4); + case VariantDefs::kInt8: + case VariantDefs::kTimestamp: + case VariantDefs::kTimestampNtz: + return ReadLong(value, pos + 1, 8); + default: + return Status::Invalid(kExceptionMessage); + } +} + +Result VariantBinaryUtil::GetDouble(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kDouble) { + return UnexpectedType(VariantValueType::kDouble); + } + PAIMON_ASSIGN_OR_RAISE(int64_t bits, ReadLong(value, pos + 1, 8)); + double result; + memcpy(&result, &bits, sizeof(result)); + return result; +} + +namespace { +// Checks whether the precision and scale of the decimal are within the limit. +Status CheckDecimal(const VariantDecimal& d, int32_t max_precision) { + if (d.Precision() > max_precision || d.scale > max_precision) { + return VariantBinaryUtil::MalformedVariant( + fmt::format("decimal precision {} or scale {} exceeds the maximum precision {}", + d.Precision(), d.scale, max_precision)); + } + return Status::OK(); +} +} // namespace + +Result VariantBinaryUtil::GetDecimalWithOriginalScale(std::string_view value, + int32_t pos) { + auto length = static_cast(value.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive) { + return UnexpectedType(VariantValueType::kDecimal); + } + PAIMON_RETURN_NOT_OK(CheckIndex(pos + 1, length)); + // Interpret the scale byte as unsigned. If it is a negative byte, the unsigned value must be + // greater than `kMaxDecimal16Precision` and will trigger an error in `CheckDecimal`. + int32_t scale = static_cast(value[pos + 1]); + VariantDecimal result; + result.scale = scale; + switch (type_info) { + case VariantDefs::kDecimal4: { + PAIMON_ASSIGN_OR_RAISE(int64_t unscaled, ReadLong(value, pos + 2, 4)); + result.unscaled = unscaled; + PAIMON_RETURN_NOT_OK(CheckDecimal(result, VariantDefs::kMaxDecimal4Precision)); + break; + } + case VariantDefs::kDecimal8: { + PAIMON_ASSIGN_OR_RAISE(int64_t unscaled, ReadLong(value, pos + 2, 8)); + result.unscaled = unscaled; + PAIMON_RETURN_NOT_OK(CheckDecimal(result, VariantDefs::kMaxDecimal8Precision)); + break; + } + case VariantDefs::kDecimal16: { + PAIMON_RETURN_NOT_OK(CheckIndex(pos + 17, length)); + __uint128_t unscaled = 0; + for (int32_t i = 0; i < 16; ++i) { + unscaled |= static_cast<__uint128_t>(static_cast(value[pos + 2 + i])) + << (8 * i); + } + result.unscaled = static_cast<__int128_t>(unscaled); + PAIMON_RETURN_NOT_OK(CheckDecimal(result, VariantDefs::kMaxDecimal16Precision)); + break; + } + default: + return UnexpectedType(VariantValueType::kDecimal); + } + return result; +} + +Result VariantBinaryUtil::GetDecimal(std::string_view value, int32_t pos) { + PAIMON_ASSIGN_OR_RAISE(VariantDecimal decimal, GetDecimalWithOriginalScale(value, pos)); + return decimal.StripTrailingZeros(); +} + +Result VariantBinaryUtil::GetFloat(std::string_view value, int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kFloat) { + return UnexpectedType(VariantValueType::kFloat); + } + PAIMON_ASSIGN_OR_RAISE(int64_t bits, ReadLong(value, pos + 1, 4)); + auto int_bits = static_cast(bits); + float result; + memcpy(&result, &int_bits, sizeof(result)); + return result; +} + +Result VariantBinaryUtil::GetBinary(std::string_view value, int32_t pos) { + auto length = static_cast(value.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kBinary) { + return UnexpectedType(VariantValueType::kBinary); + } + int32_t start = pos + 1 + VariantDefs::kU32Size; + PAIMON_ASSIGN_OR_RAISE(int32_t data_size, ReadUnsigned(value, pos + 1, VariantDefs::kU32Size)); + if (data_size > 0) { + PAIMON_RETURN_NOT_OK(CheckIndex(start + data_size - 1, length)); + } + return value.substr(start, data_size); +} + +Result VariantBinaryUtil::GetString(std::string_view value, int32_t pos) { + auto length = static_cast(value.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type == VariantDefs::kShortStr || + (basic_type == VariantDefs::kPrimitive && type_info == VariantDefs::kLongStr)) { + int32_t start; + int32_t str_size; + if (basic_type == VariantDefs::kShortStr) { + start = pos + 1; + str_size = type_info; + } else { + start = pos + 1 + VariantDefs::kU32Size; + PAIMON_ASSIGN_OR_RAISE(str_size, ReadUnsigned(value, pos + 1, VariantDefs::kU32Size)); + } + if (str_size > 0) { + PAIMON_RETURN_NOT_OK(CheckIndex(start + str_size - 1, length)); + } + return value.substr(start, str_size); + } + return UnexpectedType(VariantValueType::kString); +} + +Result VariantBinaryUtil::GetUuid(std::string_view value, int32_t pos) { + auto length = static_cast(value.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(pos, length)); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kPrimitive || type_info != VariantDefs::kUuid) { + return UnexpectedType(VariantValueType::kUuid); + } + int32_t start = pos + 1; + PAIMON_RETURN_NOT_OK(CheckIndex(start + 15, length)); + return value.substr(start, 16); +} + +std::string VariantBinaryUtil::UuidToString(std::string_view uuid_bytes) { + constexpr char kHexDigits[] = "0123456789abcdef"; + std::string result; + result.reserve(36); + for (size_t i = 0; i < 16; ++i) { + if (i == 4 || i == 6 || i == 8 || i == 10) { + result.push_back('-'); + } + auto byte = static_cast(uuid_bytes[i]); + result.push_back(kHexDigits[byte >> 4]); + result.push_back(kHexDigits[byte & 0xF]); + } + return result; +} + +Result VariantBinaryUtil::GetObjectInfo(std::string_view value, + int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kObject) { + return UnexpectedType(VariantValueType::kObject); + } + // Refer to the comment of the `VariantDefs::kObject` constant for the details of the object + // header encoding. Suppose `type_info` has a bit representation of 0_b4_b3b2_b1b0, the + // following line extracts b4 to determine whether the object uses a 1/4-byte size. + bool large_size = ((type_info >> 4) & 0x1) != 0; + int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; + ObjectInfo info; + PAIMON_ASSIGN_OR_RAISE(info.num_elements, ReadUnsigned(value, pos + 1, size_bytes)); + // Extracts b3b2 to determine the integer size of the field id list. + info.id_size = ((type_info >> 2) & 0x3) + 1; + // Extracts b1b0 to determine the integer size of the offset list. + info.offset_size = (type_info & 0x3) + 1; + info.id_start = pos + 1 + size_bytes; + // A corrupted variant can claim a near-INT32_MAX element count; compute the layout in + // 64-bit and bound it by the buffer before 32-bit arithmetic could overflow. + int64_t offset_start = static_cast(info.id_start) + + static_cast(info.num_elements) * info.id_size; + int64_t data_start = + offset_start + (static_cast(info.num_elements) + 1) * info.offset_size; + if (data_start > static_cast(value.size())) { + return MalformedVariant("object layout exceeds the value buffer"); + } + info.offset_start = static_cast(offset_start); + info.data_start = static_cast(data_start); + return info; +} + +Result VariantBinaryUtil::GetArrayInfo(std::string_view value, + int32_t pos) { + PAIMON_RETURN_NOT_OK(CheckIndex(pos, static_cast(value.size()))); + auto header = static_cast(value[pos]); + int32_t basic_type = header & VariantDefs::kBasicTypeMask; + int32_t type_info = (header >> VariantDefs::kBasicTypeBits) & VariantDefs::kTypeInfoMask; + if (basic_type != VariantDefs::kArray) { + return UnexpectedType(VariantValueType::kArray); + } + // Suppose `type_info` has a bit representation of 000_b2_b1b0, the following line extracts b2 + // to determine whether the array uses a 1/4-byte size. + bool large_size = ((type_info >> 2) & 0x1) != 0; + int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; + ArrayInfo info; + PAIMON_ASSIGN_OR_RAISE(info.num_elements, ReadUnsigned(value, pos + 1, size_bytes)); + // Extracts b1b0 to determine the integer size of the offset list. + info.offset_size = (type_info & 0x3) + 1; + info.offset_start = pos + 1 + size_bytes; + // See GetObjectInfo: bound the 64-bit layout by the buffer before 32-bit overflow. + int64_t data_start = static_cast(info.offset_start) + + (static_cast(info.num_elements) + 1) * info.offset_size; + if (data_start > static_cast(value.size())) { + return MalformedVariant("array layout exceeds the value buffer"); + } + info.data_start = static_cast(data_start); + return info; +} + +Result VariantBinaryUtil::GetMetadataKey(std::string_view metadata, int32_t id) { + auto length = static_cast(metadata.size()); + PAIMON_RETURN_NOT_OK(CheckIndex(0, length)); + // Extracts the highest 2 bits in the metadata header to determine the integer size of the + // offset list. + int32_t offset_size = ((static_cast(metadata[0]) >> 6) & 0x3) + 1; + PAIMON_ASSIGN_OR_RAISE(int32_t dict_size, ReadUnsigned(metadata, 1, offset_size)); + if (id >= dict_size) { + return MalformedVariant(fmt::format( + "metadata key id {} is out of bounds for a dictionary of {} keys", id, dict_size)); + } + // There are a header byte, a `dict_size` with `offset_size` bytes, and `(dict_size + 1)` + // offsets before the string data. + // Bound the offset-table layout in 64-bit before 32-bit arithmetic could overflow on a + // corrupted dictionary size. + int64_t string_start64 = 1 + (static_cast(dict_size) + 2) * offset_size; + if (string_start64 > static_cast(length)) { + return MalformedVariant("metadata dictionary layout exceeds the metadata buffer"); + } + auto string_start = static_cast(string_start64); + PAIMON_ASSIGN_OR_RAISE(int32_t offset, + ReadUnsigned(metadata, 1 + (id + 1) * offset_size, offset_size)); + PAIMON_ASSIGN_OR_RAISE(int32_t next_offset, + ReadUnsigned(metadata, 1 + (id + 2) * offset_size, offset_size)); + if (offset > next_offset) { + return MalformedVariant("metadata key offsets are not monotonic"); + } + if (static_cast(string_start) + next_offset > static_cast(length)) { + return MalformedVariant("metadata key data exceeds the metadata buffer"); + } + return metadata.substr(string_start + offset, next_offset - offset); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_binary_util.h b/src/paimon/common/data/variant/variant_binary_util.h new file mode 100644 index 00000000..01a55f4b --- /dev/null +++ b/src/paimon/common/data/variant/variant_binary_util.h @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { + +/// The value type of a variant value. It is determined by the header byte but not a 1:1 mapping +/// (for example, INT1/2/4/8 all map to `kLong`). +enum class VariantValueType { + kObject, + kArray, + kNull, + kBoolean, + kLong, + kString, + kDouble, + kDecimal, + kDate, + kTimestamp, + kTimestampNtz, + kFloat, + kBinary, + kUuid, +}; + +/// An arbitrary-precision (up to 38 digits) decimal value decoded from a variant binary. The +/// numeric value is `unscaled * 10^(-scale)`. Unlike `paimon::Decimal`, `scale` may become +/// negative after `StripTrailingZeros` (mirroring `java.math.BigDecimal`). +struct VariantDecimal { + __int128_t unscaled = 0; + int32_t scale = 0; + + /// Number of decimal digits in the unscaled value (a value of zero has precision 1). + int32_t Precision() const; + + /// Removes trailing zero digits from the unscaled value, increasing `10^-scale` accordingly. + VariantDecimal StripTrailingZeros() const; + + /// Plain (non-scientific) string representation, e.g. `-12.340` or `100`. + std::string ToPlainString() const; + + bool operator==(const VariantDecimal& other) const { + return unscaled == other.unscaled && scale == other.scale; + } +}; + +/// Static functions for manipulating variant binaries. See `VariantDefs` for the binary format. +class VariantBinaryUtil { + public: + VariantBinaryUtil() = delete; + ~VariantBinaryUtil() = delete; + + /// The decoded layout of a variant object value. + struct ObjectInfo { + /// Number of object fields. + int32_t num_elements; + /// The integer size of the field id list. + int32_t id_size; + /// The integer size of the offset list. + int32_t offset_size; + /// The starting index of the field id list in the variant value. + int32_t id_start; + /// The starting index of the offset list in the variant value. + int32_t offset_start; + /// The starting index of field data in the variant value. + int32_t data_start; + }; + + /// The decoded layout of a variant array value. + struct ArrayInfo { + /// Number of array elements. + int32_t num_elements; + /// The integer size of the offset list. + int32_t offset_size; + /// The starting index of the offset list in the variant value. + int32_t offset_start; + /// The starting index of element data in the variant value. + int32_t data_start; + }; + + /// Creates the MALFORMED_VARIANT error. `message` describes the specific corruption for + /// debugging and is appended to the error when non-empty. + static Status MalformedVariant(const std::string& message = ""); + static Status UnknownPrimitiveTypeInVariant(int32_t id); + static Status VariantConstructorSizeLimit(); + static Status UnexpectedType(VariantValueType type); + + /// Checks the validity of an index `pos` in a buffer of `length` bytes. Returns + /// `MALFORMED_VARIANT` if it is out of bound. + static Status CheckIndex(int32_t pos, int32_t length); + + /// Writes the least significant `num_bytes` bytes in `value` into + /// `bytes[pos, pos + num_bytes)` in little endian. + static void WriteLong(int64_t value, int32_t num_bytes, uint8_t* bytes, int32_t pos); + + /// Reads a little-endian signed long value from `bytes[pos, pos + num_bytes)`. + static Result ReadLong(std::string_view bytes, int32_t pos, int32_t num_bytes); + + /// Reads a little-endian unsigned int value from `bytes[pos, pos + num_bytes)`. The value + /// must fit into a non-negative int32. + static Result ReadUnsigned(std::string_view bytes, int32_t pos, int32_t num_bytes); + + /// Adds a buffer-relative element `offset` to `base` in 64-bit and validates the result + /// stays inside `buffer_size`, guarding the 32-bit addition against corrupted offsets. + static Result CheckedElementPos(int32_t base, int32_t offset, size_t buffer_size) { + int64_t pos = static_cast(base) + offset; + if (pos >= static_cast(buffer_size)) { + return MalformedVariant("element offset points outside the value buffer"); + } + return static_cast(pos); + } + + static uint8_t PrimitiveHeader(int32_t type); + static uint8_t ShortStrHeader(int32_t size); + static uint8_t ObjectHeader(bool large_size, int32_t id_size, int32_t offset_size); + static uint8_t ArrayHeader(bool large_size, int32_t offset_size); + + /// Gets the type info bits from the variant value `value[pos...]`. + static Result GetTypeInfo(std::string_view value, int32_t pos); + + /// Gets the value type of the variant value `value[pos...]`. It is only legal to call `Get*` + /// if `GetType` returns the corresponding type (for example, it is only legal to call + /// `GetLong` if `GetType` returns `kLong`). + static Result GetType(std::string_view value, int32_t pos); + + /// Computes the size in bytes of the variant value `value[pos...]`. `value.size() - pos` is + /// an upper bound of the size, but the actual size can be smaller. + static Result ValueSize(std::string_view value, int32_t pos); + + static Result GetBoolean(std::string_view value, int32_t pos); + + /// Gets a long value from the variant value `value[pos...]`. It is only legal to call it if + /// `GetType` returns one of `kLong/kDate/kTimestamp/kTimestampNtz`. If the type is `kDate`, + /// the return value is guaranteed to fit into an int32 and represents the number of days from + /// the Unix epoch. If the type is `kTimestamp/kTimestampNtz`, the return value represents the + /// number of microseconds from the Unix epoch. + static Result GetLong(std::string_view value, int32_t pos); + + static Result GetDouble(std::string_view value, int32_t pos); + + /// Gets a decimal value from the variant value `value[pos...]`, keeping the stored scale. + static Result GetDecimalWithOriginalScale(std::string_view value, int32_t pos); + + /// Gets a decimal value from the variant value `value[pos...]` with trailing zeros stripped. + static Result GetDecimal(std::string_view value, int32_t pos); + + static Result GetFloat(std::string_view value, int32_t pos); + + /// Gets a binary value from the variant value `value[pos...]`. The returned view aliases + /// `value` and remains valid only as long as the underlying buffer. + static Result GetBinary(std::string_view value, int32_t pos); + + /// Gets a string value from the variant value `value[pos...]`. The returned view aliases + /// `value` and remains valid only as long as the underlying buffer. + static Result GetString(std::string_view value, int32_t pos); + + /// Gets a UUID value (16 bytes, big-endian) from the variant value `value[pos...]`. The + /// returned view aliases `value`. + static Result GetUuid(std::string_view value, int32_t pos); + + /// Formats a 16-byte big-endian UUID as the canonical lower-case string, e.g. + /// `123e4567-e89b-12d3-a456-426614174000`. + static std::string UuidToString(std::string_view uuid_bytes); + + /// Decodes the layout of the variant object value `value[pos...]`. + static Result GetObjectInfo(std::string_view value, int32_t pos); + + /// Decodes the layout of the variant array value `value[pos...]`. + static Result GetArrayInfo(std::string_view value, int32_t pos); + + /// Gets the key at `id` in the variant metadata. An out-of-bound `id` is considered a + /// malformed variant because it is read from the corresponding variant value. + static Result GetMetadataKey(std::string_view metadata, int32_t id); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_builder.cpp b/src/paimon/common/data/variant/variant_builder.cpp new file mode 100644 index 00000000..b3cf7ee5 --- /dev/null +++ b/src/paimon/common/data/variant/variant_builder.cpp @@ -0,0 +1,654 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_builder.h" + +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "rapidjson/error/en.h" +#include "rapidjson/memorystream.h" +#include "rapidjson/reader.h" + +namespace paimon { + +namespace { + +// A rapidjson SAX handler that feeds parsed JSON events into a `VariantBuilder`, mirroring +// `GenericVariantBuilder.buildJson` in the Java implementation. Numbers are delivered as raw +// text (`kParseNumbersAsStringsFlag`) so that exact decimal semantics match Jackson's. +class JsonToVariantHandler + : public rapidjson::BaseReaderHandler, JsonToVariantHandler> { + public: + explicit JsonToVariantHandler(VariantBuilder* builder) : builder_(builder) {} + + bool Null() { + BeforeValue(); + return Ok(builder_->AppendNull()); + } + + bool Bool(bool b) { + BeforeValue(); + return Ok(builder_->AppendBoolean(b)); + } + + bool RawNumber(const char* str, rapidjson::SizeType length, bool /*copy*/) { + BeforeValue(); + return Ok(AppendNumber(std::string_view(str, length))); + } + + bool String(const char* str, rapidjson::SizeType length, bool /*copy*/) { + BeforeValue(); + return Ok(builder_->AppendString(std::string_view(str, length))); + } + + bool StartObject() { + BeforeValue(); + contexts_.emplace_back(Context{true, builder_->GetWritePos(), {}, {}, {}}); + return true; + } + + bool Key(const char* str, rapidjson::SizeType length, bool /*copy*/) { + contexts_.back().pending_key.assign(str, length); + return true; + } + + bool EndObject(rapidjson::SizeType /*member_count*/) { + Context context = std::move(contexts_.back()); + contexts_.pop_back(); + return Ok(builder_->FinishWritingObject(context.start, &context.fields)); + } + + bool StartArray() { + BeforeValue(); + contexts_.emplace_back(Context{false, builder_->GetWritePos(), {}, {}, {}}); + return true; + } + + bool EndArray(rapidjson::SizeType /*element_count*/) { + Context context = std::move(contexts_.back()); + contexts_.pop_back(); + return Ok(builder_->FinishWritingArray(context.start, context.offsets)); + } + + const Status& status() const { + return status_; + } + + private: + struct Context { + bool is_object; + int32_t start; + std::vector fields; + std::vector offsets; + std::string pending_key; + }; + + bool Ok(const Status& status) { + if (!status.ok()) { + status_ = status; + return false; + } + return true; + } + + // Records the offset of the value that is about to be appended in the enclosing container. + void BeforeValue() { + if (contexts_.empty()) { + return; + } + Context& top = contexts_.back(); + int32_t offset = builder_->GetWritePos() - top.start; + if (top.is_object) { + int32_t id = builder_->AddKey(top.pending_key); + top.fields.emplace_back(top.pending_key, id, offset); + } else { + top.offsets.push_back(offset); + } + } + + // Mirrors the Java number handling: integers that fit in a long are appended as long; + // everything else is first tried as an exact decimal and falls back to double. + Status AppendNumber(std::string_view text) { + bool integral = true; + for (char c : text) { + if (c != '-' && !(c >= '0' && c <= '9')) { + integral = false; + break; + } + } + if (integral) { + int64_t long_value = 0; + auto [ptr, ec] = std::from_chars(text.data(), text.data() + text.size(), long_value); + if (ec == std::errc() && ptr == text.data() + text.size()) { + return builder_->AppendLong(long_value); + } + } + PAIMON_ASSIGN_OR_RAISE(bool appended, TryAppendDecimal(text)); + if (appended) { + return Status::OK(); + } + char* end = nullptr; + std::string text_copy(text); + double double_value = std::strtod(text_copy.c_str(), &end); + if (end != text_copy.c_str() + text_copy.size()) { + return Status::Invalid(fmt::format("Invalid JSON number: {}", text)); + } + return builder_->AppendDouble(double_value); + } + + // Tries to append a JSON number as an exact decimal. Returns whether it succeeded. The input + // must only use the decimal format (an integer value with an optional '.' in it) and must not + // use scientific notation. It also must fit into the precision limitation of decimal types. + Result TryAppendDecimal(std::string_view text) { + for (char c : text) { + if (c != '-' && c != '.' && !(c >= '0' && c <= '9')) { + return false; + } + } + bool negative = false; + size_t i = 0; + if (i < text.size() && text[i] == '-') { + negative = true; + ++i; + } + __int128_t unscaled = 0; + int32_t scale = 0; + int32_t significant_digits = 0; + bool seen_point = false; + bool seen_nonzero = false; + for (; i < text.size(); ++i) { + char c = text[i]; + if (c == '.') { + seen_point = true; + continue; + } + if (seen_point) { + ++scale; + } + if (c != '0' || seen_nonzero) { + seen_nonzero = true; + ++significant_digits; + } + if (significant_digits > VariantDefs::kMaxDecimal16Precision) { + return false; + } + unscaled = unscaled * 10 + (c - '0'); + } + if (scale > VariantDefs::kMaxDecimal16Precision) { + return false; + } + VariantDecimal decimal; + decimal.unscaled = negative ? -unscaled : unscaled; + decimal.scale = scale; + PAIMON_RETURN_NOT_OK(builder_->AppendDecimal(decimal)); + return true; + } + + VariantBuilder* builder_; + std::vector contexts_; + Status status_; +}; + +} // namespace + +Result> VariantBuilder::ParseJson( + std::string_view json, bool allow_duplicate_keys, const std::shared_ptr& pool) { + VariantBuilder builder(allow_duplicate_keys); + JsonToVariantHandler handler(&builder); + rapidjson::Reader reader; + rapidjson::MemoryStream stream(json.data(), json.size()); + rapidjson::ParseResult result = + reader.Parse(stream, handler); + if (!result) { + if (!handler.status().ok()) { + return handler.status(); + } + return Status::Invalid(fmt::format("Failed to parse JSON: {} (at offset {})", + rapidjson::GetParseError_En(result.Code()), + result.Offset())); + } + return builder.Build(pool); +} + +Result> VariantBuilder::Build( + const std::shared_ptr& pool) { + auto num_keys = static_cast(dictionary_keys_.size()); + // Use int64 to avoid overflow in accumulating lengths. + int64_t dictionary_string_size = 0; + for (const std::string& key : dictionary_keys_) { + dictionary_string_size += static_cast(key.size()); + } + // Determine the number of bytes required per offset entry. The largest offset is the + // one-past-the-end value, which is the total string size. It's very unlikely that the number + // of keys could be larger, but incorporate that into the calculation in case of pathological + // data. + int64_t max_size = std::max(dictionary_string_size, static_cast(num_keys)); + if (max_size > VariantDefs::kSizeLimit) { + return Status::Invalid("VARIANT_SIZE_LIMIT"); + } + int32_t offset_size = GetIntegerSize(static_cast(max_size)); + + int32_t offset_start = 1 + offset_size; + int32_t string_start = offset_start + (num_keys + 1) * offset_size; + int64_t metadata_size = string_start + dictionary_string_size; + if (metadata_size > VariantDefs::kSizeLimit) { + return Status::Invalid("VARIANT_SIZE_LIMIT"); + } + + std::shared_ptr metadata = + Bytes::AllocateBytes(static_cast(metadata_size), pool.get()); + auto* metadata_data = reinterpret_cast(metadata->data()); + int32_t header_byte = VariantDefs::kVersion | ((offset_size - 1) << 6); + VariantBinaryUtil::WriteLong(header_byte, 1, metadata_data, 0); + VariantBinaryUtil::WriteLong(num_keys, offset_size, metadata_data, 1); + int32_t current_offset = 0; + for (int32_t i = 0; i < num_keys; ++i) { + VariantBinaryUtil::WriteLong(current_offset, offset_size, metadata_data, + offset_start + i * offset_size); + const std::string& key = dictionary_keys_[i]; + memcpy(metadata_data + string_start + current_offset, key.data(), key.size()); + current_offset += static_cast(key.size()); + } + VariantBinaryUtil::WriteLong(current_offset, offset_size, metadata_data, + offset_start + num_keys * offset_size); + + std::shared_ptr value = + Bytes::AllocateBytes(static_cast(write_pos_), pool.get()); + memcpy(value->data(), write_buffer_.data(), static_cast(write_pos_)); + return GenericVariant::Create(std::move(value), std::move(metadata)); +} + +Status VariantBuilder::AppendString(std::string_view str) { + bool long_str = static_cast(str.size()) > VariantDefs::kMaxShortStrSize; + PAIMON_RETURN_NOT_OK(CheckCapacity((long_str ? 1 + VariantDefs::kU32Size : 1) + + static_cast(str.size()))); + if (long_str) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kLongStr); + VariantBinaryUtil::WriteLong(static_cast(str.size()), VariantDefs::kU32Size, + write_buffer_.data(), write_pos_); + write_pos_ += VariantDefs::kU32Size; + } else { + write_buffer_[write_pos_++] = + VariantBinaryUtil::ShortStrHeader(static_cast(str.size())); + } + memcpy(write_buffer_.data() + write_pos_, str.data(), str.size()); + write_pos_ += static_cast(str.size()); + return Status::OK(); +} + +Status VariantBuilder::AppendNull() { + PAIMON_RETURN_NOT_OK(CheckCapacity(1)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kNull); + return Status::OK(); +} + +Status VariantBuilder::AppendBoolean(bool b) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1)); + write_buffer_[write_pos_++] = + VariantBinaryUtil::PrimitiveHeader(b ? VariantDefs::kTrue : VariantDefs::kFalse); + return Status::OK(); +} + +Status VariantBuilder::AppendLong(int64_t l) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 8)); + if (l == static_cast(l)) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kInt1); + VariantBinaryUtil::WriteLong(l, 1, write_buffer_.data(), write_pos_); + write_pos_ += 1; + } else if (l == static_cast(l)) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kInt2); + VariantBinaryUtil::WriteLong(l, 2, write_buffer_.data(), write_pos_); + write_pos_ += 2; + } else if (l == static_cast(l)) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kInt4); + VariantBinaryUtil::WriteLong(l, 4, write_buffer_.data(), write_pos_); + write_pos_ += 4; + } else { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kInt8); + VariantBinaryUtil::WriteLong(l, 8, write_buffer_.data(), write_pos_); + write_pos_ += 8; + } + return Status::OK(); +} + +Status VariantBuilder::AppendDouble(double d) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 8)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDouble); + int64_t bits; + memcpy(&bits, &d, sizeof(bits)); + VariantBinaryUtil::WriteLong(bits, 8, write_buffer_.data(), write_pos_); + write_pos_ += 8; + return Status::OK(); +} + +Status VariantBuilder::AppendDecimal(const VariantDecimal& d) { + PAIMON_RETURN_NOT_OK(CheckCapacity(2 + 16)); + int32_t precision = d.Precision(); + if (d.scale < 0 || d.scale > VariantDefs::kMaxDecimal16Precision || + precision > VariantDefs::kMaxDecimal16Precision) { + return Status::Invalid( + fmt::format("Decimal precision {} and scale {} must fit into the variant decimal " + "limit {}", + precision, d.scale, VariantDefs::kMaxDecimal16Precision)); + } + if (d.scale <= VariantDefs::kMaxDecimal4Precision && + precision <= VariantDefs::kMaxDecimal4Precision) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDecimal4); + write_buffer_[write_pos_++] = static_cast(d.scale); + VariantBinaryUtil::WriteLong(static_cast(d.unscaled), 4, write_buffer_.data(), + write_pos_); + write_pos_ += 4; + } else if (d.scale <= VariantDefs::kMaxDecimal8Precision && + precision <= VariantDefs::kMaxDecimal8Precision) { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDecimal8); + write_buffer_[write_pos_++] = static_cast(d.scale); + VariantBinaryUtil::WriteLong(static_cast(d.unscaled), 8, write_buffer_.data(), + write_pos_); + write_pos_ += 8; + } else { + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDecimal16); + write_buffer_[write_pos_++] = static_cast(d.scale); + auto bits = static_cast<__uint128_t>(d.unscaled); + for (int32_t i = 0; i < 16; ++i) { + write_buffer_[write_pos_ + i] = static_cast((bits >> (8 * i)) & 0xFF); + } + write_pos_ += 16; + } + return Status::OK(); +} + +Status VariantBuilder::AppendDate(int32_t days_since_epoch) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 4)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kDate); + VariantBinaryUtil::WriteLong(days_since_epoch, 4, write_buffer_.data(), write_pos_); + write_pos_ += 4; + return Status::OK(); +} + +Status VariantBuilder::AppendTimestamp(int64_t micros_since_epoch) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 8)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kTimestamp); + VariantBinaryUtil::WriteLong(micros_since_epoch, 8, write_buffer_.data(), write_pos_); + write_pos_ += 8; + return Status::OK(); +} + +Status VariantBuilder::AppendTimestampNtz(int64_t micros_since_epoch) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 8)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kTimestampNtz); + VariantBinaryUtil::WriteLong(micros_since_epoch, 8, write_buffer_.data(), write_pos_); + write_pos_ += 8; + return Status::OK(); +} + +Status VariantBuilder::AppendFloat(float f) { + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 4)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kFloat); + int32_t bits; + memcpy(&bits, &f, sizeof(bits)); + VariantBinaryUtil::WriteLong(bits, 4, write_buffer_.data(), write_pos_); + write_pos_ += 4; + return Status::OK(); +} + +Status VariantBuilder::AppendBinary(std::string_view binary) { + PAIMON_RETURN_NOT_OK( + CheckCapacity(1 + VariantDefs::kU32Size + static_cast(binary.size()))); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kBinary); + VariantBinaryUtil::WriteLong(static_cast(binary.size()), VariantDefs::kU32Size, + write_buffer_.data(), write_pos_); + write_pos_ += VariantDefs::kU32Size; + memcpy(write_buffer_.data() + write_pos_, binary.data(), binary.size()); + write_pos_ += static_cast(binary.size()); + return Status::OK(); +} + +Status VariantBuilder::AppendUuid(std::string_view uuid_bytes) { + if (uuid_bytes.size() != 16) { + return Status::Invalid("UUID must be 16 bytes"); + } + PAIMON_RETURN_NOT_OK(CheckCapacity(1 + 16)); + write_buffer_[write_pos_++] = VariantBinaryUtil::PrimitiveHeader(VariantDefs::kUuid); + // UUID is stored big-endian, so don't use WriteLong. + memcpy(write_buffer_.data() + write_pos_, uuid_bytes.data(), 16); + write_pos_ += 16; + return Status::OK(); +} + +int32_t VariantBuilder::AddKey(std::string_view key) { + auto it = dictionary_.find(std::string(key)); + if (it != dictionary_.end()) { + return it->second; + } + auto id = static_cast(dictionary_keys_.size()); + dictionary_.emplace(std::string(key), id); + dictionary_keys_.emplace_back(key); + return id; +} + +Status VariantBuilder::FinishWritingObject(int32_t start, std::vector* fields) { + auto size = static_cast(fields->size()); + std::sort(fields->begin(), fields->end(), + [](const FieldEntry& a, const FieldEntry& b) { return a.key < b.key; }); + int32_t max_id = size == 0 ? 0 : (*fields)[0].id; + if (allow_duplicate_keys_) { + int32_t distinct_pos = 0; + // Maintain a list of distinct keys in-place. + for (int32_t i = 1; i < size; ++i) { + max_id = std::max(max_id, (*fields)[i].id); + if ((*fields)[i].id == (*fields)[i - 1].id) { + // Found a duplicate key. Keep the field with a greater offset. + if ((*fields)[distinct_pos].offset < (*fields)[i].offset) { + (*fields)[distinct_pos].offset = (*fields)[i].offset; + } + } else { + // Found a distinct key. Add the field to the list. + ++distinct_pos; + (*fields)[distinct_pos] = (*fields)[i]; + } + } + if (distinct_pos + 1 < size) { + size = distinct_pos + 1; + fields->erase(fields->begin() + size, fields->end()); + // Sort the fields by offsets so that we can move the value data of each field to the + // new offset without overwriting the fields after it. + std::sort(fields->begin(), fields->end(), + [](const FieldEntry& a, const FieldEntry& b) { return a.offset < b.offset; }); + int32_t current_offset = 0; + for (int32_t i = 0; i < size; ++i) { + int32_t old_offset = (*fields)[i].offset; + PAIMON_ASSIGN_OR_RAISE( + int32_t field_size, + VariantBinaryUtil::ValueSize( + std::string_view(reinterpret_cast(write_buffer_.data()), + static_cast(write_pos_)), + start + old_offset)); + memmove(write_buffer_.data() + start + current_offset, + write_buffer_.data() + start + old_offset, static_cast(field_size)); + (*fields)[i].offset = current_offset; + current_offset += field_size; + } + write_pos_ = start + current_offset; + // Change back to the sort order by field keys to meet the variant spec. + std::sort(fields->begin(), fields->end(), + [](const FieldEntry& a, const FieldEntry& b) { return a.key < b.key; }); + } + } else { + for (int32_t i = 1; i < size; ++i) { + max_id = std::max(max_id, (*fields)[i].id); + if ((*fields)[i].key == (*fields)[i - 1].key) { + return Status::Invalid("VARIANT_DUPLICATE_KEY"); + } + } + } + int32_t data_size = write_pos_ - start; + bool large_size = size > VariantDefs::kU8Max; + int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; + int32_t id_size = GetIntegerSize(max_id); + int32_t offset_size = GetIntegerSize(data_size); + // The space for the header byte, object size, id list, and offset list. + int32_t header_size = 1 + size_bytes + size * id_size + (size + 1) * offset_size; + PAIMON_RETURN_NOT_OK(CheckCapacity(header_size)); + // Shift the just-written field data to make room for the object header section. + memmove(write_buffer_.data() + start + header_size, write_buffer_.data() + start, + static_cast(data_size)); + write_pos_ += header_size; + write_buffer_[start] = VariantBinaryUtil::ObjectHeader(large_size, id_size, offset_size); + VariantBinaryUtil::WriteLong(size, size_bytes, write_buffer_.data(), start + 1); + int32_t id_start = start + 1 + size_bytes; + int32_t offset_start = id_start + size * id_size; + for (int32_t i = 0; i < size; ++i) { + VariantBinaryUtil::WriteLong((*fields)[i].id, id_size, write_buffer_.data(), + id_start + i * id_size); + VariantBinaryUtil::WriteLong((*fields)[i].offset, offset_size, write_buffer_.data(), + offset_start + i * offset_size); + } + VariantBinaryUtil::WriteLong(data_size, offset_size, write_buffer_.data(), + offset_start + size * offset_size); + return Status::OK(); +} + +Status VariantBuilder::FinishWritingArray(int32_t start, const std::vector& offsets) { + int32_t data_size = write_pos_ - start; + auto size = static_cast(offsets.size()); + bool large_size = size > VariantDefs::kU8Max; + int32_t size_bytes = large_size ? VariantDefs::kU32Size : 1; + int32_t offset_size = GetIntegerSize(data_size); + // The space for the header byte, array size, and offset list. + int32_t header_size = 1 + size_bytes + (size + 1) * offset_size; + PAIMON_RETURN_NOT_OK(CheckCapacity(header_size)); + // Shift the just-written element data to make room for the header section. + memmove(write_buffer_.data() + start + header_size, write_buffer_.data() + start, + static_cast(data_size)); + write_pos_ += header_size; + write_buffer_[start] = VariantBinaryUtil::ArrayHeader(large_size, offset_size); + VariantBinaryUtil::WriteLong(size, size_bytes, write_buffer_.data(), start + 1); + int32_t offset_start = start + 1 + size_bytes; + for (int32_t i = 0; i < size; ++i) { + VariantBinaryUtil::WriteLong(offsets[i], offset_size, write_buffer_.data(), + offset_start + i * offset_size); + } + VariantBinaryUtil::WriteLong(data_size, offset_size, write_buffer_.data(), + offset_start + size * offset_size); + return Status::OK(); +} + +Status VariantBuilder::AppendVariant(const GenericVariant& v) { + return AppendVariantImpl(v.RawValue(), v.Metadata(), v.Pos()); +} + +Status VariantBuilder::AppendVariantImpl(std::string_view value, std::string_view metadata, + int32_t pos) { + PAIMON_RETURN_NOT_OK(VariantBinaryUtil::CheckIndex(pos, static_cast(value.size()))); + int32_t basic_type = static_cast(value[pos]) & VariantDefs::kBasicTypeMask; + switch (basic_type) { + case VariantDefs::kObject: { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ObjectInfo info, + VariantBinaryUtil::GetObjectInfo(value, pos)); + std::vector fields; + fields.reserve(info.num_elements); + int32_t start = write_pos_; + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE(int32_t id, + VariantBinaryUtil::ReadUnsigned( + value, info.id_start + info.id_size * i, info.id_size)); + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(value, info.offset_start + info.offset_size * i, + info.offset_size)); + int32_t element_pos = info.data_start + offset; + PAIMON_ASSIGN_OR_RAISE(std::string_view key, + VariantBinaryUtil::GetMetadataKey(metadata, id)); + int32_t new_id = AddKey(key); + fields.emplace_back(std::string(key), new_id, write_pos_ - start); + PAIMON_RETURN_NOT_OK(AppendVariantImpl(value, metadata, element_pos)); + } + return FinishWritingObject(start, &fields); + } + case VariantDefs::kArray: { + PAIMON_ASSIGN_OR_RAISE(VariantBinaryUtil::ArrayInfo info, + VariantBinaryUtil::GetArrayInfo(value, pos)); + std::vector offsets; + offsets.reserve(info.num_elements); + int32_t start = write_pos_; + for (int32_t i = 0; i < info.num_elements; ++i) { + PAIMON_ASSIGN_OR_RAISE( + int32_t offset, + VariantBinaryUtil::ReadUnsigned(value, info.offset_start + info.offset_size * i, + info.offset_size)); + int32_t element_pos = info.data_start + offset; + offsets.push_back(write_pos_ - start); + PAIMON_RETURN_NOT_OK(AppendVariantImpl(value, metadata, element_pos)); + } + return FinishWritingArray(start, offsets); + } + default: + return ShallowAppendVariant(value, pos); + } +} + +Status VariantBuilder::ShallowAppendVariant(std::string_view value, int32_t pos) { + PAIMON_ASSIGN_OR_RAISE(int32_t size, VariantBinaryUtil::ValueSize(value, pos)); + PAIMON_RETURN_NOT_OK( + VariantBinaryUtil::CheckIndex(pos + size - 1, static_cast(value.size()))); + PAIMON_RETURN_NOT_OK(CheckCapacity(size)); + memcpy(write_buffer_.data() + write_pos_, value.data() + pos, static_cast(size)); + write_pos_ += size; + return Status::OK(); +} + +Status VariantBuilder::CheckCapacity(int32_t additional) { + int32_t required = write_pos_ + additional; + if (required > static_cast(write_buffer_.size())) { + // Allocate a new buffer with a capacity of the next power of 2 of `required`. + auto new_capacity = static_cast(write_buffer_.size()); + while (new_capacity < required) { + new_capacity *= 2; + if (new_capacity > VariantDefs::kSizeLimit) { + return Status::Invalid("VARIANT_SIZE_LIMIT"); + } + } + write_buffer_.resize(static_cast(new_capacity)); + } + return Status::OK(); +} + +int32_t VariantBuilder::GetIntegerSize(int32_t value) { + if (value <= VariantDefs::kU8Max) { + return 1; + } + if (value <= VariantDefs::kU16Max) { + return 2; + } + if (value <= VariantDefs::kU24Max) { + return 3; + } + return 4; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_builder.h b/src/paimon/common/data/variant/variant_builder.h new file mode 100644 index 00000000..4c959592 --- /dev/null +++ b/src/paimon/common/data/variant/variant_builder.h @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon { + +/// Builds variant value and metadata binaries, either by parsing JSON values or by appending +/// values directly. +class VariantBuilder { + public: + /// Temporarily stores the information of a field. All fields of a JSON object are collected, + /// sorted by their keys, and then the variant object is built in sorted order. + struct FieldEntry { + std::string key; + int32_t id; + int32_t offset; + + FieldEntry(std::string key, int32_t id, int32_t offset) + : key(std::move(key)), id(id), offset(offset) {} + }; + + explicit VariantBuilder(bool allow_duplicate_keys) + : allow_duplicate_keys_(allow_duplicate_keys) {} + + /// Parses a JSON string as a variant value. + /// + /// When `allow_duplicate_keys` is true, the last occurrence of a duplicate object key wins; + /// otherwise duplicate keys make the parse fail. + static Result> ParseJson( + std::string_view json, bool allow_duplicate_keys, const std::shared_ptr& pool); + + /// Builds the variant metadata from the collected dictionary keys and returns the variant + /// result. + Result> Build(const std::shared_ptr& pool); + + /// The variant value written so far, without metadata. Used in shredding to produce a final + /// value where all shredded values refer to a common metadata. + std::string_view ValueWithoutMetadata() const { + return {reinterpret_cast(write_buffer_.data()), + static_cast(write_pos_)}; + } + + Status AppendString(std::string_view str); + Status AppendNull(); + Status AppendBoolean(bool b); + /// Appends a long value. The actual used integer type depends on the value range. + Status AppendLong(int64_t l); + Status AppendDouble(double d); + /// Appends a decimal value. Its precision and scale must fit into `kMaxDecimal16Precision`. + Status AppendDecimal(const VariantDecimal& d); + Status AppendDate(int32_t days_since_epoch); + Status AppendTimestamp(int64_t micros_since_epoch); + Status AppendTimestampNtz(int64_t micros_since_epoch); + Status AppendFloat(float f); + Status AppendBinary(std::string_view binary); + /// Appends a UUID value (16 bytes, big-endian). + Status AppendUuid(std::string_view uuid_bytes); + + /// Adds a key to the variant dictionary and returns its id. If the key already exists, the + /// dictionary is not modified. + int32_t AddKey(std::string_view key); + + /// The current write position of the variant builder. It is used together with + /// `FinishWritingObject` or `FinishWritingArray`. + int32_t GetWritePos() const { + return write_pos_; + } + + /// Finishes writing a variant object after all of its fields have already been written. The + /// process is as follows: + /// 1. The caller calls `GetWritePos` before writing any fields to obtain the `start` + /// parameter. + /// 2. The caller appends all the object fields to the builder. In the meantime, it should + /// maintain the `fields` parameter. Before appending each field, it should append an entry + /// to `fields` to record the offset of the field, computed as `GetWritePos() - start`. + /// 3. The caller calls `FinishWritingObject` to finish writing a variant object. + /// + /// This function sorts the fields by key. If there are duplicate field keys: + /// - when `allow_duplicate_keys` is true, the field with the greatest offset value (the last + /// appended one) is kept; + /// - otherwise, the call fails. + Status FinishWritingObject(int32_t start, std::vector* fields); + + /// Finishes writing a variant array after all of its elements have already been written. The + /// process is similar to that of `FinishWritingObject`. + Status FinishWritingArray(int32_t start, const std::vector& offsets); + + /// Appends a variant value. The keys of the input variant are inserted into the current + /// variant dictionary and the value is rebuilt with new field ids. For scalar values, the + /// binary slice is copied directly. + Status AppendVariant(const GenericVariant& v); + + /// Appends the variant value without rewriting or creating any metadata. This is used when + /// building an object during shredding, where there is a fixed pre-existing metadata that all + /// shredded values refer to. + Status ShallowAppendVariant(std::string_view value, int32_t pos); + + private: + Status CheckCapacity(int32_t additional); + Status AppendVariantImpl(std::string_view value, std::string_view metadata, int32_t pos); + static int32_t GetIntegerSize(int32_t value); + + // The write buffer in building the variant value. Its first `write_pos_` bytes have been + // written. + std::vector write_buffer_ = std::vector(128); + int32_t write_pos_ = 0; + // Maps keys to a monotonically increasing id. + std::unordered_map dictionary_; + // Stores all keys in `dictionary_` in the order of id. + std::vector dictionary_keys_; + const bool allow_duplicate_keys_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_defs.h b/src/paimon/common/data/variant/variant_defs.h new file mode 100644 index 00000000..45f5614f --- /dev/null +++ b/src/paimon/common/data/variant/variant_defs.h @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +namespace paimon { + +/// Constants of the Paimon Variant type and the Variant Binary Encoding, which follows the +/// parquet-format VariantEncoding.md specification (compatible with the Java / Spark +/// implementation). +/// +/// A variant is made up of 2 binaries: value and metadata. A variant value consists of a one-byte +/// header and a number of content bytes (can be zero). The header byte is divided into upper 6 +/// bits (called "type info") and lower 2 bits (called "basic type"). +/// +/// The variant metadata includes a version id and a dictionary of distinct strings +/// (case-sensitive). Its binary format is: +/// - Version: 1-byte unsigned integer. The only acceptable value is 1 currently. +/// - Dictionary size: `offset_size`-byte little-endian unsigned integer. The number of keys in the +/// dictionary. +/// - Offsets: (size + 1) * `offset_size`-byte little-endian unsigned integers. `offsets[i]` +/// represents the starting position of string i, counting starting from the address of +/// `offsets[0]`. Strings must be stored contiguously, so we don't need to store the string size, +/// instead, we compute it with `offset[i + 1] - offset[i]`. +/// - UTF-8 string data. +/// +/// A Variant field uses `struct` as its +/// underlying physical storage in Apache Arrow Schema, and is marked as the Paimon Variant +/// extension type by attaching specific **KeyValueMetadata** on the outer field. +class VariantDefs { + public: + VariantDefs() = delete; + ~VariantDefs() = delete; + + /// Metadata key identifying a Paimon extension type field (shared with BLOB). + static constexpr char kExtensionTypeKey[] = "paimon.extension.type"; + /// Metadata value identifying a Paimon Variant extension type field. + static constexpr char kExtensionTypeValue[] = "paimon.type.variant"; + + /// Name of the binary child field holding the variant value. + static constexpr char kValueFieldName[] = "value"; + /// Name of the binary child field holding the variant metadata. + static constexpr char kMetadataFieldName[] = "metadata"; + /// Name of the typed child field of a shredded variant (parquet VariantShredding.md). + static constexpr char kTypedValueFieldName[] = "typed_value"; + /// Paimon field id of the `value` child field. + static constexpr int32_t kValueFieldId = 0; + /// Paimon field id of the `metadata` child field. + static constexpr int32_t kMetadataFieldId = 1; + + static constexpr int32_t kBasicTypeBits = 2; + static constexpr int32_t kBasicTypeMask = 0x3; + static constexpr int32_t kTypeInfoMask = 0x3F; + /// The inclusive maximum value of the type info value. It is the size limit of `kShortStr`. + static constexpr int32_t kMaxShortStrSize = 0x3F; + + /// Primitive value. The type info value must be one of the primitive type values below. + static constexpr int32_t kPrimitive = 0; + /// Short string value. The type info value is the string size, which must be in + /// `[0, kMaxShortStrSize]`. The string content bytes directly follow the header byte. + static constexpr int32_t kShortStr = 1; + /// Object value. The content contains a size, a list of field ids, a list of field offsets, + /// and the actual field data. The length of the id list is `size`, while the length of the + /// offset list is `size + 1`, where the last offset represents the total size of the field + /// data. The fields in an object must be sorted by the field name in alphabetical order. + /// Duplicate field names in one object are not allowed. + /// The type info is 0_b4_b3b2_b1b0 (MSB is 0), where: + /// - b4 specifies the type of size. When it is 0/1, `size` is a little-endian 1/4-byte + /// unsigned integer. + /// - b3b2/b1b0 specifies the integer type of id and offset. When the 2 bits are 0/1/2, the + /// list contains 1/2/3-byte little-endian unsigned integers. + static constexpr int32_t kObject = 2; + /// Array value. The content contains a size, a list of field offsets, and the actual element + /// data. It is similar to an object without the id list. The type info is 000_b2_b1b0: + /// - b2 specifies the type of size. + /// - b1b0 specifies the integer type of offset. + static constexpr int32_t kArray = 3; + + /// JSON null value. Empty content. + static constexpr int32_t kNull = 0; + /// True value. Empty content. + static constexpr int32_t kTrue = 1; + /// False value. Empty content. + static constexpr int32_t kFalse = 2; + /// 1-byte little-endian signed integer. + static constexpr int32_t kInt1 = 3; + /// 2-byte little-endian signed integer. + static constexpr int32_t kInt2 = 4; + /// 4-byte little-endian signed integer. + static constexpr int32_t kInt4 = 5; + /// 8-byte little-endian signed integer. + static constexpr int32_t kInt8 = 6; + /// 8-byte IEEE double. + static constexpr int32_t kDouble = 7; + /// 4-byte decimal. Content is 1-byte scale + 4-byte little-endian signed integer. + static constexpr int32_t kDecimal4 = 8; + /// 8-byte decimal. Content is 1-byte scale + 8-byte little-endian signed integer. + static constexpr int32_t kDecimal8 = 9; + /// 16-byte decimal. Content is 1-byte scale + 16-byte little-endian signed integer. + static constexpr int32_t kDecimal16 = 10; + /// Date value. Content is 4-byte little-endian signed integer that represents the number of + /// days from the Unix epoch. + static constexpr int32_t kDate = 11; + /// Timestamp value. Content is 8-byte little-endian signed integer that represents the number + /// of microseconds elapsed since the Unix epoch, 1970-01-01 00:00:00 UTC. It is displayed to + /// users in their local time zones and may be displayed differently depending on the + /// execution environment. + static constexpr int32_t kTimestamp = 12; + /// Timestamp_ntz value. It has the same content as `kTimestamp` but should always be + /// interpreted as if the local time zone is UTC. + static constexpr int32_t kTimestampNtz = 13; + /// 4-byte IEEE float. + static constexpr int32_t kFloat = 14; + /// Binary value. The content is (4-byte little-endian unsigned integer representing the + /// binary size) + (size bytes of binary content). + static constexpr int32_t kBinary = 15; + /// Long string value. The content is (4-byte little-endian unsigned integer representing the + /// string size) + (size bytes of string content). + static constexpr int32_t kLongStr = 16; + /// UUID, 16-byte big-endian. + static constexpr int32_t kUuid = 20; + + /// The only acceptable variant version. It is stored in the lower 4 bits of the first + /// metadata byte. + static constexpr uint8_t kVersion = 1; + static constexpr uint8_t kVersionMask = 0x0F; + + static constexpr int32_t kU8Max = 0xFF; + static constexpr int32_t kU16Max = 0xFFFF; + static constexpr int32_t kU24Max = 0xFFFFFF; + static constexpr int32_t kU24Size = 3; + static constexpr int32_t kU32Size = 4; + + /// Both variant value and variant metadata need to be no longer than 128MiB. + static constexpr int32_t kSizeLimit = 128 * 1024 * 1024; + + static constexpr int32_t kMaxDecimal4Precision = 9; + static constexpr int32_t kMaxDecimal8Precision = 18; + static constexpr int32_t kMaxDecimal16Precision = 38; + + /// Object field lookup switches from linear search to binary search when the object size + /// reaches this threshold. + static constexpr int32_t kBinarySearchThreshold = 32; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_get.cpp b/src/paimon/common/data/variant/variant_get.cpp new file mode 100644 index 00000000..9bb8bab5 --- /dev/null +++ b/src/paimon/common/data/variant/variant_get.cpp @@ -0,0 +1,422 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_get.h" + +#include +#include + +#include "arrow/api.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_json_utils.h" +#include "paimon/common/data/variant/variant_path_segment.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/core/casting/cast_executor_factory.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" + +namespace paimon { + +namespace { + +Result> InvalidCast(const std::shared_ptr& variant, + const std::shared_ptr& target_type, + const VariantCastArgs& cast_args) { + if (cast_args.fail_on_error) { + PAIMON_ASSIGN_OR_RAISE(std::string json, variant->ToJson(cast_args.zone_id)); + return Status::Invalid(fmt::format("Invalid cast {} to {}", json, target_type->ToString())); + } + return std::optional(std::nullopt); +} + +Result> CastVariant(const std::shared_ptr& variant, + const std::shared_ptr& target_type, + const VariantCastArgs& cast_args) { + switch (target_type->id()) { + case arrow::Type::type::STRUCT: + case arrow::Type::type::LIST: + case arrow::Type::type::MAP: + return Status::NotImplemented(fmt::format( + "variant_get to nested type {} is not supported", target_type->ToString())); + default: + break; + } + + PAIMON_ASSIGN_OR_RAISE(VariantValueType variant_type, variant->GetType()); + if (variant_type == VariantValueType::kNull) { + return std::optional(std::nullopt); + } + + bool target_is_string = target_type->id() == arrow::Type::type::STRING; + if (variant_type == VariantValueType::kUuid) { + // There's no UUID type in Paimon. We only allow it to be cast to string. + if (target_is_string) { + PAIMON_ASSIGN_OR_RAISE(std::string_view uuid, variant->GetUuid()); + std::string uuid_str = VariantBinaryUtil::UuidToString(uuid); + return std::optional( + Literal(FieldType::STRING, uuid_str.data(), uuid_str.size())); + } + return InvalidCast(variant, target_type, cast_args); + } + + std::optional input; + std::shared_ptr input_type; + switch (variant_type) { + case VariantValueType::kObject: + case VariantValueType::kArray: { + if (target_is_string) { + PAIMON_ASSIGN_OR_RAISE(std::string json, variant->ToJson(cast_args.zone_id)); + return std::optional(Literal(FieldType::STRING, json.data(), json.size())); + } + return InvalidCast(variant, target_type, cast_args); + } + case VariantValueType::kBoolean: { + PAIMON_ASSIGN_OR_RAISE(bool value, variant->GetBoolean()); + input = Literal(value); + input_type = arrow::boolean(); + break; + } + case VariantValueType::kLong: { + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant->GetLong()); + input = Literal(value); + input_type = arrow::int64(); + break; + } + case VariantValueType::kString: { + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant->GetString()); + input = Literal(FieldType::STRING, value.data(), value.size()); + input_type = arrow::utf8(); + break; + } + case VariantValueType::kDouble: { + PAIMON_ASSIGN_OR_RAISE(double value, variant->GetDouble()); + if (target_is_string) { + // Match `GenericVariant::ToJson` and Java's `Double.toString` instead of the + // arrow cast formatting. + std::string str = VariantJsonUtils::JavaDoubleToString(value); + return std::optional(Literal(FieldType::STRING, str.data(), str.size())); + } + input = Literal(value); + input_type = arrow::float64(); + break; + } + case VariantValueType::kDecimal: { + PAIMON_ASSIGN_OR_RAISE(VariantDecimal value, variant->GetDecimal()); + if (value.scale < 0) { + // `paimon::Decimal` requires a non-negative scale; scale the value back up. + for (; value.scale < 0; ++value.scale) { + value.unscaled *= 10; + } + } + int32_t precision = std::max(value.Precision(), value.scale); + int32_t scale = value.scale; + input = Literal(Decimal(precision, scale, value.unscaled)); + input_type = arrow::decimal128(precision, scale); + break; + } + case VariantValueType::kDate: { + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant->GetLong()); + input = Literal(FieldType::DATE, static_cast(value)); + input_type = arrow::date32(); + break; + } + case VariantValueType::kFloat: { + PAIMON_ASSIGN_OR_RAISE(float value, variant->GetFloat()); + if (target_is_string) { + // Match `GenericVariant::ToJson` and Java's `Float.toString`. + std::string str = VariantJsonUtils::JavaFloatToString(value); + return std::optional(Literal(FieldType::STRING, str.data(), str.size())); + } + input = Literal(value); + input_type = arrow::float32(); + break; + } + case VariantValueType::kBinary: { + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant->GetBinary()); + input = Literal(FieldType::BINARY, value.data(), value.size()); + input_type = arrow::binary(); + break; + } + case VariantValueType::kTimestamp: + case VariantValueType::kTimestampNtz: { + PAIMON_ASSIGN_OR_RAISE(int64_t micros, variant->GetLong()); + // Floor the division so negative epochs keep a non-negative sub-millisecond part. + int64_t millis = micros / 1000; + auto sub_micros = static_cast(micros % 1000); + if (sub_micros < 0) { + millis -= 1; + sub_micros += 1000; + } + input = Literal(Timestamp::FromEpochMillis(millis, sub_micros * 1000)); + input_type = variant_type == VariantValueType::kTimestamp + ? arrow::timestamp(arrow::TimeUnit::MICRO, "UTC") + : arrow::timestamp(arrow::TimeUnit::MICRO); + break; + } + default: + return Status::Invalid(fmt::format("Unsupported variant type in variant_get: {}", + static_cast(variant_type))); + } + + if (input_type->Equals(*target_type)) { + return input; + } + + PAIMON_ASSIGN_OR_RAISE(FieldType input_field_type, + FieldTypeUtils::ConvertToFieldType(input_type->id())); + PAIMON_ASSIGN_OR_RAISE(FieldType target_field_type, + FieldTypeUtils::ConvertToFieldType(target_type->id())); + std::shared_ptr executor = + CastExecutorFactory::GetCastExecutorFactory()->GetCastExecutor(input_field_type, + target_field_type); + if (executor == nullptr) { + return InvalidCast(variant, target_type, cast_args); + } + Result cast_result = executor->Cast(*input, target_type); + if (!cast_result.ok()) { + return InvalidCast(variant, target_type, cast_args); + } + return std::optional(std::move(cast_result).value()); +} + +Status AppendLiteralToBuilder(const Literal& literal, + const std::shared_ptr& target_type, + arrow::ArrayBuilder* builder) { + switch (target_type->id()) { + case arrow::Type::type::BOOL: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::INT8: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::INT16: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::INT32: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::INT64: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::FLOAT: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::DOUBLE: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::STRING: + return ToPaimonStatus(static_cast(builder)->Append( + literal.GetValue())); + case arrow::Type::type::BINARY: + return ToPaimonStatus(static_cast(builder)->Append( + literal.GetValue())); + case arrow::Type::type::DATE32: + return ToPaimonStatus( + static_cast(builder)->Append(literal.GetValue())); + case arrow::Type::type::TIMESTAMP: { + auto timestamp = literal.GetValue(); + const auto& timestamp_type = static_cast(*target_type); + int64_t value; + switch (timestamp_type.unit()) { + case arrow::TimeUnit::SECOND: + value = timestamp.GetMillisecond() / 1000; + break; + case arrow::TimeUnit::MILLI: + value = timestamp.GetMillisecond(); + break; + case arrow::TimeUnit::MICRO: + value = timestamp.ToMicrosecond(); + break; + case arrow::TimeUnit::NANO: + value = timestamp.ToNanosecond(); + break; + default: + return Status::Invalid("Unsupported timestamp unit"); + } + return ToPaimonStatus(static_cast(builder)->Append(value)); + } + case arrow::Type::type::DECIMAL128: { + auto decimal = literal.GetValue(); + arrow::Decimal128 value(static_cast(decimal.HighBits()), decimal.LowBits()); + return ToPaimonStatus(static_cast(builder)->Append(value)); + } + default: + return Status::Invalid( + fmt::format("Unsupported variant_get target type: {}", target_type->ToString())); + } +} + +} // namespace + +Status VariantGetExecutor::CastToBuilder(const std::shared_ptr& variant, + const std::shared_ptr& target_field, + const VariantCastArgs& cast_args, + const std::shared_ptr& pool, + arrow::ArrayBuilder* builder) { + if (variant == nullptr) { + return ToPaimonStatus(builder->AppendNull()); + } + + auto invalid_cast = [&]() -> Status { + if (cast_args.fail_on_error) { + PAIMON_ASSIGN_OR_RAISE(std::string json, variant->ToJson(cast_args.zone_id)); + return Status::Invalid( + fmt::format("Invalid cast {} to {}", json, target_field->type()->ToString())); + } + return ToPaimonStatus(builder->AppendNull()); + }; + + if (VariantTypeUtils::IsVariantField(target_field)) { + VariantBuilder variant_builder(/*allow_duplicate_keys=*/false); + PAIMON_RETURN_NOT_OK(variant_builder.AppendVariant(*variant)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr copied, variant_builder.Build(pool)); + auto* struct_builder = static_cast(builder); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + PAIMON_ASSIGN_OR_RAISE(std::string_view value, copied->Value()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(struct_builder->field_builder(0))->Append(value)); + return ToPaimonStatus(static_cast(struct_builder->field_builder(1)) + ->Append(copied->Metadata())); + } + + PAIMON_ASSIGN_OR_RAISE(VariantValueType variant_type, variant->GetType()); + if (variant_type == VariantValueType::kNull) { + return ToPaimonStatus(builder->AppendNull()); + } + + switch (target_field->type()->id()) { + case arrow::Type::type::STRUCT: { + if (variant_type != VariantValueType::kObject) { + return invalid_cast(); + } + auto* struct_builder = static_cast(builder); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + const auto& struct_type = static_cast(*target_field->type()); + for (int i = 0; i < struct_type.num_fields(); ++i) { + const std::shared_ptr& child_field = struct_type.field(i); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child, + variant->GetFieldByKey(child_field->name())); + PAIMON_RETURN_NOT_OK(CastToBuilder(child, child_field, cast_args, pool, + struct_builder->field_builder(i))); + } + return Status::OK(); + } + case arrow::Type::type::MAP: { + const auto& map_type = static_cast(*target_field->type()); + if (map_type.key_type()->id() != arrow::Type::type::STRING || + variant_type != VariantValueType::kObject) { + return invalid_cast(); + } + auto* map_builder = static_cast(builder); + PAIMON_RETURN_NOT_OK_FROM_ARROW(map_builder->Append()); + PAIMON_ASSIGN_OR_RAISE(int32_t object_size, variant->ObjectSize()); + for (int32_t i = 0; i < object_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::optional field, + variant->GetFieldAtIndex(i)); + if (!field.has_value()) { + return Status::Invalid(fmt::format("Malformed variant object at index {}", i)); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(map_builder->key_builder()) + ->Append(field->key)); + PAIMON_RETURN_NOT_OK(CastToBuilder(field->value, map_type.item_field(), cast_args, + pool, map_builder->item_builder())); + } + return Status::OK(); + } + case arrow::Type::type::LIST: { + if (variant_type != VariantValueType::kArray) { + return invalid_cast(); + } + auto* list_builder = static_cast(builder); + PAIMON_RETURN_NOT_OK_FROM_ARROW(list_builder->Append()); + const auto& list_type = static_cast(*target_field->type()); + PAIMON_ASSIGN_OR_RAISE(int32_t array_size, variant->ArraySize()); + for (int32_t i = 0; i < array_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element, + variant->GetElementAtIndex(i)); + PAIMON_RETURN_NOT_OK(CastToBuilder(element, list_type.value_field(), cast_args, + pool, list_builder->value_builder())); + } + return Status::OK(); + } + default: { + PAIMON_ASSIGN_OR_RAISE(std::optional literal, + CastVariant(variant, target_field->type(), cast_args)); + if (!literal.has_value()) { + return ToPaimonStatus(builder->AppendNull()); + } + return AppendLiteralToBuilder(*literal, target_field->type(), builder); + } + } +} + +Result> VariantGetExecutor::GetAsArrow( + const std::shared_ptr& variant, const std::string& path, + const std::shared_ptr& target_field, const VariantCastArgs& cast_args, + const std::shared_ptr& pool, const std::shared_ptr& arrow_pool) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr extracted, ExtractByPath(variant, path)); + std::unique_ptr builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::MakeBuilder(arrow_pool.get(), target_field->type(), &builder)); + PAIMON_RETURN_NOT_OK(CastToBuilder(extracted, target_field, cast_args, pool, builder.get())); + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&array)); + return array; +} + +Result> VariantGetExecutor::ExtractByPath( + const std::shared_ptr& variant, const std::string& path) { + PAIMON_ASSIGN_OR_RAISE(std::vector segments, + VariantPathSegment::Parse(path)); + std::shared_ptr current = variant; + for (const VariantPathSegment& segment : segments) { + PAIMON_ASSIGN_OR_RAISE(VariantValueType type, current->GetType()); + if (segment.kind == VariantPathSegment::Kind::kObjectExtraction && + type == VariantValueType::kObject) { + PAIMON_ASSIGN_OR_RAISE(current, current->GetFieldByKey(segment.key)); + } else if (segment.kind == VariantPathSegment::Kind::kArrayExtraction && + type == VariantValueType::kArray) { + PAIMON_ASSIGN_OR_RAISE(current, current->GetElementAtIndex(segment.index)); + } else { + return std::shared_ptr(nullptr); + } + if (current == nullptr) { + return std::shared_ptr(nullptr); + } + } + return current; +} + +Result> VariantGetExecutor::Get( + const std::shared_ptr& variant, const std::string& path, + const std::shared_ptr& target_type, const VariantCastArgs& cast_args) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr extracted, ExtractByPath(variant, path)); + if (extracted == nullptr) { + return std::optional(std::nullopt); + } + return CastVariant(extracted, target_type, cast_args); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_get.h b/src/paimon/common/data/variant/variant_get.h new file mode 100644 index 00000000..9cfa8d33 --- /dev/null +++ b/src/paimon/common/data/variant/variant_get.h @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/data/variant.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/literal.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class ArrayBuilder; +class DataType; +class Field; +class MemoryPool; +} // namespace arrow + +namespace paimon { + +/// Implements `variant_get` semantics: extracting a sub-variant by a JSONPath-like path and +/// casting it to a target type. +class VariantGetExecutor { + public: + VariantGetExecutor() = delete; + ~VariantGetExecutor() = delete; + + /// Extracts a sub-variant value according to a path which starts with a `$`, e.g. `$.key`, + /// `$['key']`, `$["key"]`, `$.array[0]`. Returns nullptr if the path does not match the + /// variant structure. + static Result> ExtractByPath( + const std::shared_ptr& variant, const std::string& path); + + /// Extracts a sub-variant by `path` and casts it to `target_type`. Returns nullopt for SQL + /// NULL (unmatched path, variant null, or an invalid cast with `fail_on_error == false`). + /// + /// Nested target types (ROW/ARRAY/MAP/VARIANT) are not supported by the `Literal` result + /// type; use `CastToBuilder` / `GetAsArrow` for structured results. + static Result> Get(const std::shared_ptr& variant, + const std::string& path, + const std::shared_ptr& target_type, + const VariantCastArgs& cast_args); + + /// Casts `variant` to the type of `target_field` and appends the result to `builder`. + /// + /// Supported targets beyond scalars: a variant-marked struct field (the variant is deeply + /// re-encoded), STRUCT (from a variant object, children matched by name; unmatched children + /// are null), MAP with string keys (from a variant object), and LIST (from a variant array). + /// A nullptr `variant`, a variant null, or an invalid cast with `fail_on_error == false` + /// appends null. + static Status CastToBuilder(const std::shared_ptr& variant, + const std::shared_ptr& target_field, + const VariantCastArgs& cast_args, + const std::shared_ptr& pool, + arrow::ArrayBuilder* builder); + + /// Extracts a sub-variant by `path`, casts it to the (possibly nested) type of + /// `target_field`, and returns a length-1 arrow array holding the result (a null slot + /// represents SQL NULL). `arrow_pool` must outlive the returned array. + static Result> GetAsArrow( + const std::shared_ptr& variant, const std::string& path, + const std::shared_ptr& target_field, const VariantCastArgs& cast_args, + const std::shared_ptr& pool, + const std::shared_ptr& arrow_pool); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_get_test.cpp b/src/paimon/common/data/variant/variant_get_test.cpp new file mode 100644 index 00000000..ba8d592f --- /dev/null +++ b/src/paimon/common/data/variant/variant_get_test.cpp @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_get.h" + +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_path_segment.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/data/decimal.h" +#include "paimon/data/timestamp.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class VariantGetTest : public ::testing::Test { + public: + void SetUp() override { + // The same document as the Java GenericVariantTest#testVariantGet. + std::string json = + "{\n" + " \"object\": {\n" + " \"name\": \"Apache Paimon\",\n" + " \"age\": 2,\n" + " \"address\": {\n" + " \"street\": \"Main St\",\n" + " \"city\": \"Hangzhou\"\n" + " }\n" + " },\n" + " \"array\": [1, 2, 3, 4, 5],\n" + " \"string\": \"Hello, World!\",\n" + " \"long\": 12345678901234,\n" + " \"double\": 1.0123456789012345678901234567890123456789,\n" + " \"decimal\": 100.99,\n" + " \"boolean1\": true,\n" + " \"boolean2\": false,\n" + " \"nullField\": null\n" + "}\n"; + ASSERT_OK_AND_ASSIGN(variant_, GenericVariant::FromJson(json, pool_)); + cast_args_.fail_on_error = false; + } + + std::optional Get(const std::string& path, + const std::shared_ptr& target) { + auto result = VariantGetExecutor::Get(variant_, path, target, cast_args_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return result.value(); + } + + std::string GetString(const std::string& path, const std::shared_ptr& target) { + std::optional literal = Get(path, target); + EXPECT_TRUE(literal.has_value()); + return literal->ToString(); + } + + Result> GetAsArrow( + const std::string& path, const std::shared_ptr& target_field) { + return VariantGetExecutor::GetAsArrow(variant_, path, target_field, cast_args_, pool_, + arrow_pool_); + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); + std::shared_ptr arrow_pool_ = GetArrowPool(pool_); + std::shared_ptr variant_; + VariantCastArgs cast_args_; +}; + +TEST_F(VariantGetTest, PathSegmentParse) { + ASSERT_OK_AND_ASSIGN(auto segments, + VariantPathSegment::Parse("$[\"object\"]['address'].city[3]")); + ASSERT_EQ(segments.size(), 4); + ASSERT_EQ(segments[0].kind, VariantPathSegment::Kind::kObjectExtraction); + ASSERT_EQ(segments[0].key, "object"); + ASSERT_EQ(segments[1].kind, VariantPathSegment::Kind::kObjectExtraction); + ASSERT_EQ(segments[1].key, "address"); + ASSERT_EQ(segments[2].kind, VariantPathSegment::Kind::kObjectExtraction); + ASSERT_EQ(segments[2].key, "city"); + ASSERT_EQ(segments[3].kind, VariantPathSegment::Kind::kArrayExtraction); + ASSERT_EQ(segments[3].index, 3); + + ASSERT_OK_AND_ASSIGN(auto root_only, VariantPathSegment::Parse("$")); + ASSERT_TRUE(root_only.empty()); + + // Java parity: the root `$` is located anywhere in the path (Java uses Matcher#find), so a + // prefix before the first `$` is tolerated and ignored. + ASSERT_OK_AND_ASSIGN(auto prefixed, VariantPathSegment::Parse("abc$.x")); + ASSERT_EQ(prefixed.size(), 1); + ASSERT_EQ(prefixed[0].kind, VariantPathSegment::Kind::kObjectExtraction); + ASSERT_EQ(prefixed[0].key, "x"); + + ASSERT_NOK(VariantPathSegment::Parse("")); + ASSERT_NOK(VariantPathSegment::Parse("no_root")); + ASSERT_NOK(VariantPathSegment::Parse("$.")); + ASSERT_NOK(VariantPathSegment::Parse("$[abc]")); + ASSERT_NOK(VariantPathSegment::Parse("$['unterminated]")); +} + +TEST_F(VariantGetTest, ScalarTargets) { + ASSERT_EQ(GetString("$.string", arrow::utf8()), "Hello, World!"); + auto long_value = Get("$.long", arrow::int64()); + ASSERT_TRUE(long_value.has_value()); + ASSERT_EQ(long_value->GetValue(), 12345678901234LL); + ASSERT_EQ(GetString("$.long", arrow::utf8()), "12345678901234"); + auto double_value = Get("$.double", arrow::float64()); + ASSERT_TRUE(double_value.has_value()); + ASSERT_DOUBLE_EQ(double_value->GetValue(), 1.0123456789012346); + auto decimal_value = Get("$.decimal", arrow::decimal128(5, 2)); + ASSERT_TRUE(decimal_value.has_value()); + ASSERT_EQ(decimal_value->GetValue().ToUnscaledLong(), 10099); + ASSERT_EQ(GetString("$.decimal", arrow::utf8()), "100.99"); + auto bool1 = Get("$.boolean1", arrow::boolean()); + ASSERT_TRUE(bool1.has_value()); + ASSERT_TRUE(bool1->GetValue()); + auto bool2 = Get("$.boolean2", arrow::boolean()); + ASSERT_TRUE(bool2.has_value()); + ASSERT_FALSE(bool2->GetValue()); + // Variant null maps to SQL NULL. + ASSERT_FALSE(Get("$.nullField", arrow::boolean()).has_value()); + auto elem = Get("$.array[3]", arrow::int64()); + ASSERT_TRUE(elem.has_value()); + ASSERT_EQ(elem->GetValue(), 4); +} + +TEST_F(VariantGetTest, ContainerToJsonString) { + ASSERT_EQ(GetString("$.object", arrow::utf8()), + "{\"address\":{\"city\":\"Hangzhou\",\"street\":\"Main St\"},\"age\":2,\"name\":" + "\"Apache Paimon\"}"); + ASSERT_EQ(GetString("$.object.name", arrow::utf8()), "Apache Paimon"); + ASSERT_EQ(GetString("$.object.address.street", arrow::utf8()), "Main St"); + ASSERT_EQ(GetString("$[\"object\"]['address'].city", arrow::utf8()), "Hangzhou"); + ASSERT_EQ(GetString("$.array", arrow::utf8()), "[1,2,3,4,5]"); +} + +TEST_F(VariantGetTest, UnmatchedPathAndInvalidCast) { + // A path that does not exist yields SQL NULL. + ASSERT_FALSE(Get("$.missing", arrow::utf8()).has_value()); + ASSERT_FALSE(Get("$.string[0]", arrow::utf8()).has_value()); + ASSERT_FALSE(Get("$.array.key", arrow::utf8()).has_value()); + // An invalid cast yields SQL NULL when fail_on_error is false. + ASSERT_FALSE(Get("$.object", arrow::int64()).has_value()); + // ... and an error when fail_on_error is true. + cast_args_.fail_on_error = true; + auto result = VariantGetExecutor::Get(variant_, "$.object", arrow::int64(), cast_args_); + ASSERT_NOK(result); +} + +TEST_F(VariantGetTest, NestedTargetsNotImplemented) { + auto result = + VariantGetExecutor::Get(variant_, "$.array", arrow::list(arrow::int32()), cast_args_); + ASSERT_TRUE(result.status().IsNotImplemented()); +} + +TEST_F(VariantGetTest, BinaryAndTimestampSources) { + // Java-encoded data can carry binary/timestamp scalars that JSON parsing never produces. + { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(builder.AppendBinary(std::string_view("\x01\x02\x03", 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN(std::optional literal, + VariantGetExecutor::Get(variant, "$", arrow::binary(), cast_args_)); + ASSERT_TRUE(literal.has_value()); + ASSERT_EQ(literal->GetValue(), std::string("\x01\x02\x03", 3)); + } + { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(builder.AppendTimestamp(1700000000123456)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN( + std::optional literal, + VariantGetExecutor::Get(variant, "$", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), + cast_args_)); + ASSERT_TRUE(literal.has_value()); + ASSERT_EQ(literal->GetValue().ToMicrosecond(), 1700000000123456); + } + { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + ASSERT_OK(builder.AppendTimestampNtz(-1001)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, builder.Build(pool_)); + ASSERT_OK_AND_ASSIGN( + std::optional literal, + VariantGetExecutor::Get(variant, "$", arrow::timestamp(arrow::TimeUnit::MICRO), + cast_args_)); + ASSERT_TRUE(literal.has_value()); + // Negative epochs must floor, not truncate toward zero. + ASSERT_EQ(literal->GetValue().ToMicrosecond(), -1001); + } +} + +TEST_F(VariantGetTest, ExtractByPath) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr address, + VariantGetExecutor::ExtractByPath(variant_, "$.object.address")); + ASSERT_NE(address, nullptr); + ASSERT_OK_AND_ASSIGN(std::string json, address->ToJson()); + ASSERT_EQ(json, "{\"city\":\"Hangzhou\",\"street\":\"Main St\"}"); + ASSERT_OK_AND_ASSIGN(std::shared_ptr missing, + VariantGetExecutor::ExtractByPath(variant_, "$.object.missing")); + ASSERT_EQ(missing, nullptr); +} + +TEST_F(VariantGetTest, CastToStructTarget) { + auto target = arrow::field( + "r", arrow::struct_( + {arrow::field("name", arrow::utf8()), arrow::field("age", arrow::int64()), + arrow::field("address", arrow::struct_({arrow::field("city", arrow::utf8())})), + arrow::field("missing", arrow::utf8())})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.object", target)); + ASSERT_EQ(array->length(), 1); + const auto& row = static_cast(*array); + ASSERT_FALSE(row.IsNull(0)); + ASSERT_EQ(static_cast(*row.field(0)).GetString(0), "Apache Paimon"); + ASSERT_EQ(static_cast(*row.field(1)).Value(0), 2); + const auto& address = static_cast(*row.field(2)); + ASSERT_EQ(static_cast(*address.field(0)).GetString(0), "Hangzhou"); + // A target field absent from the variant object is null. + ASSERT_TRUE(row.field(3)->IsNull(0)); + + // With fail_on_error == false a child that cannot cast becomes null while the parent row + // stays non-null. + auto mixed_target = arrow::field("r", arrow::struct_({arrow::field("string", arrow::int64()), + arrow::field("long", arrow::int64())})); + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$", mixed_target)); + const auto& mixed_row = static_cast(*array); + ASSERT_FALSE(mixed_row.IsNull(0)); + // "string" holds "Hello, World!", which cannot cast to int64. + ASSERT_TRUE(mixed_row.field(0)->IsNull(0)); + ASSERT_FALSE(mixed_row.field(1)->IsNull(0)); + ASSERT_EQ(static_cast(*mixed_row.field(1)).Value(0), 12345678901234); +} + +TEST_F(VariantGetTest, CastToStructFromNonObject) { + auto target = arrow::field("r", arrow::struct_({arrow::field("a", arrow::int64())})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.array", target)); + ASSERT_TRUE(array->IsNull(0)); + cast_args_.fail_on_error = true; + auto result = + VariantGetExecutor::GetAsArrow(variant_, "$.array", target, cast_args_, pool_, arrow_pool_); + ASSERT_NOK(result); +} + +TEST_F(VariantGetTest, CastToMapTarget) { + auto target = arrow::field("m", arrow::map(arrow::utf8(), arrow::utf8())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + GetAsArrow("$.object.address", target)); + const auto& map = static_cast(*array); + ASSERT_EQ(map.value_length(0), 2); + const auto& keys = static_cast(*map.keys()); + const auto& items = static_cast(*map.items()); + ASSERT_EQ(keys.GetString(0), "city"); + ASSERT_EQ(items.GetString(0), "Hangzhou"); + ASSERT_EQ(keys.GetString(1), "street"); + ASSERT_EQ(items.GetString(1), "Main St"); + // A map with a non-string key type is an invalid cast. + auto bad_target = arrow::field("m", arrow::map(arrow::int64(), arrow::utf8())); + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$.object.address", bad_target)); + ASSERT_TRUE(array->IsNull(0)); +} + +TEST_F(VariantGetTest, CastToListTarget) { + auto target = arrow::field("l", arrow::list(arrow::int64())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.array", target)); + const auto& list = static_cast(*array); + ASSERT_EQ(list.value_length(0), 5); + const auto& values = static_cast(*list.values()); + for (int64_t i = 0; i < 5; ++i) { + ASSERT_EQ(values.Value(i), i + 1); + } + auto list_of_structs = + arrow::field("l", arrow::list(arrow::struct_({arrow::field("a", arrow::int64())}))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr rows, + GenericVariant::FromJson("[{\"a\": 1}, {\"a\": 2}]", pool_)); + ASSERT_OK_AND_ASSIGN(array, VariantGetExecutor::GetAsArrow(rows, "$", list_of_structs, + cast_args_, pool_, arrow_pool_)); + const auto& struct_list = static_cast(*array); + ASSERT_EQ(struct_list.value_length(0), 2); + const auto& elements = static_cast(*struct_list.values()); + ASSERT_EQ(static_cast(*elements.field(0)).Value(0), 1); + ASSERT_EQ(static_cast(*elements.field(0)).Value(1), 2); +} + +TEST_F(VariantGetTest, CastToVariantTarget) { + // Re-encodes the extracted sub-variant against a fresh metadata dictionary. + auto target = VariantTypeUtils::ToArrowField("v", /*nullable=*/true, {}); + auto read_variant_json = [&](const arrow::Array& array) { + const auto& row = static_cast(array); + EXPECT_FALSE(row.IsNull(0)); + std::string_view value = static_cast(*row.field(0)).GetView(0); + std::string_view metadata = + static_cast(*row.field(1)).GetView(0); + EXPECT_OK_AND_ASSIGN(std::shared_ptr copied, + GenericVariant::Create(value, metadata, pool_)); + EXPECT_OK_AND_ASSIGN(std::string json, copied->ToJson()); + return json; + }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + GetAsArrow("$.object.address", target)); + ASSERT_EQ(read_variant_json(*array), "{\"city\":\"Hangzhou\",\"street\":\"Main St\"}"); + + // A variant null cast to a VARIANT target stays an encoded variant null (a non-null row + // whose value renders as JSON null); only scalar targets turn a variant null into SQL NULL. + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$.nullField", target)); + ASSERT_EQ(read_variant_json(*array), "null"); + // An unmatched path is SQL NULL by contrast. + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$.missing", target)); + ASSERT_TRUE(array->IsNull(0)); +} + +TEST_F(VariantGetTest, NestedTargetNullSemantics) { + auto target = arrow::field("r", arrow::struct_({arrow::field("a", arrow::int64())})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.missing", target)); + ASSERT_TRUE(array->IsNull(0)); + // A variant null yields SQL NULL even with fail_on_error == true. + cast_args_.fail_on_error = true; + ASSERT_OK_AND_ASSIGN(array, GetAsArrow("$.nullField", target)); + ASSERT_TRUE(array->IsNull(0)); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_json_utils.cpp b/src/paimon/common/data/variant/variant_json_utils.cpp new file mode 100644 index 00000000..897c000e --- /dev/null +++ b/src/paimon/common/data/variant/variant_json_utils.cpp @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_json_utils.h" + +#include +#include +#include +#include + +#include "arrow/vendored/datetime.h" +#include "fmt/format.h" + +namespace paimon { + +namespace { + +constexpr int64_t kMicrosPerSecond = 1000000LL; +constexpr int64_t kMicrosPerDay = 86400LL * kMicrosPerSecond; + +// Formats the shortest round-trip decimal digits + decimal exponent into the textual form of +// `java.lang.Double.toString` / `java.lang.Float.toString`: plain notation when +// `1e-3 <= |value| < 1e7`, computerized scientific notation otherwise. +std::string JavaFloatingToString(std::string_view digits, int32_t exp, bool negative) { + std::string result; + if (negative) { + result.push_back('-'); + } + if (exp >= -3 && exp < 7) { + if (exp >= 0) { + size_t int_digits = static_cast(exp) + 1; + if (digits.size() > int_digits) { + result.append(digits, 0, int_digits); + result.push_back('.'); + result.append(digits.substr(int_digits)); + } else { + result.append(digits); + result.append(int_digits - digits.size(), '0'); + result.append(".0"); + } + } else { + result.append("0."); + result.append(static_cast(-exp) - 1, '0'); + result.append(digits); + } + } else { + result.push_back(digits[0]); + result.push_back('.'); + if (digits.size() > 1) { + result.append(digits.substr(1)); + } else { + result.push_back('0'); + } + result.push_back('E'); + result.append(std::to_string(exp)); + } + return result; +} + +template +std::string FloatingToJavaString(T value) { + if (std::isnan(value)) { + return "NaN"; + } + if (std::isinf(value)) { + return value > 0 ? "Infinity" : "-Infinity"; + } + if (value == 0) { + return std::signbit(value) ? "-0.0" : "0.0"; + } + // The shortest round-trip representation in scientific form, e.g. `1.234e+05`. gcc 8's + // lacks floating-point to_chars, so probe increasing precision until the value + // round-trips; dropping a trailing zero digit is an exact rounding, so the first precision + // that round-trips carries no trailing zeros. + constexpr int kMaxFractionDigits = std::is_same_v ? 8 : 16; + char buf[64]; + int len = 0; + for (int precision = 0; precision <= kMaxFractionDigits; ++precision) { + len = std::snprintf(buf, sizeof(buf), "%.*e", precision, static_cast(value)); + T parsed; + if constexpr (std::is_same_v) { + parsed = std::strtof(buf, nullptr); + } else { + parsed = std::strtod(buf, nullptr); + } + if (parsed == value) { + break; + } + } + auto parse = [](std::string_view repr, std::string* digits, int32_t* exp, bool* negative) { + *negative = repr[0] == '-'; + if (*negative) { + repr.remove_prefix(1); + } + size_t e_pos = repr.find('e'); + std::string_view mantissa = repr.substr(0, e_pos); + size_t exp_start = e_pos + 1; + // `std::from_chars` does not accept a leading '+'. + if (repr[exp_start] == '+') { + ++exp_start; + } + *exp = 0; + std::from_chars(repr.data() + exp_start, repr.data() + repr.size(), *exp); + digits->clear(); + digits->push_back(mantissa[0]); + if (mantissa.size() > 2) { + digits->append(mantissa.substr(2)); + } + }; + std::string digits; + int32_t exp = 0; + bool negative = false; + parse(std::string_view(buf, static_cast(len)), &digits, &exp, &negative); + if (digits.size() == 1 && (exp < -3 || exp >= 8)) { + // Java's FloatingDecimal emits at least two significant digits when the first digit + // alone would terminate in scientific form; re-render the correctly rounded two-digit + // representation (e.g. `Double.MIN_VALUE` is `4.9E-324`, not `5.0E-324`). + len = std::snprintf(buf, sizeof(buf), "%.1e", static_cast(value)); + parse(std::string_view(buf, static_cast(len)), &digits, &exp, &negative); + } + return JavaFloatingToString(digits, exp, negative); +} + +// Appends a year like `java.time.LocalDate.toString`: absolute value padded to at least 4 +// digits; years above 9999 get a `+` prefix. +void AppendJavaYear(int64_t year, std::string* out) { + if (year < 0) { + out->push_back('-'); + year = -year; + out->append(fmt::format("{:04}", year)); + } else { + if (year > 9999) { + out->push_back('+'); + } + out->append(fmt::format("{:04}", year)); + } +} + +void AppendDate(int64_t days_since_epoch, std::string* out) { + using arrow_vendored::date::days; + using arrow_vendored::date::sys_days; + using arrow_vendored::date::year_month_day; + year_month_day ymd{sys_days{days{days_since_epoch}}}; + AppendJavaYear(static_cast(ymd.year()), out); + out->append(fmt::format("-{:02}-{:02}", static_cast(ymd.month()), + static_cast(ymd.day()))); +} + +int64_t FloorDiv(int64_t x, int64_t y) { + int64_t quotient = x / y; + if ((x % y != 0) && ((x < 0) != (y < 0))) { + --quotient; + } + return quotient; +} + +} // namespace + +void VariantJsonUtils::AppendEscapedJson(std::string_view str, std::string* out) { + out->push_back('"'); + for (char c : str) { + switch (c) { + case '"': + out->append("\\\""); + break; + case '\\': + out->append("\\\\"); + break; + case '\b': + out->append("\\b"); + break; + case '\f': + out->append("\\f"); + break; + case '\n': + out->append("\\n"); + break; + case '\r': + out->append("\\r"); + break; + case '\t': + out->append("\\t"); + break; + default: + if (static_cast(c) < 0x20) { + out->append(fmt::format("\\u{:04x}", static_cast(c))); + } else { + out->push_back(c); + } + } + } + out->push_back('"'); +} + +std::string VariantJsonUtils::JavaDoubleToString(double value) { + return FloatingToJavaString(value); +} + +std::string VariantJsonUtils::JavaFloatToString(float value) { + return FloatingToJavaString(value); +} + +std::string VariantJsonUtils::DateToString(int32_t days_since_epoch) { + std::string result; + AppendDate(days_since_epoch, &result); + return result; +} + +std::string VariantJsonUtils::TimestampToString(int64_t micros_since_epoch, int32_t offset_seconds, + bool with_offset) { + int64_t local_micros = micros_since_epoch + static_cast(offset_seconds) * 1000000; + int64_t days = FloorDiv(local_micros, kMicrosPerDay); + int64_t micros_of_day = local_micros - days * kMicrosPerDay; + std::string result; + AppendDate(days, &result); + int64_t seconds_of_day = micros_of_day / kMicrosPerSecond; + int64_t micros_of_second = micros_of_day % kMicrosPerSecond; + result.append(fmt::format(" {:02}:{:02}:{:02}", seconds_of_day / 3600, + (seconds_of_day / 60) % 60, seconds_of_day % 60)); + if (micros_of_second != 0) { + std::string fraction = fmt::format("{:06}", micros_of_second); + while (fraction.back() == '0') { + fraction.pop_back(); + } + result.push_back('.'); + result.append(fraction); + } + if (with_offset) { + int32_t abs_offset = offset_seconds >= 0 ? offset_seconds : -offset_seconds; + result.append(fmt::format("{}{:02}:{:02}", offset_seconds >= 0 ? '+' : '-', + abs_offset / 3600, (abs_offset / 60) % 60)); + } + return result; +} + +Result VariantJsonUtils::GetZoneOffsetSeconds(const std::string& zone_id, + int64_t micros_since_epoch) { + std::string_view id = zone_id; + if (id == "Z" || id == "UTC" || id == "GMT" || id == "UT") { + return 0; + } + // `UTC+08:00` style ids: strip the prefix and parse the remaining fixed offset. + if (id.size() > 3 && (id.substr(0, 3) == "UTC" || id.substr(0, 3) == "GMT")) { + id.remove_prefix(3); + } else if (id.size() > 2 && id.substr(0, 2) == "UT" && (id[2] == '+' || id[2] == '-')) { + id.remove_prefix(2); + } + if (!id.empty() && (id[0] == '+' || id[0] == '-')) { + bool negative = id[0] == '-'; + id.remove_prefix(1); + // Accepted forms: H, HH, HH:MM, HHMM, HH:MM:SS, HHMMSS. + std::string digits; + for (char c : id) { + if (c >= '0' && c <= '9') { + digits.push_back(c); + } else if (c != ':') { + return Status::Invalid(fmt::format("Invalid zone offset: {}", zone_id)); + } + } + int32_t hours = 0; + int32_t minutes = 0; + int32_t seconds = 0; + if (digits.size() == 1 || digits.size() == 2) { + hours = std::stoi(digits); + } else if (digits.size() == 4) { + hours = std::stoi(digits.substr(0, 2)); + minutes = std::stoi(digits.substr(2, 2)); + } else if (digits.size() == 6) { + hours = std::stoi(digits.substr(0, 2)); + minutes = std::stoi(digits.substr(2, 2)); + seconds = std::stoi(digits.substr(4, 2)); + } else { + return Status::Invalid(fmt::format("Invalid zone offset: {}", zone_id)); + } + if (hours > 18 || minutes > 59 || seconds > 59) { + return Status::Invalid(fmt::format("Invalid zone offset: {}", zone_id)); + } + int32_t total = hours * 3600 + minutes * 60 + seconds; + return negative ? -total : total; + } + // IANA region id, resolved at the given instant (honoring DST). + try { + const auto* zone = arrow_vendored::date::locate_zone(zone_id); + std::chrono::time_point tp{ + std::chrono::microseconds(micros_since_epoch)}; + auto info = zone->get_info(tp); + return static_cast(info.offset.count()); + } catch (const std::exception& e) { + return Status::Invalid(fmt::format("Invalid zone id: {}, {}", zone_id, e.what())); + } +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_json_utils.h b/src/paimon/common/data/variant/variant_json_utils.h new file mode 100644 index 00000000..bf40a3e2 --- /dev/null +++ b/src/paimon/common/data/variant/variant_json_utils.h @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { + +/// Helpers for rendering variant values as JSON text, matching the output of the Java +/// implementation (`GenericVariant.toJsonImpl`) character-for-character. +class VariantJsonUtils { + public: + VariantJsonUtils() = delete; + ~VariantJsonUtils() = delete; + + /// Appends `str` as a quoted JSON string with escaping into `out`. + static void AppendEscapedJson(std::string_view str, std::string* out); + + /// Formats a double like `java.lang.Double.toString`, e.g. `1.0`, `-0.001`, `1.0E7`. + static std::string JavaDoubleToString(double value); + + /// Formats a float like `java.lang.Float.toString`. + static std::string JavaFloatToString(float value); + + /// Formats days-since-epoch like `java.time.LocalDate.toString`, e.g. `2024-01-15`. + static std::string DateToString(int32_t days_since_epoch); + + /// Formats microseconds-since-epoch at the given offset like the Java formatter + /// `yyyy-MM-dd HH:mm:ss[.fraction]` (fraction with trailing zeros trimmed), optionally + /// followed by a `+HH:MM` offset suffix. + static std::string TimestampToString(int64_t micros_since_epoch, int32_t offset_seconds, + bool with_offset); + + /// Resolves the UTC offset (in seconds) of `zone_id` at the given instant. Supports fixed + /// offsets (`+08:00`, `-05:30`, optionally prefixed with `UTC`/`GMT`), `Z`, `UTC`, `GMT`, + /// and IANA region ids such as `Asia/Shanghai`. + static Result GetZoneOffsetSeconds(const std::string& zone_id, + int64_t micros_since_epoch); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_json_utils_test.cpp b/src/paimon/common/data/variant/variant_json_utils_test.cpp new file mode 100644 index 00000000..e54c37d7 --- /dev/null +++ b/src/paimon/common/data/variant/variant_json_utils_test.cpp @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_json_utils.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(VariantJsonUtilsTest, JavaDoubleToString) { + // Mirrors java.lang.Double#toString exactly. + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(0.0), "0.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(-0.0), "-0.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::numeric_limits::infinity()), + "Infinity"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(-std::numeric_limits::infinity()), + "-Infinity"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::nan("")), "NaN"); + // Double.MIN_VALUE (the smallest positive subnormal) and Double.MAX_VALUE. + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::numeric_limits::denorm_min()), + "4.9E-324"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::numeric_limits::max()), + "1.7976931348623157E308"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(std::numeric_limits::min()), + "2.2250738585072014E-308"); + + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1.0), "1.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(-1.0), "-1.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(-0.001), "-0.001"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1e7), "1.0E7"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1234567.0), "1234567.0"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(0.001), "0.001"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1.0E-3), "0.001"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(1.0E-4), "1.0E-4"); + EXPECT_EQ(VariantJsonUtils::JavaDoubleToString(0.1 + 0.2), "0.30000000000000004"); +} + +TEST(VariantJsonUtilsTest, JavaFloatToString) { + // Mirrors java.lang.Float#toString exactly. + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(0.0F), "0.0"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(-0.0F), "-0.0"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(std::numeric_limits::infinity()), + "Infinity"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(-std::numeric_limits::infinity()), + "-Infinity"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(std::nanf("")), "NaN"); + // Float.MIN_VALUE (the smallest positive subnormal) and Float.MAX_VALUE. + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(std::numeric_limits::denorm_min()), + "1.4E-45"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(std::numeric_limits::max()), + "3.4028235E38"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(1.0F), "1.0"); + EXPECT_EQ(VariantJsonUtils::JavaFloatToString(2.5F), "2.5"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_path_segment.cpp b/src/paimon/common/data/variant/variant_path_segment.cpp new file mode 100644 index 00000000..2dbf6d8d --- /dev/null +++ b/src/paimon/common/data/variant/variant_path_segment.cpp @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_path_segment.h" + +#include +#include + +#include "fmt/format.h" + +namespace paimon { + +namespace { + +// Matches `[123]` at the beginning of `remaining` and consumes it. +bool TryParseIndex(std::string_view* remaining, int32_t* index) { + std::string_view s = *remaining; + if (s.empty() || s[0] != '[') { + return false; + } + size_t i = 1; + int64_t value = 0; + size_t digits = 0; + while (i < s.size() && s[i] >= '0' && s[i] <= '9') { + value = value * 10 + (s[i] - '0'); + if (value > std::numeric_limits::max()) { + return false; + } + ++i; + ++digits; + } + if (digits == 0 || i >= s.size() || s[i] != ']') { + return false; + } + *index = static_cast(value); + remaining->remove_prefix(i + 1); + return true; +} + +// Matches `['key']` or `["key"]` at the beginning of `remaining` and consumes it. +bool TryParseQuotedKey(std::string_view* remaining, std::string* key) { + std::string_view s = *remaining; + if (s.size() < 4 || s[0] != '[' || (s[1] != '\'' && s[1] != '"')) { + return false; + } + char quote = s[1]; + size_t end = s.find(quote, 2); + if (end == std::string_view::npos || end == 2 || end + 1 >= s.size() || s[end + 1] != ']') { + return false; + } + key->assign(s.substr(2, end - 2)); + remaining->remove_prefix(end + 2); + return true; +} + +// Matches `.key` (one or more characters that are neither `.` nor `[`) at the beginning of +// `remaining` and consumes it. +bool TryParseDotKey(std::string_view* remaining, std::string* key) { + std::string_view s = *remaining; + if (s.size() < 2 || s[0] != '.') { + return false; + } + size_t end = 1; + while (end < s.size() && s[end] != '.' && s[end] != '[') { + ++end; + } + if (end == 1) { + return false; + } + key->assign(s.substr(1, end - 1)); + remaining->remove_prefix(end); + return true; +} + +} // namespace + +Result> VariantPathSegment::Parse(const std::string& path) { + // Mirrors the Java parser: the root `$` is located with a find, so segments are parsed + // after the FIRST `$` and any prefix before it is ignored. + size_t root = path.find('$'); + if (path.empty() || root == std::string::npos) { + return Status::Invalid(fmt::format("Invalid path: {}", path)); + } + std::string_view remaining = std::string_view(path).substr(root + 1); + std::vector segments; + while (!remaining.empty()) { + int32_t index = 0; + if (TryParseIndex(&remaining, &index)) { + segments.push_back(ArrayExtraction(index)); + continue; + } + std::string key; + if (TryParseDotKey(&remaining, &key) || TryParseQuotedKey(&remaining, &key)) { + segments.push_back(ObjectExtraction(std::move(key))); + continue; + } + return Status::Invalid(fmt::format("Invalid path: {}", path)); + } + return segments; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_path_segment.h b/src/paimon/common/data/variant/variant_path_segment.h new file mode 100644 index 00000000..fa012d7e --- /dev/null +++ b/src/paimon/common/data/variant/variant_path_segment.h @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" + +namespace paimon { + +/// A path segment for variant get, representing either an object key access or an array index +/// access. +struct VariantPathSegment { + enum class Kind { kObjectExtraction, kArrayExtraction }; + + Kind kind; + /// The object key when `kind` is `kObjectExtraction`. + std::string key; + /// The array index when `kind` is `kArrayExtraction`. + int32_t index = 0; + + static VariantPathSegment ObjectExtraction(std::string key) { + VariantPathSegment segment; + segment.kind = Kind::kObjectExtraction; + segment.key = std::move(key); + return segment; + } + + static VariantPathSegment ArrayExtraction(int32_t index) { + VariantPathSegment segment; + segment.kind = Kind::kArrayExtraction; + segment.index = index; + return segment; + } + + /// Parses a path starting with `$`. Supported segments after the root are `.key`, `['key']`, + /// `["key"]` and `[index]`, e.g. `$.user.addresses[0]['city']`. + static Result> Parse(const std::string& path); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_reassembler.cpp b/src/paimon/common/data/variant/variant_reassembler.cpp new file mode 100644 index 00000000..7d301ff7 --- /dev/null +++ b/src/paimon/common/data/variant/variant_reassembler.cpp @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_reassembler.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/memory/memory_pool.h" + +namespace paimon { + +namespace { + +// A row view over a shredded struct array, the Arrow analog of the Java `ShreddedRow`. +struct Cursor { + const arrow::StructArray* array; + int64_t row; + + bool IsNullAt(int32_t field) const { + return array->field(field)->IsNull(row); + } + + std::string_view GetBinary(int32_t field) const { + return static_cast(*array->field(field)).GetView(row); + } +}; + +Status Rebuild(const Cursor& cursor, std::string_view metadata, const VariantSchema& schema, + const std::shared_ptr& pool, VariantBuilder* builder); + +Status RebuildTypedScalar(const Cursor& cursor, const VariantSchema& schema, + VariantBuilder* builder) { + int32_t typed_idx = schema.typed_idx; + const arrow::Array& typed_array = *cursor.array->field(typed_idx); + int64_t row = cursor.row; + const VariantSchema::ScalarType& scalar = schema.scalar_schema.value(); + switch (scalar.kind) { + case VariantSchema::ScalarKind::kString: + return builder->AppendString( + static_cast(typed_array).GetView(row)); + case VariantSchema::ScalarKind::kByte: + return builder->AppendLong( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kShort: + return builder->AppendLong( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kInt: + return builder->AppendLong( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kLong: + return builder->AppendLong( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kFloat: + return builder->AppendFloat( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kDouble: + return builder->AppendDouble( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kBoolean: + return builder->AppendBoolean( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kBinary: + return builder->AppendBinary( + static_cast(typed_array).GetView(row)); + case VariantSchema::ScalarKind::kDecimal: { + const auto& decimal_array = static_cast(typed_array); + arrow::Decimal128 value(decimal_array.GetValue(row)); + VariantDecimal decimal; + decimal.unscaled = (static_cast<__int128_t>(value.high_bits()) << 64) | + static_cast<__int128_t>(static_cast<__uint128_t>(value.low_bits())); + decimal.scale = scalar.scale; + return builder->AppendDecimal(decimal); + } + case VariantSchema::ScalarKind::kDate: + return builder->AppendDate( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kTimestampLtz: + return builder->AppendTimestamp( + static_cast(typed_array).Value(row)); + case VariantSchema::ScalarKind::kTimestampNtz: + return builder->AppendTimestampNtz( + static_cast(typed_array).Value(row)); + default: + return Status::NotImplemented("unsupported variant scalar kind in reassembly"); + } +} + +// Rebuilds a variant value from the shredded data according to the reconstruction algorithm in +// the parquet-format VariantShredding.md specification, appending the result to `builder`. +Status Rebuild(const Cursor& cursor, std::string_view metadata, const VariantSchema& schema, + const std::shared_ptr& pool, VariantBuilder* builder) { + int32_t typed_idx = schema.typed_idx; + int32_t variant_idx = schema.variant_idx; + if (typed_idx >= 0 && !cursor.IsNullAt(typed_idx)) { + if (schema.scalar_schema.has_value()) { + return RebuildTypedScalar(cursor, schema, builder); + } else if (schema.array_schema != nullptr) { + const auto& list_array = + static_cast(*cursor.array->field(typed_idx)); + const auto& element_array = + static_cast(*list_array.values()); + int64_t element_start = list_array.value_offset(cursor.row); + int64_t element_end = list_array.value_offset(cursor.row + 1); + int32_t start = builder->GetWritePos(); + std::vector offsets; + offsets.reserve(element_end - element_start); + for (int64_t i = element_start; i < element_end; ++i) { + offsets.push_back(builder->GetWritePos() - start); + PAIMON_RETURN_NOT_OK(Rebuild(Cursor{&element_array, i}, metadata, + *schema.array_schema, pool, builder)); + } + return builder->FinishWritingArray(start, offsets); + } else { + const auto& object_array = + static_cast(*cursor.array->field(typed_idx)); + Cursor object_cursor{&object_array, cursor.row}; + std::vector fields; + int32_t start = builder->GetWritePos(); + for (size_t field_idx = 0; field_idx < schema.object_schema.size(); ++field_idx) { + // Shredded fields must not be null. + if (object_cursor.IsNullAt(static_cast(field_idx))) { + return VariantBinaryUtil::MalformedVariant( + "a shredded object field group is null"); + } + const std::string& field_name = schema.object_schema[field_idx].name; + const VariantSchema& field_schema = *schema.object_schema[field_idx].schema; + const auto& field_array = static_cast( + *object_array.field(static_cast(field_idx))); + Cursor field_cursor{&field_array, cursor.row}; + // If the field doesn't have a non-null `typed_value` or `value`, it is missing. + if ((field_schema.typed_idx >= 0 && + !field_cursor.IsNullAt(field_schema.typed_idx)) || + (field_schema.variant_idx >= 0 && + !field_cursor.IsNullAt(field_schema.variant_idx))) { + int32_t id = builder->AddKey(field_name); + fields.emplace_back(field_name, id, builder->GetWritePos() - start); + PAIMON_RETURN_NOT_OK( + Rebuild(field_cursor, metadata, field_schema, pool, builder)); + } + } + if (variant_idx >= 0 && !cursor.IsNullAt(variant_idx)) { + // Add the leftover fields in the variant binary. + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr leftover, + GenericVariant::Create(cursor.GetBinary(variant_idx), metadata, pool)); + PAIMON_ASSIGN_OR_RAISE(VariantValueType leftover_type, leftover->GetType()); + if (leftover_type != VariantValueType::kObject) { + return VariantBinaryUtil::MalformedVariant( + "the value column of a shredded object is not an object"); + } + PAIMON_ASSIGN_OR_RAISE(int32_t leftover_size, leftover->ObjectSize()); + for (int32_t i = 0; i < leftover_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::optional field, + leftover->GetFieldAtIndex(i)); + if (!field.has_value()) { + return VariantBinaryUtil::MalformedVariant( + "a leftover object field is missing"); + } + // `value` must not contain any shredded field. + if (schema.object_schema_map.count(field->key) > 0) { + return VariantBinaryUtil::MalformedVariant(fmt::format( + "the value column duplicates the shredded field '{}'", field->key)); + } + int32_t id = builder->AddKey(field->key); + fields.emplace_back(field->key, id, builder->GetWritePos() - start); + PAIMON_RETURN_NOT_OK(builder->AppendVariant(*field->value)); + } + } + return builder->FinishWritingObject(start, &fields); + } + } else if (variant_idx >= 0 && !cursor.IsNullAt(variant_idx)) { + // `typed_value` doesn't exist or is null. Read from `value`. + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(cursor.GetBinary(variant_idx), metadata, pool)); + return builder->AppendVariant(*variant); + } else { + // The variant is missing in a context where it must be present; the data is invalid. + return VariantBinaryUtil::MalformedVariant( + "both typed_value and value of a required variant are null"); + } +} + +} // namespace + +Status VariantReassembler::RebuildValue(const arrow::StructArray& shredded, int64_t row, + std::string_view metadata, const VariantSchema& schema, + const std::shared_ptr& pool, + VariantBuilder* builder) { + return Rebuild(Cursor{&shredded, row}, metadata, schema, pool, builder); +} + +Result> VariantReassembler::AssembleVariantArray( + const std::shared_ptr& shredded, + const std::shared_ptr& schema, const std::shared_ptr& pool, + arrow::MemoryPool* arrow_pool) { + if (schema->top_level_metadata_idx < 0) { + return VariantBinaryUtil::MalformedVariant("a shredded file column misses metadata"); + } + auto output_type = VariantTypeUtils::UnshreddedStructType(); + std::unique_ptr output_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::MakeBuilder(arrow_pool, output_type, &output_builder)); + auto* struct_builder = static_cast(output_builder.get()); + auto* value_builder = static_cast(struct_builder->field_builder(0)); + auto* metadata_builder = static_cast(struct_builder->field_builder(1)); + + bool unshredded = schema->IsUnshredded(); + for (int64_t row = 0; row < shredded->length(); ++row) { + if (shredded->IsNull(row)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->AppendNull()); + continue; + } + Cursor cursor{shredded.get(), row}; + if (cursor.IsNullAt(schema->top_level_metadata_idx)) { + return VariantBinaryUtil::MalformedVariant("the variant metadata column is null"); + } + std::string_view metadata = cursor.GetBinary(schema->top_level_metadata_idx); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + if (unshredded) { + // Rebuilding is unnecessary for unshredded variants. + // TODO(nicholas): avoid copying the value/metadata binaries through the builder for + // unshredded files; the physical arrays could be returned directly with at most a + // per-row malformed-variant check. + if (cursor.IsNullAt(schema->variant_idx)) { + return VariantBinaryUtil::MalformedVariant( + "the value column of an unshredded variant is null"); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW( + value_builder->Append(cursor.GetBinary(schema->variant_idx))); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata_builder->Append(metadata)); + } else { + VariantBuilder builder(/*allow_duplicate_keys=*/false); + PAIMON_RETURN_NOT_OK(Rebuild(cursor, metadata, *schema, pool, &builder)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, builder.Build(pool)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(variant->RawValue())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata_builder->Append(variant->Metadata())); + } + } + std::shared_ptr result; + PAIMON_RETURN_NOT_OK_FROM_ARROW(output_builder->Finish(&result)); + return result; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_reassembler.h b/src/paimon/common/data/variant/variant_reassembler.h new file mode 100644 index 00000000..ac5d9d5a --- /dev/null +++ b/src/paimon/common/data/variant/variant_reassembler.h @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include + +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class MemoryPool; +class StructArray; +} // namespace arrow + +namespace paimon { + +class VariantBuilder; + +/// Reassembles shredded variant columns back into the unshredded +/// `struct` representation, implementing the reconstruction +/// algorithm of the parquet-format VariantShredding.md specification (mirroring the Java +/// `ShreddingUtils.rebuild`). +class VariantReassembler { + public: + VariantReassembler() = delete; + ~VariantReassembler() = delete; + + /// Reassembles a shredded variant column into a `struct` array. + /// + /// @param shredded The physical shredded array read from the file. + /// @param schema The shredding schema of the column + /// (`VariantShreddingUtils::BuildVariantSchema` of the file type). + /// @param pool The memory pool used for intermediate variant rebuilding. + /// @param arrow_pool The Arrow memory pool used for the output array. + /// @return The unshredded variant array (`VariantTypeUtils::UnshreddedStructType`). + static Result> AssembleVariantArray( + const std::shared_ptr& shredded, + const std::shared_ptr& schema, const std::shared_ptr& pool, + arrow::MemoryPool* arrow_pool); + + /// Rebuilds the variant value at `row` of a shredded (sub-)struct into `builder`, following + /// the same reconstruction algorithm. `schema` describes `shredded`, which may be any level + /// of the shredded tree; `metadata` is the column's top-level metadata binary. + static Status RebuildValue(const arrow::StructArray& shredded, int64_t row, + std::string_view metadata, const VariantSchema& schema, + const std::shared_ptr& pool, VariantBuilder* builder); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_schema.h b/src/paimon/common/data/variant/variant_schema.h new file mode 100644 index 00000000..d796c4a6 --- /dev/null +++ b/src/paimon/common/data/variant/variant_schema.h @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace paimon { + +/// Defines a valid shredding schema, as described in the parquet-format VariantShredding.md +/// specification. A shredding schema contains a `value` and an optional `typed_value` field. If a +/// `typed_value` is an array or struct, it recursively contains its own shredding schema for +/// elements and fields, respectively. The schema also contains a `metadata` field at the top +/// level, but not in recursively shredded fields. +class VariantSchema { + public: + enum class ScalarKind { + kBoolean, + kByte, + kShort, + kInt, + kLong, + kFloat, + kDouble, + kString, + kBinary, + kDecimal, + kDate, + kTimestampLtz, + kTimestampNtz, + kUuid, + }; + + struct ScalarType { + ScalarKind kind; + // Only meaningful when `kind` is `kDecimal`. + int32_t precision = 0; + int32_t scale = 0; + }; + + /// Represents one field of an object in the shredding schema. + struct ObjectField { + std::string name; + std::shared_ptr schema; + }; + + /// The index of the typed_value, value, and metadata fields in the schema, respectively. If a + /// given field is not in the schema, its value must be set to -1 to indicate that it is + /// invalid. The indices of valid fields are contiguous and start from 0. + int32_t typed_idx = -1; + int32_t variant_idx = -1; + /// Must be non-negative in the top-level schema, and -1 at all other nesting levels. + int32_t top_level_metadata_idx = -1; + /// The number of fields in the schema, i.e. a value between 1 and 3, depending on which of + /// value, typed_value and metadata are present. + int32_t num_fields = 0; + + /// Exactly one of the following describes typed_value (or none if there is no typed_value). + std::optional scalar_schema; + bool has_object_schema = false; + std::vector object_schema; + /// Fast lookup of object fields by name; values are indices into `object_schema`. + std::unordered_map object_schema_map; + std::shared_ptr array_schema; + + /// Whether the variant column is unshredded. The user is not required to do anything special, + /// but can have certain optimizations for unshredded variants. + bool IsUnshredded() const { + return top_level_metadata_idx >= 0 && variant_idx >= 0 && typed_idx < 0; + } +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_batch_converter.cpp b/src/paimon/common/data/variant/variant_shredding_batch_converter.cpp new file mode 100644 index 00000000..a2cb234e --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_batch_converter.cpp @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_batch_converter.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/util/bitmap_ops.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_shredding_writer.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +namespace { + +/// Whether any enclosing struct is null at `row`. The contents of child slots under a null +/// ancestor are unspecified in Arrow and must not be decoded. +bool AnyAncestorNull(const std::vector& ancestors, int64_t row) { + for (const arrow::Array* ancestor : ancestors) { + if (ancestor->IsNull(row)) { + return true; + } + } + return false; +} + +/// Shreds one variant column array into its physical shredded representation. +Result> ShredVariantColumn( + const arrow::Array& column, const std::string& field_name, + const std::vector& ancestors, + const std::shared_ptr& variant_schema, + const std::shared_ptr& physical_type, const std::shared_ptr& pool, + arrow::MemoryPool* arrow_pool) { + if (column.type_id() != arrow::Type::STRUCT) { + return Status::Invalid( + fmt::format("variant column {} is not a struct column", field_name)); + } + const auto& variant_column = arrow::internal::checked_cast(column); + if (variant_column.num_fields() != 2) { + return Status::Invalid( + fmt::format("variant column {} is not a struct column", field_name)); + } + const auto& value_column = + arrow::internal::checked_cast(*variant_column.field(0)); + const auto& metadata_column = + arrow::internal::checked_cast(*variant_column.field(1)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr writer, + VariantShreddedColumnWriter::Create(variant_schema, physical_type, arrow_pool)); + for (int64_t row = 0; row < variant_column.length(); ++row) { + if (variant_column.IsNull(row) || AnyAncestorNull(ancestors, row)) { + PAIMON_RETURN_NOT_OK(writer->AppendNull()); + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(value_column.GetView(row), metadata_column.GetView(row), pool)); + PAIMON_RETURN_NOT_OK(writer->Append(*variant)); + } + return writer->Finish(); +} + +} // namespace + +VariantShreddingBatchConverter::VariantShreddingBatchConverter( + const std::shared_ptr& plan, const std::shared_ptr& pool) + : plan_(plan), pool_(pool), arrow_pool_(GetArrowPool(pool)) {} + +Result> VariantShreddingBatchConverter::Create( + const std::shared_ptr& plan, + const std::shared_ptr& pool) { + if (!plan) { + return Status::Invalid("variant shredding batch converter requires a write plan"); + } + return std::shared_ptr( + new VariantShreddingBatchConverter(plan, pool)); +} + +const std::shared_ptr& VariantShreddingBatchConverter::GetPhysicalSchema() const { + return plan_->PhysicalSchema(); +} + +Result> VariantShreddingBatchConverter::ConvertField( + const std::shared_ptr& logical, + const std::shared_ptr& logical_field, + const std::shared_ptr& physical_field, std::vector* path, + std::vector* ancestors) const { + for (const auto& column : plan_->Columns()) { + if (column.path == *path) { + return ShredVariantColumn(*logical, logical_field->name(), *ancestors, + column.variant_schema, column.physical_type, pool_, + arrow_pool_.get()); + } + } + if (logical_field->type()->Equals(*physical_field->type())) { + return logical; + } + // The types differ below this struct field: convert the planned descendants recursively and + // rebuild the struct with its original validity. + if (logical->type_id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("variant shredding cannot convert non-struct field {}", + logical_field->name())); + } + const auto& logical_struct = arrow::internal::checked_cast(*logical); + const auto& logical_type = + arrow::internal::checked_cast(*logical_field->type()); + const auto& physical_type = + arrow::internal::checked_cast(*physical_field->type()); + if (logical_type.num_fields() != physical_type.num_fields()) { + return Status::Invalid(fmt::format("variant shredding physical struct {} does not match", + physical_field->name())); + } + arrow::ArrayVector converted_children(logical_struct.num_fields()); + ancestors->push_back(logical.get()); + for (int32_t i = 0; i < logical_struct.num_fields(); ++i) { + path->push_back(i); + PAIMON_ASSIGN_OR_RAISE(converted_children[i], + ConvertField(logical_struct.field(i), logical_type.field(i), + physical_type.field(i), path, ancestors)); + path->pop_back(); + } + ancestors->pop_back(); + std::shared_ptr validity; + int64_t null_count = logical_struct.null_count(); + if (null_count > 0) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + validity, + arrow::internal::CopyBitmap(arrow_pool_.get(), logical_struct.null_bitmap_data(), + logical_struct.offset(), logical_struct.length())); + } + return std::make_shared(physical_field->type(), logical_struct.length(), + std::move(converted_children), std::move(validity), + null_count); +} + +Result> VariantShreddingBatchConverter::Convert( + ArrowArray* logical_batch) { + auto logical_struct_type = arrow::struct_(plan_->LogicalSchema()->fields()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr logical_array, + arrow::ImportArray(logical_batch, logical_struct_type)); + const auto& logical_struct = std::static_pointer_cast(logical_array); + + const auto& logical_fields = plan_->LogicalSchema()->fields(); + const auto& physical_fields = plan_->PhysicalSchema()->fields(); + arrow::ArrayVector physical_arrays(logical_struct->num_fields()); + std::vector path; + std::vector ancestors; + for (int32_t i = 0; i < logical_struct->num_fields(); ++i) { + path.push_back(i); + PAIMON_ASSIGN_OR_RAISE(physical_arrays[i], + ConvertField(logical_struct->field(i), logical_fields[i], + physical_fields[i], &path, &ancestors)); + path.pop_back(); + } + + arrow::FieldVector physical_struct_fields(physical_fields.begin(), physical_fields.end()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr physical_struct, + arrow::StructArray::Make(physical_arrays, physical_struct_fields)); + auto result = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*physical_struct, result.get())); + return result; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_batch_converter.h b/src/paimon/common/data/variant/variant_shredding_batch_converter.h new file mode 100644 index 00000000..706af162 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_batch_converter.h @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/common/data/shredding/shredding_batch_converter.h" +#include "paimon/common/data/variant/variant_shredding_write_plan.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +struct ArrowArray; + +namespace arrow { +class Array; +class Field; +class MemoryPool; +class Schema; +} // namespace arrow + +namespace paimon { + +/// Converts logical batches containing VARIANT columns into physical batches where each planned +/// variant column is replaced by its shredded struct representation. Unplanned columns are +/// passed through unchanged. +class VariantShreddingBatchConverter : public ShreddingBatchConverter { + public: + static Result> Create( + const std::shared_ptr& plan, + const std::shared_ptr& pool); + + /// The physical schema produced by this converter. + const std::shared_ptr& GetPhysicalSchema() const override; + + /// Converts a logical batch to a physical batch. + /// @param logical_batch Input ArrowArray (C ABI) with the logical schema. Consumed on + /// success. + /// @return Owned physical ArrowArray (C ABI) with the physical schema. + Result> Convert(ArrowArray* logical_batch) override; + + private: + VariantShreddingBatchConverter(const std::shared_ptr& plan, + const std::shared_ptr& pool); + + /// Converts the logical array at field-index path `path`, shredding it when planned and + /// otherwise recursing into struct children whose subtree contains a planned column. + /// `ancestors` holds the enclosing struct arrays; rows that are null at any level shred to + /// null without decoding the (unspecified) child slot contents. + Result> ConvertField( + const std::shared_ptr& logical, + const std::shared_ptr& logical_field, + const std::shared_ptr& physical_field, std::vector* path, + std::vector* ancestors) const; + + std::shared_ptr plan_; + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp new file mode 100644 index 00000000..f41adff7 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp @@ -0,0 +1,598 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_read_plan_factory.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_binary_util.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_get.h" +#include "paimon/common/data/variant/variant_reassembler.h" +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +namespace { + +/// Reassembles the full variant of a shredded file column back into +/// `struct` (a plain VARIANT read). +class FullVariantColumnReadPlan : public ShreddingColumnReadPlan { + public: + FullVariantColumnReadPlan(std::shared_ptr logical_field, + std::shared_ptr physical_field, + std::shared_ptr schema, + std::shared_ptr pool) + : logical_field_(std::move(logical_field)), + physical_field_(std::move(physical_field)), + schema_(std::move(schema)), + pool_(std::move(pool)) {} + + const std::shared_ptr& LogicalField() const override { + return logical_field_; + } + + const std::shared_ptr& PhysicalField() const override { + return physical_field_; + } + + Result> Assemble(const std::shared_ptr& physical, + arrow::MemoryPool* pool) const override { + if (physical->type_id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("cannot cast shredded variant field {} to a struct", + physical_field_->name())); + } + auto physical_struct = std::static_pointer_cast(physical); + return VariantReassembler::AssembleVariantArray(physical_struct, schema_, pool_, pool); + } + + private: + std::shared_ptr logical_field_; + std::shared_ptr physical_field_; + std::shared_ptr schema_; + std::shared_ptr pool_; +}; + +/// A node of a nested variant plan tree: a variant position with its own leaf plan, or a nested +/// container level to descend through. +struct NestedVariantNode { + /// The leaf plan when this position is a variant column: a full reassembly or a + /// variant-access extraction. + std::shared_ptr plan; + /// The children whose subtree holds planned variants, by Arrow child index. The index + /// addresses struct fields, the list element field, and the map entries field alike. + std::map children; +}; + +/// Applies the leaf plans of the variant columns nested inside a top-level STRUCT / LIST / MAP +/// column, rebuilding the containers around them. +class NestedVariantColumnReadPlan : public ShreddingColumnReadPlan { + public: + NestedVariantColumnReadPlan(std::shared_ptr logical_field, + std::shared_ptr physical_field, + NestedVariantNode root) + : logical_field_(std::move(logical_field)), + physical_field_(std::move(physical_field)), + root_(std::move(root)) {} + + const std::shared_ptr& LogicalField() const override { + return logical_field_; + } + + const std::shared_ptr& PhysicalField() const override { + return physical_field_; + } + + Result> Assemble(const std::shared_ptr& physical, + arrow::MemoryPool* pool) const override { + return AssembleNode(physical, logical_field_, root_, pool); + } + + private: + Result> AssembleNode( + const std::shared_ptr& physical, + const std::shared_ptr& logical_field, const NestedVariantNode& node, + arrow::MemoryPool* pool) const { + if (node.plan != nullptr) { + return node.plan->Assemble(physical, pool); + } + const std::shared_ptr& logical_type = logical_field->type(); + const std::shared_ptr& physical_data = physical->data(); + if (physical->type_id() != logical_type->id()) { + return Status::Invalid(fmt::format( + "shredded variant field {} is stored as {} but read as {}", logical_field->name(), + physical->type()->ToString(), logical_type->ToString())); + } + // Swap the planned children into a copy of the physical array's own data: the validity + // bitmap, list offsets and slice offset carry over untouched, so STRUCT, LIST and MAP + // rebuild alike. Children are assembled unsliced to stay aligned with those offsets. + std::shared_ptr data = physical_data->Copy(); + data->type = logical_type; + for (const auto& [index, child_node] : node.children) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr child, + AssembleNode(arrow::MakeArray(physical_data->child_data[index]), + logical_type->field(index), child_node, pool)); + data->child_data[index] = child->data(); + } + return arrow::MakeArray(data); + } + + std::shared_ptr logical_field_; + std::shared_ptr physical_field_; + NestedVariantNode root_; +}; + +/// Whether the field is read as a variant column: either as a plain VARIANT or as a +/// variant-access projection (which keeps the variant marker but replaces the type). +bool IsVariantReadField(const std::shared_ptr& field) { + return VariantTypeUtils::IsVariantField(field) || + VariantAccessUtils::IsVariantAccessType(field->type()); +} + +/// Whether the type is a nested container the plan tree descends through. +bool IsNestedContainer(const std::shared_ptr& type) { + return type->id() == arrow::Type::STRUCT || type->id() == arrow::Type::LIST || + type->id() == arrow::Type::MAP; +} + +/// Whether the field is a nested container (that is not itself a variant) holding a variant +/// field in its subtree. +bool ContainsNestedVariant(const std::shared_ptr& field) { + if (IsVariantReadField(field) || !IsNestedContainer(field->type())) { + return false; + } + for (const auto& child : field->type()->fields()) { + if (IsVariantReadField(child) || ContainsNestedVariant(child)) { + return true; + } + } + return false; +} + +// Both are defined below: CreateVariantColumnPlan after the access plan it builds, and +// BuildNestedVariantPlan because it and PlanChild are mutually recursive. +Result> CreateVariantColumnPlan( + const std::shared_ptr& read_field, + const std::shared_ptr& file_field, const std::shared_ptr& pool, + bool allow_pruning); + +Result BuildNestedVariantPlan(const std::shared_ptr& read_field, + const std::shared_ptr& file_field, + const std::shared_ptr& pool, bool inside_repeated, + std::shared_ptr* physical_field, + NestedVariantNode* node); + +/// Plans one child position: a variant leaf, or a nested container to descend into. Returns +/// whether the position needs a plan; `physical_child` receives the field to push down for it. +Result PlanChild(const std::shared_ptr& read_child, + const std::shared_ptr& file_child, + const std::shared_ptr& pool, bool inside_repeated, + std::shared_ptr* physical_child, NestedVariantNode* node) { + *physical_child = read_child; + if (IsVariantReadField(read_child)) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr plan, + CreateVariantColumnPlan(read_child, file_child, pool, !inside_repeated)); + if (plan == nullptr) { + // An unshredded column read as a plain VARIANT needs no plan. + return false; + } + *physical_child = read_child->WithType(plan->PhysicalField()->type()); + node->plan = std::move(plan); + return true; + } + if (!ContainsNestedVariant(read_child) || + file_child->type()->id() != read_child->type()->id()) { + return false; + } + return BuildNestedVariantPlan(read_child, file_child, pool, inside_repeated, physical_child, + node); +} + +/// Whether the read and file children of a level inside a repeated group line up one to one, +/// which they must because the file subtree is pushed down verbatim there. +/// +/// Only STRUCT children are matched by name: LIST and MAP child names are format conventions +/// that differ between the read schema and the file (`element` vs `item`, `entries` vs the +/// parquet `key_value` group), so those match positionally. +bool ChildrenLineUp(const arrow::DataType& read_type, const arrow::DataType& file_type) { + if (read_type.num_fields() != file_type.num_fields()) { + return false; + } + if (read_type.id() != arrow::Type::STRUCT) { + return true; + } + for (int32_t i = 0; i < read_type.num_fields(); ++i) { + if (read_type.field(i)->name() != file_type.field(i)->name()) { + return false; + } + } + return true; +} + +/// Builds the nested plan of one column subtree: substitutes the physical file types at the +/// nested variant positions into `read_field` (producing the physical field to push down) and +/// records their leaf plans in `node`. Returns whether any nested position needs a plan. +/// +/// A STRUCT level is matched by field name and keeps per-child pruning. A LIST or MAP level is +/// matched positionally and pushes the file subtree down verbatim, because the parquet reader +/// rejects partial projection inside a repeated group; everything below it is therefore read in +/// full and only reassembled back. +Result BuildNestedVariantPlan(const std::shared_ptr& read_field, + const std::shared_ptr& file_field, + const std::shared_ptr& pool, bool inside_repeated, + std::shared_ptr* physical_field, + NestedVariantNode* node) { + const arrow::DataType& read_type = *read_field->type(); + const arrow::DataType& file_type = *file_field->type(); + // True for a LIST or MAP level and for everything below it, including plain STRUCT levels. + const bool in_repeated_subtree = inside_repeated || read_type.id() != arrow::Type::STRUCT; + *physical_field = read_field; + if (in_repeated_subtree && !ChildrenLineUp(read_type, file_type)) { + return false; + } + + arrow::FieldVector physical_children = read_type.fields(); + bool needs_plan = false; + for (int32_t i = 0; i < read_type.num_fields(); ++i) { + const std::shared_ptr& read_child = read_type.field(i); + std::shared_ptr file_child = + in_repeated_subtree + ? file_type.field(i) + : arrow::internal::checked_cast(file_type).GetFieldByName( + read_child->name()); + if (file_child == nullptr) { + // The nested column is absent in the file (schema evolution); it is filled with + // nulls downstream. + continue; + } + NestedVariantNode child_node; + PAIMON_ASSIGN_OR_RAISE(bool child_needs_plan, + PlanChild(read_child, file_child, pool, in_repeated_subtree, + &physical_children[i], &child_node)); + if (child_needs_plan) { + node->children[i] = std::move(child_node); + needs_plan = true; + } + } + if (!needs_plan) { + return false; + } + *physical_field = + in_repeated_subtree ? file_field : read_field->WithType(arrow::struct_(physical_children)); + return true; +} + +/// One access path segment resolved against the shredded schema of one file. +struct ResolvedSegment { + VariantPathSegment raw; + bool is_object = false; + // The `typed_value` index at this level, or -1 when the path leaves the shredded schema + // here and continues inside the `value` binary. + int32_t typed_idx = -1; + // The object field index inside the typed object, or the array element index. + int32_t extraction_idx = -1; +}; + +struct ResolvedSpec { + VariantAccessSpec spec; + std::vector segments; +}; + +ResolvedSpec ResolveSpec(const VariantAccessSpec& spec, const VariantSchema* root) { + ResolvedSpec resolved; + resolved.spec = spec; + const VariantSchema* schema = root; + for (const auto& segment : spec.segments) { + ResolvedSegment r; + r.raw = segment; + if (segment.kind == VariantPathSegment::Kind::kObjectExtraction) { + r.is_object = true; + if (schema != nullptr && !schema->object_schema.empty()) { + auto it = schema->object_schema_map.find(segment.key); + if (it != schema->object_schema_map.end()) { + r.typed_idx = schema->typed_idx; + r.extraction_idx = it->second; + schema = schema->object_schema[it->second].schema.get(); + } else { + schema = nullptr; + } + } else { + schema = nullptr; + } + } else { + if (schema != nullptr && schema->array_schema != nullptr) { + r.typed_idx = schema->typed_idx; + r.extraction_idx = segment.index; + schema = schema->array_schema.get(); + } else { + schema = nullptr; + } + } + resolved.segments.push_back(std::move(r)); + } + return resolved; +} + +/// Extracts the paths described by a variant-access projection, reading typed sub-columns +/// directly and falling back to the `value` binary where the path is not shredded. +class VariantAccessColumnReadPlan : public ShreddingColumnReadPlan { + public: + VariantAccessColumnReadPlan(std::shared_ptr logical_field, + std::shared_ptr physical_field, + std::shared_ptr schema, + std::vector specs, std::shared_ptr pool) + : logical_field_(std::move(logical_field)), + physical_field_(std::move(physical_field)), + schema_(std::move(schema)), + specs_(std::move(specs)), + pool_(std::move(pool)) {} + + const std::shared_ptr& LogicalField() const override { + return logical_field_; + } + + const std::shared_ptr& PhysicalField() const override { + return physical_field_; + } + + Result> Assemble(const std::shared_ptr& physical, + arrow::MemoryPool* pool) const override { + if (physical->type_id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("cannot cast shredded variant field {} to a struct", + physical_field_->name())); + } + const auto& physical_struct = static_cast(*physical); + std::unique_ptr builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::MakeBuilder(pool, logical_field_->type(), &builder)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Reserve(physical_struct.length())); + auto* struct_builder = static_cast(builder.get()); + for (int64_t row = 0; row < physical_struct.length(); ++row) { + if (physical_struct.IsNull(row)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->AppendNull()); + continue; + } + if (physical_struct.field(schema_->top_level_metadata_idx)->IsNull(row)) { + return VariantBinaryUtil::MalformedVariant("the variant metadata column is null"); + } + std::string_view metadata = static_cast( + *physical_struct.field(schema_->top_level_metadata_idx)) + .GetView(row); + PAIMON_RETURN_NOT_OK_FROM_ARROW(struct_builder->Append()); + for (size_t i = 0; i < specs_.size(); ++i) { + PAIMON_RETURN_NOT_OK( + ExtractField(physical_struct, row, metadata, specs_[i], + struct_builder->field_builder(static_cast(i)))); + } + } + std::shared_ptr result; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&result)); + return result; + } + + private: + Status ExtractField(const arrow::StructArray& root, int64_t root_row, std::string_view metadata, + const ResolvedSpec& resolved, arrow::ArrayBuilder* builder) const { + const arrow::StructArray* current = &root; + int64_t row = root_row; + const VariantSchema* schema = schema_.get(); + size_t segment_idx = 0; + while (segment_idx < resolved.segments.size()) { + const ResolvedSegment& segment = resolved.segments[segment_idx]; + if (segment.typed_idx < 0) { + // The path leaves the shredded schema here; walk the remaining raw path inside + // the `value` binary. + return ExtractFromBinary(*current, row, metadata, resolved, segment_idx, schema, + builder); + } + if (current->field(segment.typed_idx)->IsNull(row)) { + return ToPaimonStatus(builder->AppendNull()); + } + if (segment.is_object) { + const auto& object_array = + static_cast(*current->field(segment.typed_idx)); + const auto& field_array = static_cast( + *object_array.field(segment.extraction_idx)); + if (field_array.IsNull(row)) { + // Shredded object fields must not be null. + return VariantBinaryUtil::MalformedVariant( + "a shredded object field group is null"); + } + schema = schema->object_schema[segment.extraction_idx].schema.get(); + current = &field_array; + // A field is missing when neither its typed_value nor its value is present. + bool typed_present = + schema->typed_idx >= 0 && !current->field(schema->typed_idx)->IsNull(row); + bool variant_present = + schema->variant_idx >= 0 && !current->field(schema->variant_idx)->IsNull(row); + if (!typed_present && !variant_present) { + return ToPaimonStatus(builder->AppendNull()); + } + } else { + const auto& list_array = + static_cast(*current->field(segment.typed_idx)); + if (segment.extraction_idx >= list_array.value_length(row)) { + return ToPaimonStatus(builder->AppendNull()); + } + int64_t element_row = list_array.value_offset(row) + segment.extraction_idx; + const auto& element_array = + static_cast(*list_array.values()); + if (element_array.IsNull(element_row)) { + // Shredded array elements must not be null. + return VariantBinaryUtil::MalformedVariant( + "a shredded array element group is null"); + } + schema = schema->array_schema.get(); + current = &element_array; + row = element_row; + } + ++segment_idx; + } + + // The terminal position: rebuild the (sub-)variant and cast it to the target type. + if (schema->typed_idx >= 0 && !current->field(schema->typed_idx)->IsNull(row)) { + VariantBuilder variant_builder(/*allow_duplicate_keys=*/false); + PAIMON_RETURN_NOT_OK(VariantReassembler::RebuildValue(*current, row, metadata, *schema, + pool_, &variant_builder)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + variant_builder.Build(pool_)); + return VariantGetExecutor::CastToBuilder(variant, resolved.spec.target_field, + resolved.spec.cast_args, pool_, builder); + } + if (schema->variant_idx >= 0 && !current->field(schema->variant_idx)->IsNull(row)) { + std::string_view value = + static_cast(*current->field(schema->variant_idx)) + .GetView(row); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + GenericVariant::Create(value, metadata, pool_)); + return VariantGetExecutor::CastToBuilder(variant, resolved.spec.target_field, + resolved.spec.cast_args, pool_, builder); + } + return VariantBinaryUtil::MalformedVariant( + "both typed_value and value of a required variant are null"); + } + + Status ExtractFromBinary(const arrow::StructArray& current, int64_t row, + std::string_view metadata, const ResolvedSpec& resolved, + size_t segment_idx, const VariantSchema* schema, + arrow::ArrayBuilder* builder) const { + if (schema->variant_idx < 0 || current.field(schema->variant_idx)->IsNull(row)) { + return ToPaimonStatus(builder->AppendNull()); + } + std::string_view value = + static_cast(*current.field(schema->variant_idx)) + .GetView(row); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + GenericVariant::Create(value, metadata, pool_)); + for (; segment_idx < resolved.segments.size() && variant != nullptr; ++segment_idx) { + const VariantPathSegment& raw = resolved.segments[segment_idx].raw; + PAIMON_ASSIGN_OR_RAISE(VariantValueType type, variant->GetType()); + if (raw.kind == VariantPathSegment::Kind::kObjectExtraction && + type == VariantValueType::kObject) { + PAIMON_ASSIGN_OR_RAISE(variant, variant->GetFieldByKey(raw.key)); + } else if (raw.kind == VariantPathSegment::Kind::kArrayExtraction && + type == VariantValueType::kArray) { + PAIMON_ASSIGN_OR_RAISE(variant, variant->GetElementAtIndex(raw.index)); + } else { + variant = nullptr; + } + } + return VariantGetExecutor::CastToBuilder(variant, resolved.spec.target_field, + resolved.spec.cast_args, pool_, builder); + } + + std::shared_ptr logical_field_; + std::shared_ptr physical_field_; + std::shared_ptr schema_; + std::vector specs_; + std::shared_ptr pool_; +}; + +/// Builds the leaf read plan of one variant position, at the top level or nested inside a +/// container column: a variant-access projection extracts the described paths (from a shredded +/// or an unshredded file column), and a plain VARIANT read reassembles a shredded file column. +/// Returns nullptr when a plain VARIANT read of an unshredded file column needs no plan. +/// +/// `allow_pruning` narrows the scan to the sub-columns the access paths need. It is off inside a +/// repeated group, where the file subtree must be read whole. +Result> CreateVariantColumnPlan( + const std::shared_ptr& read_field, + const std::shared_ptr& file_field, const std::shared_ptr& pool, + bool allow_pruning) { + if (VariantAccessUtils::IsVariantAccessType(read_field->type())) { + PAIMON_ASSIGN_OR_RAISE(std::vector specs, + VariantAccessUtils::ParseAccessSpecs(read_field)); + std::shared_ptr physical_field = file_field; + if (allow_pruning) { + PAIMON_ASSIGN_OR_RAISE(physical_field, + VariantAccessUtils::ClipShreddedFileField(specs, file_field)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(physical_field->type())); + std::vector resolved; + resolved.reserve(specs.size()); + for (const auto& spec : specs) { + resolved.push_back(ResolveSpec(spec, schema.get())); + } + return std::make_shared( + read_field, physical_field, std::move(schema), std::move(resolved), pool); + } + if (!VariantShreddingUtils::IsShreddedFileType(file_field->type())) { + return std::shared_ptr(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(file_field->type())); + return std::make_shared(read_field, file_field, std::move(schema), + pool); +} + +} // namespace + +Result>> +VariantShreddingReadPlanFactory::CreateReadPlans(const std::shared_ptr& read_schema, + const std::shared_ptr& file_schema, + const std::shared_ptr& pool) { + std::map> plans; + for (const auto& read_field : read_schema->fields()) { + bool nested_variant = ContainsNestedVariant(read_field); + if (!IsVariantReadField(read_field) && !nested_variant) { + continue; + } + auto file_field = file_schema->GetFieldByName(read_field->name()); + if (file_field == nullptr) { + // The column is absent in the file (schema evolution); it is filled with nulls + // downstream. + continue; + } + if (nested_variant) { + if (file_field->type()->id() != read_field->type()->id()) { + continue; + } + std::shared_ptr physical_field; + NestedVariantNode root; + PAIMON_ASSIGN_OR_RAISE( + bool needs_plan, + BuildNestedVariantPlan(read_field, file_field, pool, + /*inside_repeated=*/false, &physical_field, &root)); + if (needs_plan) { + plans.emplace(read_field->name(), std::make_shared( + read_field, physical_field, std::move(root))); + } + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr plan, + CreateVariantColumnPlan(read_field, file_field, pool, /*allow_pruning=*/true)); + if (plan != nullptr) { + plans.emplace(read_field->name(), std::move(plan)); + } + } + return plans; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_read_plan_factory.h b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.h new file mode 100644 index 00000000..2df392d4 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.h @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/common/data/shredding/shredding_read_plan.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +/// Builds per-column read plans for VARIANT columns: +/// - a plain VARIANT read of a shredded file reassembles the full variant; +/// - a variant-access projection (struct whose children carry `__VARIANT_METADATA` +/// descriptions) extracts the described paths, reading only the required shredded +/// sub-columns from a shredded file (or the binary from an unshredded file). +class VariantShreddingReadPlanFactory { + public: + VariantShreddingReadPlanFactory() = delete; + ~VariantShreddingReadPlanFactory() = delete; + + /// Creates the per-column read plans for the variant columns of `read_schema` against + /// `file_schema`; the map is empty when no plan applies (no shredded file column and no + /// variant-access projection). + static Result>> CreateReadPlans( + const std::shared_ptr& read_schema, + const std::shared_ptr& file_schema, const std::shared_ptr& pool); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_test.cpp b/src/paimon/common/data/variant/variant_shredding_test.cpp new file mode 100644 index 00000000..cfbf1571 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_test.cpp @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_reassembler.h" +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_shredding_writer.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class VariantShreddingTest : public ::testing::Test { + public: + // Shreds the given JSON documents (nullptr = null variant) with the given shredding type, + // asserts the reassembled variants render back to the same JSON, and returns the shredded + // array for structural checks. + std::shared_ptr RoundTrip( + const std::shared_ptr& shredding_type, + const std::vector& jsons) { + EXPECT_OK_AND_ASSIGN(std::shared_ptr physical, + VariantShreddingUtils::VariantShreddingSchema(shredding_type)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(physical)); + + EXPECT_OK_AND_ASSIGN( + std::unique_ptr writer, + VariantShreddedColumnWriter::Create(schema, physical, arrow::default_memory_pool())); + std::vector expected_jsons; + for (const char* json : jsons) { + if (json == nullptr) { + EXPECT_OK(writer->AppendNull()); + expected_jsons.emplace_back(); + continue; + } + EXPECT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::FromJson(json, pool_)); + EXPECT_OK_AND_ASSIGN(std::string expected_json, variant->ToJson()); + expected_jsons.push_back(std::move(expected_json)); + EXPECT_OK(writer->Append(*variant)); + } + EXPECT_OK_AND_ASSIGN(std::shared_ptr shredded_array, writer->Finish()); + auto shredded = std::static_pointer_cast(shredded_array); + + EXPECT_OK_AND_ASSIGN(std::shared_ptr assembled_array, + VariantReassembler::AssembleVariantArray( + shredded, schema, pool_, arrow::default_memory_pool())); + auto assembled = std::static_pointer_cast(assembled_array); + EXPECT_EQ(assembled->length(), static_cast(jsons.size())); + auto value_column = std::static_pointer_cast(assembled->field(0)); + auto metadata_column = std::static_pointer_cast(assembled->field(1)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + if (jsons[i] == nullptr) { + EXPECT_TRUE(assembled->IsNull(i)); + continue; + } + EXPECT_FALSE(assembled->IsNull(i)); + EXPECT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(i), + metadata_column->GetView(i), pool_)); + EXPECT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + EXPECT_EQ(actual_json, expected_jsons[i]); + } + return shredded; + } + + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(VariantShreddingTest, ShreddingSchemaShape) { + auto shredding_type = + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr physical, + VariantShreddingUtils::VariantShreddingSchema(shredding_type)); + // struct{metadata: binary not null, value: binary, typed_value: struct{a: + // struct{value, typed_value} not null, b: ... not null}} + auto expected = arrow::struct_( + {arrow::field("metadata", arrow::binary(), false), + arrow::field("value", arrow::binary(), true), + arrow::field( + "typed_value", + arrow::struct_( + {arrow::field("a", + arrow::struct_({arrow::field("value", arrow::binary(), true), + arrow::field("typed_value", arrow::int32(), true)}), + false), + arrow::field("b", + arrow::struct_({arrow::field("value", arrow::binary(), true), + arrow::field("typed_value", arrow::utf8(), true)}), + false)}), + true)}); + ASSERT_TRUE(physical->Equals(*expected)) << physical->ToString(); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + VariantShreddingUtils::BuildVariantSchema(physical)); + ASSERT_EQ(schema->top_level_metadata_idx, 0); + ASSERT_EQ(schema->variant_idx, 1); + ASSERT_EQ(schema->typed_idx, 2); + ASSERT_TRUE(schema->has_object_schema); + ASSERT_EQ(schema->object_schema.size(), 2); + ASSERT_FALSE(schema->IsUnshredded()); + ASSERT_TRUE(VariantShreddingUtils::IsShreddedFileType(physical)); + ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType( + arrow::struct_({arrow::field("value", arrow::binary(), false), + arrow::field("metadata", arrow::binary(), false)}))); + + // Invalid shredding types are rejected. + ASSERT_NOK(VariantShreddingUtils::VariantShreddingSchema(arrow::date32())); + ASSERT_NOK( + VariantShreddingUtils::VariantShreddingSchema(arrow::map(arrow::utf8(), arrow::int32()))); +} + +TEST_F(VariantShreddingTest, ShredObject) { + // Mirrors the Java GenericVariantTest#testShredding scenarios. + auto variant_json = R"({"a": 1, "b": "hello"})"; + // Happy path: all fields shredded, no residual value. + { + auto shredded = RoundTrip( + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())}), + {variant_json}); + ASSERT_TRUE(shredded->field(1)->IsNull(0)); // top-level value (residual) is null + ASSERT_FALSE(shredded->field(2)->IsNull(0)); // typed_value is present + } + // Missing field "c" in the data: present in schema, both children null. + { + auto shredded = RoundTrip( + arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("c", arrow::utf8()), + arrow::field("b", arrow::utf8())}), + {variant_json}); + auto typed = std::static_pointer_cast(shredded->field(2)); + auto c_group = std::static_pointer_cast(typed->field(1)); + ASSERT_FALSE(c_group->IsNull(0)); + ASSERT_TRUE(c_group->field(0)->IsNull(0)); + ASSERT_TRUE(c_group->field(1)->IsNull(0)); + } + // "a" is not present in the shredding schema: it goes to the residual value. + { + auto shredded = RoundTrip( + arrow::struct_({arrow::field("b", arrow::utf8()), arrow::field("c", arrow::utf8())}), + {variant_json}); + auto value_column = std::static_pointer_cast(shredded->field(1)); + ASSERT_FALSE(value_column->IsNull(0)); + // The residual must equal the standalone encoding of {"a": 1}. + ASSERT_OK_AND_ASSIGN(std::shared_ptr residual_expected, + GenericVariant::FromJson("{\"a\": 1}", pool_)); + ASSERT_OK_AND_ASSIGN(std::string_view residual_value, residual_expected->Value()); + ASSERT_EQ(value_column->GetView(0), residual_value); + } +} + +TEST_F(VariantShreddingTest, ShredAllTypes) { + // Mirrors the Java GenericVariantTest#testShreddingAllTypes. + const char* json = + "{\n" + " \"c1\": \"Hello, World!\",\n" + " \"c2\": 12345678901234,\n" + " \"c3\": 1.0123456789012345678901234567890123456789,\n" + " \"c4\": 100.99,\n" + " \"c5\": true,\n" + " \"c6\": null,\n" + " \"c7\": {\"street\" : \"Main St\",\"city\" : \"Hangzhou\"},\n" + " \"c8\": [1, 2]\n" + "}\n"; + auto shredding_type = arrow::struct_( + {arrow::field("c1", arrow::utf8()), arrow::field("c2", arrow::int64()), + arrow::field("c3", arrow::float64()), arrow::field("c4", arrow::decimal128(5, 2)), + arrow::field("c5", arrow::boolean()), arrow::field("c6", arrow::utf8()), + arrow::field("c7", arrow::struct_({arrow::field("street", arrow::utf8()), + arrow::field("city", arrow::utf8())})), + arrow::field("c8", arrow::list(arrow::int32()))}); + auto shredded = RoundTrip(shredding_type, {json, nullptr, json}); + + // c6 is a variant null: it stays in the field's value column ("00"), typed_value is null. + auto typed = std::static_pointer_cast(shredded->field(2)); + auto c6_group = std::static_pointer_cast(typed->field(5)); + ASSERT_FALSE(c6_group->IsNull(0)); + auto c6_value = std::static_pointer_cast(c6_group->field(0)); + ASSERT_FALSE(c6_value->IsNull(0)); + ASSERT_EQ(c6_value->GetView(0), std::string_view("\x00", 1)); + ASSERT_TRUE(c6_group->field(1)->IsNull(0)); + + // Nothing was left over at the top level. + ASSERT_TRUE(shredded->field(1)->IsNull(0)); + + // No shredding at all: everything stays in the top-level value. + auto no_match_type = arrow::struct_({arrow::field("other", arrow::utf8())}); + auto unshredded = RoundTrip(no_match_type, {json}); + auto value_column = std::static_pointer_cast(unshredded->field(1)); + ASSERT_FALSE(value_column->IsNull(0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(json, pool_)); + ASSERT_OK_AND_ASSIGN(std::string_view expected_value, expected->Value()); + ASSERT_EQ(value_column->GetView(0), expected_value); +} + +TEST_F(VariantShreddingTest, ShredScalarsAndMismatches) { + // Top-level scalar shredding with type mismatches falling back to the value column. + auto long_type = arrow::struct_({arrow::field("x", arrow::int64())}); + RoundTrip(long_type, + {"{\"x\": 5}", R"({"x": "not a number"})", "{\"x\": 3.25}", "{\"x\": [1]}", "{}"}); + // Decimal rescale: 100.99 fits decimal(9, 4) exactly (allowNumericScaleChanges). + auto decimal_type = arrow::struct_({arrow::field("x", arrow::decimal128(9, 4))}); + RoundTrip(decimal_type, + {"{\"x\": 100.99}", "{\"x\": 42}", "{\"x\": 0.123456789}", "{\"x\": 100.00}"}); + // A 38-digit unscaled value cannot rescale to scale 1 without overflowing the 128-bit + // decimal; it must fall back to the value column instead of being written corrupted. + auto wide_decimal_type = arrow::struct_({arrow::field("x", arrow::decimal128(38, 1))}); + RoundTrip(wide_decimal_type, + {"{\"x\": 99999999999999999999999999999999999999}", "{\"x\": 1.5}"}); + // Integer target from decimal that is numerically integral. + auto int_type = arrow::struct_({arrow::field("x", arrow::int32())}); + RoundTrip(int_type, {"{\"x\": 5.0}", "{\"x\": 5.5}", "{\"x\": 123456789012345678}"}); + auto array_type = arrow::struct_( + {arrow::field("arr", arrow::list(arrow::struct_({arrow::field("k", arrow::utf8())})))}); + RoundTrip(array_type, {R"({"arr": [{"k": "v1"}, {"k": "v2", "extra": 1}, {"other": 2}]})", + "{\"arr\": [1, 2]}", R"({"arr": {"k": "v"}})"}); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_shredding_utils.cpp b/src/paimon/common/data/variant/variant_shredding_utils.cpp new file mode 100644 index 00000000..8d6f4382 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_utils.cpp @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_shredding_utils.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_type_utils.h" + +namespace paimon { + +namespace { + +Status InvalidVariantShreddingSchema(const std::shared_ptr& type) { + return Status::Invalid( + fmt::format("Invalid variant shredding schema: {}", type ? type->ToString() : "null")); +} + +// Mirrors the Java `PaimonShreddingUtils.variantShreddingSchema(dataType, isTopLevel, +// isObjectField)`. +Result> VariantShreddingSchemaImpl( + const std::shared_ptr& data_type, bool is_top_level, bool is_object_field) { + arrow::FieldVector fields; + if (is_top_level) { + fields.push_back(arrow::field(VariantDefs::kMetadataFieldName, arrow::binary(), + /*nullable=*/false)); + } + switch (data_type->id()) { + case arrow::Type::LIST: { + const auto& list_type = std::static_pointer_cast(data_type); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element_type, + VariantShreddingSchemaImpl(list_type->value_type(), + /*is_top_level=*/false, + /*is_object_field=*/false)); + fields.push_back( + arrow::field(VariantDefs::kValueFieldName, arrow::binary(), /*nullable=*/true)); + fields.push_back(arrow::field(VariantDefs::kTypedValueFieldName, + arrow::list(element_type), /*nullable=*/true)); + break; + } + case arrow::Type::STRUCT: { + // The field name level is always non-nullable: Variant null values are represented in + // the "value" column as "00", and missing values are represented by setting both + // "value" and "typed_value" to null. + const auto& struct_type = std::static_pointer_cast(data_type); + arrow::FieldVector shredded_fields; + for (const auto& field : struct_type->fields()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr field_type, + VariantShreddingSchemaImpl(field->type(), + /*is_top_level=*/false, + /*is_object_field=*/true)); + shredded_fields.push_back( + arrow::field(field->name(), field_type, /*nullable=*/false)); + } + fields.push_back( + arrow::field(VariantDefs::kValueFieldName, arrow::binary(), /*nullable=*/true)); + fields.push_back(arrow::field(VariantDefs::kTypedValueFieldName, + arrow::struct_(shredded_fields), /*nullable=*/true)); + break; + } + case arrow::Type::NA: { + // `arrow::null()` denotes an untyped VARIANT leaf in shredding types. It doesn't + // need a typed column. If there is no typed column, value is required for array + // elements or top-level fields, but optional for objects (where a null represents a + // missing field). + fields.push_back(arrow::field(VariantDefs::kValueFieldName, arrow::binary(), + /*nullable=*/is_object_field)); + break; + } + case arrow::Type::STRING: + case arrow::Type::BOOL: + case arrow::Type::BINARY: + case arrow::Type::DECIMAL128: + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::INT64: + case arrow::Type::FLOAT: + case arrow::Type::DOUBLE: { + fields.push_back( + arrow::field(VariantDefs::kValueFieldName, arrow::binary(), /*nullable=*/true)); + fields.push_back(arrow::field(VariantDefs::kTypedValueFieldName, data_type, + /*nullable=*/true)); + break; + } + default: + return InvalidVariantShreddingSchema(data_type); + } + return arrow::struct_(fields); +} + +Result> BuildVariantSchemaImpl( + const std::shared_ptr& type, bool top_level) { + if (type->id() != arrow::Type::STRUCT) { + return InvalidVariantShreddingSchema(type); + } + const auto& struct_type = std::static_pointer_cast(type); + // The struct must not be empty or contain duplicate field names. The latter is enforced in + // the loop below. + if (struct_type->num_fields() == 0) { + return InvalidVariantShreddingSchema(type); + } + + auto schema = std::make_shared(); + schema->num_fields = struct_type->num_fields(); + + for (int32_t i = 0; i < struct_type->num_fields(); ++i) { + const auto& field = struct_type->field(i); + const auto& field_type = field->type(); + if (field->name() == VariantDefs::kTypedValueFieldName) { + if (schema->typed_idx != -1) { + return InvalidVariantShreddingSchema(type); + } + schema->typed_idx = i; + switch (field_type->id()) { + case arrow::Type::STRUCT: { + const auto& object_type = + std::static_pointer_cast(field_type); + schema->has_object_schema = true; + schema->object_schema.reserve(object_type->num_fields()); + for (int32_t index = 0; index < object_type->num_fields(); ++index) { + const auto& object_field = object_type->field(index); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr field_schema, + BuildVariantSchemaImpl(object_field->type(), /*top_level=*/false)); + schema->object_schema.push_back( + VariantSchema::ObjectField{object_field->name(), field_schema}); + auto [it, inserted] = + schema->object_schema_map.emplace(object_field->name(), index); + if (!inserted) { + return InvalidVariantShreddingSchema(type); + } + } + break; + } + case arrow::Type::LIST: { + const auto& list_type = std::static_pointer_cast(field_type); + PAIMON_ASSIGN_OR_RAISE( + schema->array_schema, + BuildVariantSchemaImpl(list_type->value_type(), /*top_level=*/false)); + break; + } + case arrow::Type::BOOL: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kBoolean}; + break; + case arrow::Type::INT8: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kByte}; + break; + case arrow::Type::INT16: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kShort}; + break; + case arrow::Type::INT32: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kInt}; + break; + case arrow::Type::INT64: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kLong}; + break; + case arrow::Type::FLOAT: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kFloat}; + break; + case arrow::Type::DOUBLE: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kDouble}; + break; + case arrow::Type::STRING: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kString}; + break; + case arrow::Type::BINARY: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kBinary}; + break; + case arrow::Type::DATE32: + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kDate}; + break; + case arrow::Type::DECIMAL128: { + const auto& decimal_type = + std::static_pointer_cast(field_type); + schema->scalar_schema = + VariantSchema::ScalarType{VariantSchema::ScalarKind::kDecimal, + decimal_type->precision(), decimal_type->scale()}; + break; + } + case arrow::Type::TIMESTAMP: { + const auto& timestamp_type = + std::static_pointer_cast(field_type); + // The variant binary stores timestamps as microseconds since the epoch; a + // typed_value column of any other precision would misinterpret the values. + if (timestamp_type->unit() != arrow::TimeUnit::MICRO) { + return InvalidVariantShreddingSchema(type); + } + schema->scalar_schema = + VariantSchema::ScalarType{timestamp_type->timezone().empty() + ? VariantSchema::ScalarKind::kTimestampNtz + : VariantSchema::ScalarKind::kTimestampLtz}; + break; + } + default: + return InvalidVariantShreddingSchema(type); + } + } else if (field->name() == VariantDefs::kValueFieldName) { + if (schema->variant_idx != -1 || field_type->id() != arrow::Type::BINARY) { + return InvalidVariantShreddingSchema(type); + } + schema->variant_idx = i; + } else if (field->name() == VariantDefs::kMetadataFieldName) { + if (schema->top_level_metadata_idx != -1 || field_type->id() != arrow::Type::BINARY) { + return InvalidVariantShreddingSchema(type); + } + schema->top_level_metadata_idx = i; + } else { + return InvalidVariantShreddingSchema(type); + } + } + + if (top_level != (schema->top_level_metadata_idx >= 0)) { + return InvalidVariantShreddingSchema(type); + } + return schema; +} + +} // namespace + +Result> VariantShreddingUtils::VariantShreddingSchema( + const std::shared_ptr& shredding_type) { + return VariantShreddingSchemaImpl(shredding_type, /*is_top_level=*/true, + /*is_object_field=*/false); +} + +Result> VariantShreddingUtils::BuildVariantSchema( + const std::shared_ptr& struct_type) { + return BuildVariantSchemaImpl(struct_type, /*top_level=*/true); +} + +Result> VariantShreddingUtils::ScalarSchemaToArrowType( + const VariantSchema::ScalarType& scalar) { + switch (scalar.kind) { + case VariantSchema::ScalarKind::kBoolean: + return arrow::boolean(); + case VariantSchema::ScalarKind::kByte: + return arrow::int8(); + case VariantSchema::ScalarKind::kShort: + return arrow::int16(); + case VariantSchema::ScalarKind::kInt: + return arrow::int32(); + case VariantSchema::ScalarKind::kLong: + return arrow::int64(); + case VariantSchema::ScalarKind::kFloat: + return arrow::float32(); + case VariantSchema::ScalarKind::kDouble: + return arrow::float64(); + case VariantSchema::ScalarKind::kString: + return arrow::utf8(); + case VariantSchema::ScalarKind::kBinary: + return arrow::binary(); + case VariantSchema::ScalarKind::kDecimal: + return arrow::decimal128(scalar.precision, scalar.scale); + case VariantSchema::ScalarKind::kDate: + return arrow::date32(); + case VariantSchema::ScalarKind::kTimestampLtz: + return arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); + case VariantSchema::ScalarKind::kTimestampNtz: + return arrow::timestamp(arrow::TimeUnit::MICRO); + default: + return Status::NotImplemented(fmt::format("Unsupported variant scalar kind: {}", + static_cast(scalar.kind))); + } +} + +bool VariantShreddingUtils::IsShreddedFileType( + const std::shared_ptr& file_variant_type) { + if (!file_variant_type || file_variant_type->id() != arrow::Type::STRUCT) { + return false; + } + const auto& struct_type = std::static_pointer_cast(file_variant_type); + return struct_type->GetFieldByName(VariantDefs::kTypedValueFieldName) != nullptr; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_utils.h b/src/paimon/common/data/variant/variant_shredding_utils.h new file mode 100644 index 00000000..8f5b327c --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_utils.h @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +} // namespace arrow + +namespace paimon { + +/// Utils for converting between shredding schemas (`VariantSchema`) and their physical Arrow +/// representation, mirroring the Java `PaimonShreddingUtils` schema functions. +class VariantShreddingUtils { + public: + VariantShreddingUtils() = delete; + ~VariantShreddingUtils() = delete; + + /// Given an expected schema of a Variant value, returns a suitable physical schema for + /// shredding, by inserting appropriate intermediate value/typed_value fields at each level. + /// For example, to represent the JSON `{"a": 1, "b": "hello"}`, the schema + /// `struct{a: int32, b: string}` could be passed into this function, and it would return the + /// shredding schema: `struct{metadata: binary, value: binary, typed_value: struct{a: + /// struct{value: binary, typed_value: int32}, b: struct{value: binary, typed_value: + /// string}}}`. + static Result> VariantShreddingSchema( + const std::shared_ptr& shredding_type); + + /// Builds a `VariantSchema` from the physical shredded struct type (the inverse of + /// `VariantShreddingSchema`), validating field names and types. + static Result> BuildVariantSchema( + const std::shared_ptr& struct_type); + + /// The Arrow type of the typed_value column for a scalar shredding schema. + static Result> ScalarSchemaToArrowType( + const VariantSchema::ScalarType& scalar); + + /// Whether the physical struct type of a variant field in a data file is shredded (contains + /// a `typed_value` child). + static bool IsShreddedFileType(const std::shared_ptr& file_variant_type); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan.cpp new file mode 100644 index 00000000..96305db0 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan.cpp @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_write_plan.h" + +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_type_json_parser.h" +#include "rapidjson/document.h" + +namespace paimon { + +namespace { + +/// Recursively rebuilds `field`, replacing the variant fields planned under `paths` (grouped by +/// their leading index at this level) with their shredded physical types. +Result> ReplacePlannedFields( + const std::shared_ptr& field, + const std::map, std::shared_ptr>& paths, size_t depth, + std::vector* columns) { + const auto& struct_type = + arrow::internal::checked_cast(*field->type()); + arrow::FieldVector new_fields = struct_type.fields(); + bool changed = false; + auto it = paths.begin(); + while (it != paths.end()) { + int32_t index = it->first[depth]; + // Collect the consecutive paths that descend into the same child. + std::map, std::shared_ptr> child_paths; + for (; it != paths.end() && it->first[depth] == index; ++it) { + child_paths.emplace(it->first, it->second); + } + if (index < 0 || index >= struct_type.num_fields()) { + return Status::Invalid( + fmt::format("variant shredding path index {} is out of bounds", index)); + } + const std::shared_ptr& child = struct_type.field(index); + auto terminal = child_paths.begin(); + if (terminal->first.size() == depth + 1) { + // The path terminates at this child: it must be a variant field. + if (child_paths.size() > 1 || !VariantTypeUtils::IsVariantField(child)) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_type, + VariantShreddingUtils::VariantShreddingSchema(terminal->second)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant_schema, + VariantShreddingUtils::BuildVariantSchema(physical_type)); + columns->push_back(VariantShreddingWritePlan::PlannedColumn{ + terminal->first, std::move(variant_schema), physical_type}); + new_fields[index] = child->WithType(physical_type); + changed = true; + } else { + // The paths descend into a nested struct child. + if (child->type()->id() != arrow::Type::STRUCT || + VariantTypeUtils::IsVariantField(child)) { + continue; + } + size_t planned_before = columns->size(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr new_child, + ReplacePlannedFields(child, child_paths, depth + 1, columns)); + if (columns->size() > planned_before) { + new_fields[index] = new_child; + changed = true; + } + } + } + if (!changed) { + return field; + } + return field->WithType(arrow::struct_(new_fields)); +} + +} // namespace + +Result> VariantShreddingWritePlan::Create( + const std::shared_ptr& logical_schema, + const std::map>& column_shredding_types) { + std::map, std::shared_ptr> path_shredding_types; + for (int32_t i = 0; i < logical_schema->num_fields(); ++i) { + auto it = column_shredding_types.find(logical_schema->field(i)->name()); + if (it != column_shredding_types.end()) { + path_shredding_types.emplace(std::vector{i}, it->second); + } + } + return CreateFromPaths(logical_schema, path_shredding_types); +} + +Result> VariantShreddingWritePlan::CreateFromPaths( + const std::shared_ptr& logical_schema, + const std::map, std::shared_ptr>& path_shredding_types) { + std::vector columns; + auto root_field = arrow::field("root", arrow::struct_(logical_schema->fields())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr new_root, + ReplacePlannedFields(root_field, path_shredding_types, /*depth=*/0, &columns)); + if (columns.empty()) { + // No planned path matches a variant column; the file is written unshredded. + return std::shared_ptr(nullptr); + } + auto physical_schema = arrow::schema(new_root->type()->fields(), logical_schema->metadata()); + return std::shared_ptr(new VariantShreddingWritePlan( + logical_schema, std::move(physical_schema), std::move(columns))); +} + +Result> VariantShreddingWritePlan::FromConfiguredSchema( + const std::shared_ptr& logical_schema, + const std::string& configured_schema_json) { + rapidjson::Document doc; + doc.Parse(configured_schema_json.c_str()); + if (doc.HasParseError()) { + return Status::Invalid(fmt::format("failed to parse variant shredding schema json: {}", + configured_schema_json)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr configured_field, + DataTypeJsonParser::ParseType("shredding_schema", doc)); + if (configured_field->type()->id() != arrow::Type::STRUCT) { + return Status::Invalid("variant shredding schema must be a ROW type"); + } + std::map> column_shredding_types; + for (const auto& column_field : configured_field->type()->fields()) { + column_shredding_types.emplace(column_field->name(), column_field->type()); + } + return Create(logical_schema, column_shredding_types); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan.h b/src/paimon/common/data/variant/variant_shredding_write_plan.h new file mode 100644 index 00000000..d05faa25 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan.h @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/result.h" + +namespace arrow { +class DataType; +class Schema; +} // namespace arrow + +namespace paimon { + +/// A physical write plan for variant shredding: maps the logical write schema (with variant +/// columns) to the physical schema where planned variant columns are replaced by their shredded +/// struct representation. Variant columns may be at the top level or nested inside ROW (struct) +/// columns; variants nested inside arrays or maps are never shredded (as in Java). +class VariantShreddingWritePlan { + public: + /// One planned variant column, identified by its field-index path from the schema root, + /// descending only through struct fields (e.g. `{1, 2}` is the third field of the second + /// top-level column). + struct PlannedColumn { + std::vector path; + std::shared_ptr variant_schema; + std::shared_ptr physical_type; + }; + + /// Creates a plan shredding the given top-level variant columns. + /// + /// @param logical_schema The logical write schema. + /// @param column_shredding_types The shredding type per variant column name, e.g. + /// `{"v": struct{a: int32, b: string}}`. Names that are not top-level variant columns + /// of `logical_schema` are ignored. Returns nullptr when no name matches (the file is + /// written unshredded, mirroring the Java behavior). + static Result> Create( + const std::shared_ptr& logical_schema, + const std::map>& column_shredding_types); + + /// Creates a plan shredding the variant columns at the given field-index paths (top-level or + /// nested inside structs). Paths that do not point at a variant field are ignored. Returns + /// nullptr when no path matches. + static Result> CreateFromPaths( + const std::shared_ptr& logical_schema, + const std::map, std::shared_ptr>& + path_shredding_types); + + /// Creates a plan from the `variant.shreddingSchema` option value: a ROW type JSON whose + /// fields map top-level variant column names to their shredding types (nested variant + /// columns cannot be configured, as in Java). + static Result> FromConfiguredSchema( + const std::shared_ptr& logical_schema, + const std::string& configured_schema_json); + + const std::shared_ptr& LogicalSchema() const { + return logical_schema_; + } + + const std::shared_ptr& PhysicalSchema() const { + return physical_schema_; + } + + /// The planned variant columns, ordered by path. + const std::vector& Columns() const { + return columns_; + } + + private: + VariantShreddingWritePlan(std::shared_ptr logical_schema, + std::shared_ptr physical_schema, + std::vector columns) + : logical_schema_(std::move(logical_schema)), + physical_schema_(std::move(physical_schema)), + columns_(std::move(columns)) {} + + std::shared_ptr logical_schema_; + std::shared_ptr physical_schema_; + std::vector columns_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp new file mode 100644 index 00000000..bb7c5579 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/infer_variant_shredding_schema.h" +#include "paimon/common/data/variant/variant_shredding_batch_converter.h" +#include "paimon/common/data/variant/variant_shredding_write_plan.h" +#include "paimon/common/data/variant/variant_type_utils.h" + +namespace paimon { + +namespace { + +/// Collects the field-index paths of the shreddable variant fields: at the top level or nested +/// inside structs only, mirroring the Java `InferVariantShreddingSchema.getPathsToVariant`. +void CollectVariantPaths(const arrow::FieldVector& fields, std::vector* current, + std::vector>* paths) { + for (int32_t i = 0; i < static_cast(fields.size()); ++i) { + const std::shared_ptr& field = fields[i]; + current->push_back(i); + if (VariantTypeUtils::IsVariantField(field)) { + paths->push_back(*current); + } else if (field->type()->id() == arrow::Type::STRUCT) { + CollectVariantPaths(field->type()->fields(), current, paths); + } + current->pop_back(); + } +} + +std::vector> GetPathsToVariant(const arrow::Schema& schema) { + std::vector> paths; + std::vector current; + CollectVariantPaths(schema.fields(), ¤t, &paths); + return paths; +} + +/// Collects the non-null variant values of one sample batch at the given field-index path, +/// descending through struct arrays. Rows that are null at any level contribute no sample. +Result>> CollectSamplesAtPath( + const std::vector>& sample_batches, + const std::vector& path, const std::shared_ptr& pool) { + std::vector> samples; + for (const auto& sample_batch : sample_batches) { + std::shared_ptr column = sample_batch; + // The structs enclosing the variant column (the batch root excluded): child slot + // contents under a null ancestor are unspecified in Arrow and must not be decoded. + std::vector> ancestors; + for (int32_t index : path) { + if (column == nullptr || column->type_id() != arrow::Type::STRUCT) { + return Status::Invalid("sample batch does not match the variant column path"); + } + if (column != sample_batch) { + ancestors.push_back(column); + } + column = arrow::internal::checked_cast(*column).field(index); + } + if (column == nullptr) { + return Status::Invalid("sample batch misses the planned variant column"); + } + const auto& variant_array = + arrow::internal::checked_cast(*column); + const auto& value_array = + arrow::internal::checked_cast(*variant_array.field(0)); + const auto& metadata_array = + arrow::internal::checked_cast(*variant_array.field(1)); + auto row_is_null = [&](int64_t row) { + for (const auto& ancestor : ancestors) { + if (ancestor->IsNull(row)) { + return true; + } + } + return variant_array.IsNull(row); + }; + for (int64_t row = 0; row < variant_array.length(); ++row) { + if (row_is_null(row)) { + continue; + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr variant, + GenericVariant::Create(std::string_view(value_array.GetView(row)), + std::string_view(metadata_array.GetView(row)), pool)); + samples.push_back(std::move(variant)); + } + } + return samples; +} + +} // namespace + +VariantShreddingWritePlanFactory::VariantShreddingWritePlanFactory( + std::optional configured_schema, bool infer_enabled, int32_t max_schema_width, + int32_t max_schema_depth, double min_field_cardinality_ratio, int32_t max_infer_buffer_row, + const std::shared_ptr& write_schema, const std::shared_ptr& pool) + : write_schema_(write_schema), + pool_(pool), + configured_schema_(std::move(configured_schema)), + infer_enabled_(infer_enabled), + max_schema_width_(max_schema_width), + max_schema_depth_(max_schema_depth), + min_field_cardinality_ratio_(min_field_cardinality_ratio), + max_infer_buffer_row_(max_infer_buffer_row) {} + +std::shared_ptr VariantShreddingWritePlanFactory::Create( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& pool) { + return std::shared_ptr(new VariantShreddingWritePlanFactory( + options.GetVariantShreddingSchema(), options.VariantInferShreddingSchemaEnabled(), + options.GetVariantShreddingMaxSchemaWidth(), options.GetVariantShreddingMaxSchemaDepth(), + options.GetVariantShreddingMinFieldCardinalityRatio(), + options.GetVariantShreddingMaxInferBufferRow(), write_schema, pool)); +} + +bool VariantShreddingWritePlanFactory::ShouldCreateWritePlan() const { + return ContainsShreddableVariantField() && (HasConfiguredShreddingSchema() || infer_enabled_); +} + +bool VariantShreddingWritePlanFactory::ShouldInferWritePlan() const { + return ContainsShreddableVariantField() && !HasConfiguredShreddingSchema() && infer_enabled_; +} + +int32_t VariantShreddingWritePlanFactory::InferBufferRowCount() const { + return max_infer_buffer_row_; +} + +bool VariantShreddingWritePlanFactory::HasConfiguredShreddingSchema() const { + return configured_schema_.has_value(); +} + +bool VariantShreddingWritePlanFactory::ContainsShreddableVariantField() const { + return !GetPathsToVariant(*write_schema_).empty(); +} + +Result> VariantShreddingWritePlanFactory::CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) const { + if (file_format_identifier != "parquet") { + return Status::NotImplemented( + fmt::format("variant shredding is only supported by the parquet file format, got {}", + file_format_identifier)); + } + + std::shared_ptr plan; + if (HasConfiguredShreddingSchema()) { + PAIMON_ASSIGN_OR_RAISE(plan, VariantShreddingWritePlan::FromConfiguredSchema( + write_schema_, configured_schema_.value())); + } else { + InferVariantShreddingSchema inferrer(max_schema_width_, max_schema_depth_, + min_field_cardinality_ratio_); + // One budget is shared across all variant columns so that the total inferred width stays + // within `variant.shredding.maxSchemaWidth` (as in Java). + InferVariantShreddingSchema::MaxFields max_fields = inferrer.CreateMaxFieldsBudget(); + std::map, std::shared_ptr> path_shredding_types; + for (const std::vector& path : GetPathsToVariant(*write_schema_)) { + PAIMON_ASSIGN_OR_RAISE(std::vector> samples, + CollectSamplesAtPath(sample_batches, path, pool_)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_type, + inferrer.InferColumnShreddingType(samples, &max_fields)); + if (shredding_type != nullptr) { + path_shredding_types.emplace(path, std::move(shredding_type)); + } + } + if (path_shredding_types.empty()) { + // No useful shredding schema was found; write the file unshredded. + return std::shared_ptr(nullptr); + } + PAIMON_ASSIGN_OR_RAISE( + plan, VariantShreddingWritePlan::CreateFromPaths(write_schema_, path_shredding_types)); + } + if (plan == nullptr) { + // The configured schema names no variant column; write the file unshredded. + return std::shared_ptr(nullptr); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, + VariantShreddingBatchConverter::Create(plan, pool_)); + return std::shared_ptr(std::move(converter)); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h new file mode 100644 index 00000000..098da8f0 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" +#include "paimon/core/core_options.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class Schema; +} // namespace arrow + +namespace paimon { + +/// Creates VARIANT shredding batch converters, either from the configured +/// `variant.shreddingSchema` or, when `variant.inferShreddingSchema` is enabled, by inferring a +/// shredding schema from sampled rows buffered per file. Variant columns nested inside ROW +/// (struct) columns are inferred and shredded too; variants inside arrays or maps are not (as in +/// Java). +class VariantShreddingWritePlanFactory : public ShreddingWritePlanFactory { + public: + /// Creates the factory. The `variant.*` option values are already parsed and validated by + /// `CoreOptions::FromMap`, so creation cannot fail on configuration. + static std::shared_ptr Create( + const CoreOptions& options, const std::shared_ptr& write_schema, + const std::shared_ptr& pool); + + bool ShouldCreateWritePlan() const override; + + bool ShouldInferWritePlan() const override; + + int32_t InferBufferRowCount() const override; + + Result> CreateConverter( + const std::string& file_format_identifier, + const std::vector>& sample_batches) const override; + + MetadataFinalizer CreateMetadataFinalizer( + const std::shared_ptr& converter) const override { + // The shredded physical schema is self-describing; no per-file metadata is needed. + return nullptr; + } + + private: + VariantShreddingWritePlanFactory(std::optional configured_schema, + bool infer_enabled, int32_t max_schema_width, + int32_t max_schema_depth, double min_field_cardinality_ratio, + int32_t max_infer_buffer_row, + const std::shared_ptr& write_schema, + const std::shared_ptr& pool); + + bool HasConfiguredShreddingSchema() const; + /// Whether the write schema holds a shreddable variant field: at the top level or nested + /// inside structs only (variants inside arrays or maps are never shredded). + bool ContainsShreddableVariantField() const; + + std::shared_ptr write_schema_; + std::shared_ptr pool_; + + std::optional configured_schema_; + bool infer_enabled_ = false; + int32_t max_schema_width_ = 0; + int32_t max_schema_depth_ = 0; + double min_field_cardinality_ratio_ = 0.0; + int32_t max_infer_buffer_row_ = 0; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp new file mode 100644 index 00000000..b5b09e33 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/core_options.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/variant_test_data.h" + +namespace paimon::test { + +class VariantShreddingWritePlanFactoryTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("v"))}; + schema_ = DataField::ConvertDataFieldsToArrowSchema(fields); + } + + Result MakeOptions(std::map options) const { + // Keep the manifest format resolvable in test binaries without the avro plugin. + options.emplace("manifest.format", "parquet"); + return CoreOptions::FromMap(options); + } + + std::shared_ptr BuildBatch(const std::vector& jsons) { + auto result = + VariantTestData::BuildVariantBatch(schema_->field(0), schema_->field(1), jsons, pool_); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return std::move(result).value(); + } + + protected: + std::shared_ptr pool_; + std::shared_ptr schema_; +}; + +TEST_F(VariantShreddingWritePlanFactoryTest, InactiveWithoutOptions) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeOptions({})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + ASSERT_FALSE(factory->ShouldCreateWritePlan()); + ASSERT_FALSE(factory->ShouldInferWritePlan()); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, ConfiguredSchema) { + const char* shredding_schema_json = R"({ + "type": "ROW", + "fields": [ { + "id": 0, + "name": "v", + "type": { + "type": "ROW", + "fields": [ + {"id": 1, "name": "age", "type": "INT"}, + {"id": 2, "name": "city", "type": "STRING"} + ] + } + } ] + })"; + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.shreddingSchema", shredding_schema_json}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + ASSERT_TRUE(factory->ShouldCreateWritePlan()); + ASSERT_FALSE(factory->ShouldInferWritePlan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", {})); + ASSERT_NE(converter, nullptr); + auto variant_field = converter->GetPhysicalSchema()->GetFieldByName("v"); + ASSERT_NE(variant_field, nullptr); + const auto& physical_type = static_cast(*variant_field->type()); + ASSERT_NE(physical_type.GetFieldByName("typed_value"), nullptr); + // Variant shredding only supports the parquet format. + ASSERT_TRUE(factory->CreateConverter("orc", {}).status().IsNotImplemented()); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, InferredSchema) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + ASSERT_TRUE(factory->ShouldCreateWritePlan()); + ASSERT_TRUE(factory->ShouldInferWritePlan()); + ASSERT_EQ(factory->InferBufferRowCount(), 4096); + + std::vector> samples = { + BuildBatch({R"({"age": 35, "city": "Chicago"})", R"({"age": 25, "city": "Hangzhou"})"}), + BuildBatch({R"({"age": 18, "city": "Beijing"})", nullptr})}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", samples)); + ASSERT_NE(converter, nullptr); + auto variant_field = converter->GetPhysicalSchema()->GetFieldByName("v"); + ASSERT_NE(variant_field, nullptr); + const auto& physical_type = static_cast(*variant_field->type()); + auto typed_value = physical_type.GetFieldByName("typed_value"); + ASSERT_NE(typed_value, nullptr); + const auto& typed_struct = static_cast(*typed_value->type()); + ASSERT_NE(typed_struct.GetFieldByName("age"), nullptr); + ASSERT_NE(typed_struct.GetFieldByName("city"), nullptr); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, InferredSchemaWithoutSamples) { + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + // With no useful samples the file stays unshredded. + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", {})); + ASSERT_EQ(converter, nullptr); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, SharedWidthBudgetAcrossColumns) { + // Two variant columns share one maxSchemaWidth budget (as in Java): with a small limit the + // first column consumes it and the second column stays unshredded. + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("v1")), + DataField(3, VariantTypeUtils::ToArrowField("v2"))}; + auto schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}, + {"variant.shredding.maxSchemaWidth", "3"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_); + ASSERT_TRUE(factory->ShouldInferWritePlan()); + + auto build_batch = [&](const std::vector& v1_jsons, + const std::vector& v2_jsons) { + auto v1 = + VariantTestData::BuildVariantBatch(schema->field(0), schema->field(1), v1_jsons, pool_); + EXPECT_TRUE(v1.ok()) << v1.status().ToString(); + auto v2 = + VariantTestData::BuildVariantBatch(schema->field(0), schema->field(2), v2_jsons, pool_); + EXPECT_TRUE(v2.ok()) << v2.status().ToString(); + auto batch = arrow::StructArray::Make( + {v1.value()->field(0), v1.value()->field(1), v2.value()->field(1)}, + {schema->field(0), schema->field(1), schema->field(2)}) + .ValueOrDie(); + return std::shared_ptr(batch); + }; + std::vector> samples = { + build_batch({R"({"a": 1, "b": 2})"}, {R"({"c": 3})"})}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", samples)); + ASSERT_NE(converter, nullptr); + const auto& physical = converter->GetPhysicalSchema(); + // v1 got the budget and is shredded; v2 exceeded it and stays unshredded. + const auto& v1_type = + static_cast(*physical->GetFieldByName("v1")->type()); + ASSERT_NE(v1_type.GetFieldByName("typed_value"), nullptr); + const auto& v2_type = + static_cast(*physical->GetFieldByName("v2")->type()); + ASSERT_EQ(v2_type.GetFieldByName("typed_value"), nullptr); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, NestedVariantInsideStruct) { + // A VARIANT nested inside a ROW column is discovered, inferred and shredded (as in Java); + // top-level detection also recurses through structs. + auto nested_variant = VariantTypeUtils::ToArrowField("nv"); + auto struct_field = arrow::field("s", arrow::struct_({nested_variant})); + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, struct_field)}; + auto schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.inferShreddingSchema", "true"}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_); + ASSERT_TRUE(factory->ShouldCreateWritePlan()); + ASSERT_TRUE(factory->ShouldInferWritePlan()); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variant_batch, + VariantTestData::BuildVariantBatch(schema->field(0), schema->field(1)->type()->field(0), + {R"({"age": 35, "city": "Chicago"})"}, pool_)); + std::shared_ptr struct_column = + arrow::StructArray::Make({variant_batch->field(1)}, {schema->field(1)->type()->field(0)}) + .ValueOrDie(); + std::shared_ptr batch = + arrow::StructArray::Make({variant_batch->field(0), struct_column}, + {schema->field(0), schema->field(1)}) + .ValueOrDie(); + std::vector> samples = {batch}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", samples)); + ASSERT_NE(converter, nullptr); + const auto& physical = converter->GetPhysicalSchema(); + const auto& physical_struct = + static_cast(*physical->GetFieldByName("s")->type()); + const auto& physical_variant = + static_cast(*physical_struct.GetFieldByName("nv")->type()); + auto typed_value = physical_variant.GetFieldByName("typed_value"); + ASSERT_NE(typed_value, nullptr); + const auto& typed_struct = static_cast(*typed_value->type()); + ASSERT_NE(typed_struct.GetFieldByName("age"), nullptr); + ASSERT_NE(typed_struct.GetFieldByName("city"), nullptr); + + // Child slot contents under a null struct row are unspecified in Arrow: a row whose parent + // struct is null must be skipped by sampling and shredded to null by conversion, without + // decoding the (here invalid) variant bytes underneath. + arrow::BinaryBuilder garbage_builder; + ASSERT_TRUE(garbage_builder.Append("not a variant").ok()); + std::shared_ptr garbage; + ASSERT_TRUE(garbage_builder.Finish(&garbage).ok()); + std::shared_ptr garbage_variant = + arrow::StructArray::Make({garbage, garbage}, {nested_variant->type()->field(0), + nested_variant->type()->field(1)}) + .ValueOrDie(); + // Mark the single parent row null (a zeroed bitmap) while the child slots keep their bytes. + auto null_struct_column = std::make_shared( + struct_field->type(), 1, arrow::ArrayVector{garbage_variant}, + arrow::AllocateEmptyBitmap(1).ValueOrDie(), 1); + std::shared_ptr garbage_batch = + arrow::StructArray::Make({variant_batch->field(0), null_struct_column}, + {schema->field(0), schema->field(1)}) + .ValueOrDie(); + + // Sampling sees no usable value: the file stays unshredded. + std::vector> garbage_samples = {garbage_batch}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr unshredded_converter, + factory->CreateConverter("parquet", garbage_samples)); + ASSERT_EQ(unshredded_converter, nullptr); + + // Conversion with the previously inferred plan shreds the row to null instead of failing. + auto c_garbage_batch = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*garbage_batch, c_garbage_batch.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_physical, + converter->Convert(c_garbage_batch.get())); + auto physical_struct_type = arrow::struct_(physical->fields()); + std::shared_ptr physical_array = + arrow::ImportArray(c_physical.get(), physical_struct_type).ValueOrDie(); + const auto& physical_row = static_cast(*physical_array); + const auto& converted_struct = static_cast(*physical_row.field(1)); + ASSERT_TRUE(converted_struct.IsNull(0)); +} + +TEST_F(VariantShreddingWritePlanFactoryTest, ConfiguredSchemaMatchingNoColumn) { + // A configured schema naming no variant column writes the file unshredded (as in Java) + // instead of failing. + const char* shredding_schema_json = R"({ + "type": "ROW", + "fields": [ { + "id": 0, + "name": "not_a_column", + "type": {"type": "ROW", "fields": [{"id": 1, "name": "a", "type": "INT"}]} + } ] + })"; + ASSERT_OK_AND_ASSIGN(CoreOptions options, + MakeOptions({{"variant.shreddingSchema", shredding_schema_json}})); + auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + ASSERT_TRUE(factory->ShouldCreateWritePlan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + factory->CreateConverter("parquet", {})); + ASSERT_EQ(converter, nullptr); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_shredding_writer.cpp b/src/paimon/common/data/variant/variant_shredding_writer.cpp new file mode 100644 index 00000000..bb7c412e --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_writer.cpp @@ -0,0 +1,473 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#include "paimon/common/data/variant/variant_shredding_writer.h" + +#include +#include + +#include "arrow/api.h" +#include "arrow/util/checked_cast.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_builder.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +namespace { + +// Rescales `unscaled` by `10^power`, failing when the result would exceed the 38-digit +// limit. The precision check must precede each multiplication: a 38-digit value times 10 +// overflows the signed 128-bit representation before a post-check could reject it. +Result<__int128_t> ScaleUpUnscaled(__int128_t unscaled, int32_t power) { + __int128_t result = unscaled; + for (int32_t i = 0; i < power; ++i) { + VariantDecimal probe{result, 0}; + if (probe.Precision() >= VariantDefs::kMaxDecimal16Precision) { + return Status::Invalid("decimal overflow while rescaling"); + } + result *= 10; + } + return result; +} + +Status AppendDecimalTo(__int128_t unscaled, arrow::ArrayBuilder* builder) { + auto* decimal_builder = static_cast(builder); + arrow::Decimal128 value(static_cast(unscaled >> 64), + static_cast(static_cast<__uint128_t>(unscaled))); + PAIMON_RETURN_NOT_OK_FROM_ARROW(decimal_builder->Append(value)); + return Status::OK(); +} + +} // namespace + +VariantShreddedColumnWriter::VariantShreddedColumnWriter( + const std::shared_ptr& schema, + std::unique_ptr&& root_builder) + : schema_(schema), root_builder_(std::move(root_builder)) {} + +Result> VariantShreddedColumnWriter::Create( + const std::shared_ptr& schema, + const std::shared_ptr& physical_type, arrow::MemoryPool* pool) { + if (!schema || schema->top_level_metadata_idx < 0) { + return Status::Invalid("variant shredding schema must contain a top-level metadata field"); + } + std::unique_ptr root_builder; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::MakeBuilder(pool, physical_type, &root_builder)); + if (root_builder->type()->id() != arrow::Type::STRUCT) { + return Status::Invalid("variant shredded physical type must be a struct"); + } + auto writer = std::unique_ptr( + new VariantShreddedColumnWriter(schema, std::move(root_builder))); + PAIMON_RETURN_NOT_OK(BuildNode( + schema, static_cast(writer->root_builder_.get()), &writer->root_)); + return writer; +} + +Status VariantShreddedColumnWriter::BuildNode(const std::shared_ptr& schema, + arrow::StructBuilder* group, Node* node) { + node->schema = schema.get(); + node->group = group; + if (group->num_children() != schema->num_fields) { + return Status::Invalid( + fmt::format("variant shredded builder has {} children but the schema has {} fields", + group->num_children(), schema->num_fields)); + } + if (schema->top_level_metadata_idx >= 0) { + node->metadata = static_cast( + group->field_builder(schema->top_level_metadata_idx)); + } + if (schema->variant_idx >= 0) { + node->value = static_cast(group->field_builder(schema->variant_idx)); + } + if (schema->typed_idx >= 0) { + arrow::ArrayBuilder* typed_builder = group->field_builder(schema->typed_idx); + if (schema->has_object_schema) { + node->typed_object = static_cast(typed_builder); + node->object_children.resize(schema->object_schema.size()); + for (size_t i = 0; i < schema->object_schema.size(); ++i) { + auto* child_group = static_cast( + node->typed_object->field_builder(static_cast(i))); + PAIMON_RETURN_NOT_OK(BuildNode(schema->object_schema[i].schema, child_group, + &node->object_children[i])); + } + } else if (schema->array_schema) { + node->typed_list = static_cast(typed_builder); + node->array_element = std::make_unique(); + auto* element_group = + static_cast(node->typed_list->value_builder()); + PAIMON_RETURN_NOT_OK( + BuildNode(schema->array_schema, element_group, node->array_element.get())); + } else if (schema->scalar_schema) { + node->typed_scalar = typed_builder; + } else { + return Status::Invalid("variant shredding schema typed_value has no schema"); + } + } + return Status::OK(); +} + +Status VariantShreddedColumnWriter::Append(const GenericVariant& variant) { + return AppendVariantNode(variant, &root_); +} + +Status VariantShreddedColumnWriter::AppendNull() { + PAIMON_RETURN_NOT_OK_FROM_ARROW(root_.group->AppendNull()); + return Status::OK(); +} + +Result> VariantShreddedColumnWriter::Finish() { + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(root_builder_->Finish(&array)); + return array; +} + +Status VariantShreddedColumnWriter::AppendVariantNode(const GenericVariant& variant, Node* node) { + const VariantSchema& schema = *node->schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->group->Append()); + if (schema.top_level_metadata_idx >= 0) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->metadata->Append(variant.Metadata())); + } + PAIMON_ASSIGN_OR_RAISE(VariantValueType variant_type, variant.GetType()); + if (schema.array_schema != nullptr && variant_type == VariantValueType::kArray) { + // The array element is always a struct containing untyped and typed fields. + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_list->Append()); + PAIMON_ASSIGN_OR_RAISE(int32_t size, variant.ArraySize()); + for (int32_t i = 0; i < size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr element, + variant.GetElementAtIndex(i)); + PAIMON_RETURN_NOT_OK(AppendVariantNode(*element, node->array_element.get())); + } + if (node->value != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->AppendNull()); + } + } else if (schema.has_object_schema && variant_type == VariantValueType::kObject) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_object->Append()); + // Collect any field that exists in the variant, but not in the shredding schema, into a + // residual variant object that shares the top-level metadata. + VariantBuilder residual(/*allow_duplicate_keys=*/false); + std::vector field_entries; + std::vector matched(node->object_children.size(), false); + int32_t start = residual.GetWritePos(); + PAIMON_ASSIGN_OR_RAISE(int32_t object_size, variant.ObjectSize()); + for (int32_t i = 0; i < object_size; ++i) { + PAIMON_ASSIGN_OR_RAISE(std::optional field, + variant.GetFieldAtIndex(i)); + if (!field.has_value()) { + return VariantBinaryUtil::MalformedVariant("an object field is missing"); + } + auto it = schema.object_schema_map.find(field->key); + if (it != schema.object_schema_map.end()) { + PAIMON_RETURN_NOT_OK( + AppendVariantNode(*field->value, &node->object_children[it->second])); + matched[it->second] = true; + } else { + // The field is not shredded. Put it in the untyped value column. The shallow + // append is needed for correctness, since the metadata ids must stay unchanged. + PAIMON_ASSIGN_OR_RAISE(int32_t id, variant.GetDictionaryIdAtIndex(i)); + field_entries.emplace_back(field->key, id, residual.GetWritePos() - start); + PAIMON_RETURN_NOT_OK( + residual.ShallowAppendVariant(field->value->RawValue(), field->value->Pos())); + } + } + // Set missing fields to non-null with all fields set to null. + for (size_t i = 0; i < matched.size(); ++i) { + if (!matched[i]) { + PAIMON_RETURN_NOT_OK(AppendMissingNode(&node->object_children[i])); + } + } + if (residual.GetWritePos() != start) { + PAIMON_RETURN_NOT_OK(residual.FinishWritingObject(start, &field_entries)); + if (node->value == nullptr) { + return Status::Invalid( + "variant shredding schema has no value column for residual fields"); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->Append(residual.ValueWithoutMetadata())); + } else if (node->value != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->AppendNull()); + } + } else if (schema.scalar_schema.has_value()) { + bool shredded = false; + PAIMON_RETURN_NOT_OK(TryTypedShred(variant, variant_type, node, &shredded)); + if (shredded) { + if (node->value != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->AppendNull()); + } + } else { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_scalar->AppendNull()); + if (node->value == nullptr) { + return Status::Invalid( + "variant shredding schema has no value column for untyped values"); + } + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant.Value()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->Append(value)); + } + } else { + if (node->typed_list != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_list->AppendNull()); + } else if (node->typed_object != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_object->AppendNull()); + } else if (node->typed_scalar != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_scalar->AppendNull()); + } + if (node->value == nullptr) { + return Status::Invalid( + "variant shredding schema has no value column for untyped values"); + } + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant.Value()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->Append(value)); + } + return Status::OK(); +} + +Status VariantShreddedColumnWriter::AppendMissingNode(Node* node) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->group->Append()); + if (node->metadata != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->metadata->AppendNull()); + } + if (node->value != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->value->AppendNull()); + } + if (node->typed_list != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_list->AppendNull()); + } else if (node->typed_object != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_object->AppendNull()); + } else if (node->typed_scalar != nullptr) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(node->typed_scalar->AppendNull()); + } + return Status::OK(); +} + +Status VariantShreddedColumnWriter::TryTypedShred(const GenericVariant& variant, + VariantValueType variant_type, Node* node, + bool* shredded) { + const VariantSchema::ScalarType& target = node->schema->scalar_schema.value(); + *shredded = false; + switch (variant_type) { + case VariantValueType::kLong: { + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant.GetLong()); + switch (target.kind) { + // Check that the target type can hold the actual value. + case VariantSchema::ScalarKind::kByte: + if (value == static_cast(value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kShort: + if (value == static_cast(value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kInt: + if (value == static_cast(value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kLong: + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + break; + case VariantSchema::ScalarKind::kDecimal: { + // If the integer can fit in the given decimal precision, allow it. + auto scaled = ScaleUpUnscaled(value, target.scale); + if (scaled.ok()) { + VariantDecimal probe{scaled.value(), target.scale}; + if (probe.Precision() <= target.precision) { + PAIMON_RETURN_NOT_OK( + AppendDecimalTo(scaled.value(), node->typed_scalar)); + *shredded = true; + } + } + break; + } + default: + break; + } + break; + } + case VariantValueType::kDecimal: { + if (target.kind == VariantSchema::ScalarKind::kDecimal) { + // Use the original scale so that scale information is retained. + PAIMON_ASSIGN_OR_RAISE(VariantDecimal value, + VariantBinaryUtil::GetDecimalWithOriginalScale( + variant.RawValue(), variant.Pos())); + if (value.Precision() <= target.precision && value.scale == target.scale) { + PAIMON_RETURN_NOT_OK(AppendDecimalTo(value.unscaled, node->typed_scalar)); + *shredded = true; + break; + } + // Convert to the target scale, and see if it fits without losing information. + int32_t scale_diff = target.scale - value.scale; + __int128_t rescaled = value.unscaled; + bool exact = true; + if (scale_diff >= 0) { + auto scaled = ScaleUpUnscaled(value.unscaled, scale_diff); + if (scaled.ok()) { + rescaled = scaled.value(); + } else { + exact = false; + } + } else { + for (int32_t i = 0; i < -scale_diff && exact; ++i) { + if (rescaled % 10 != 0) { + exact = false; + } else { + rescaled /= 10; + } + } + } + if (exact) { + VariantDecimal probe{rescaled, target.scale}; + if (probe.Precision() <= target.precision) { + PAIMON_RETURN_NOT_OK(AppendDecimalTo(rescaled, node->typed_scalar)); + *shredded = true; + } + } + } else if (target.kind == VariantSchema::ScalarKind::kByte || + target.kind == VariantSchema::ScalarKind::kShort || + target.kind == VariantSchema::ScalarKind::kInt || + target.kind == VariantSchema::ScalarKind::kLong) { + // Check if the decimal happens to be an integer. + PAIMON_ASSIGN_OR_RAISE(VariantDecimal value, variant.GetDecimal()); + if (value.scale > 0) { + break; + } + auto scaled = ScaleUpUnscaled(value.unscaled, -value.scale); + if (!scaled.ok()) { + break; + } + __int128_t integral = scaled.value(); + if (integral != static_cast(integral)) { + break; + } + auto long_value = static_cast(integral); + switch (target.kind) { + case VariantSchema::ScalarKind::kByte: + if (long_value == static_cast(long_value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(long_value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kShort: + if (long_value == static_cast(long_value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(long_value))); + *shredded = true; + } + break; + case VariantSchema::ScalarKind::kInt: + if (long_value == static_cast(long_value)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(long_value))); + *shredded = true; + } + break; + default: + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(long_value)); + *shredded = true; + break; + } + } + break; + } + case VariantValueType::kBoolean: { + if (target.kind == VariantSchema::ScalarKind::kBoolean) { + PAIMON_ASSIGN_OR_RAISE(bool value, variant.GetBoolean()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + case VariantValueType::kString: { + if (target.kind == VariantSchema::ScalarKind::kString) { + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant.GetString()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + case VariantValueType::kDouble: { + if (target.kind == VariantSchema::ScalarKind::kDouble) { + PAIMON_ASSIGN_OR_RAISE(double value, variant.GetDouble()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + case VariantValueType::kFloat: { + if (target.kind == VariantSchema::ScalarKind::kFloat) { + PAIMON_ASSIGN_OR_RAISE(float value, variant.GetFloat()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + case VariantValueType::kDate: { + if (target.kind == VariantSchema::ScalarKind::kDate) { + PAIMON_ASSIGN_OR_RAISE(int64_t value, variant.GetLong()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar) + ->Append(static_cast(value))); + *shredded = true; + } + break; + } + case VariantValueType::kBinary: { + if (target.kind == VariantSchema::ScalarKind::kBinary) { + PAIMON_ASSIGN_OR_RAISE(std::string_view value, variant.GetBinary()); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + static_cast(node->typed_scalar)->Append(value)); + *shredded = true; + } + break; + } + default: + // TIMESTAMP/TIMESTAMP_NTZ/UUID typed columns are not producible by the configured + // shredding schema; the value stays in the untyped column. + break; + } + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_shredding_writer.h b/src/paimon/common/data/variant/variant_shredding_writer.h new file mode 100644 index 00000000..81723c28 --- /dev/null +++ b/src/paimon/common/data/variant/variant_shredding_writer.h @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* This file is based on source code from the Spark Project (http://spark.apache.org/), licensed + * by the Apache Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. */ + +#pragma once + +#include +#include + +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/result.h" + +namespace arrow { +class Array; +class ArrayBuilder; +class BinaryBuilder; +class DataType; +class ListBuilder; +class MemoryPool; +class StructBuilder; +} // namespace arrow + +namespace paimon { + +/// Shreds variant values of one column into a physical shredded Arrow array, implementing the +/// `castShredded` algorithm of the parquet-format VariantShredding.md specification (mirroring +/// the Java `VariantShreddingWriter`). Decimals and integers are allowed to shred to numerically +/// equivalent values of a different scale (`allowNumericScaleChanges` in Java is always true). +class VariantShreddedColumnWriter { + public: + /// Creates a writer for one variant column. + /// + /// @param schema The shredding schema of the column. + /// @param physical_type The physical shredded struct type + /// (`VariantShreddingUtils::VariantShreddingSchema` output). + /// @param pool The Arrow memory pool used by the builders. + static Result> Create( + const std::shared_ptr& schema, + const std::shared_ptr& physical_type, arrow::MemoryPool* pool); + + /// Shreds one variant value and appends the result row. + Status Append(const GenericVariant& variant); + + /// Appends a null variant row. + Status AppendNull(); + + /// Finishes and returns the shredded array of all appended rows. + Result> Finish(); + + private: + /// Builder handles of one shredding schema node (a `metadata`/`value`/`typed_value` group). + struct Node { + const VariantSchema* schema = nullptr; + arrow::StructBuilder* group = nullptr; + arrow::BinaryBuilder* metadata = nullptr; + arrow::BinaryBuilder* value = nullptr; + // Exactly one of the following is set when `schema->typed_idx >= 0`. + arrow::ArrayBuilder* typed_scalar = nullptr; + arrow::ListBuilder* typed_list = nullptr; + arrow::StructBuilder* typed_object = nullptr; + std::vector object_children; + std::unique_ptr array_element; + }; + + VariantShreddedColumnWriter(const std::shared_ptr& schema, + std::unique_ptr&& root_builder); + + static Status BuildNode(const std::shared_ptr& schema, + arrow::StructBuilder* group, Node* node); + + Status AppendVariantNode(const GenericVariant& variant, Node* node); + + /// Appends a missing object field: the group is present with all its children set to null. + Status AppendMissingNode(Node* node); + + /// Tries to append the variant as a typed scalar. Sets `*shredded` to whether it succeeded + /// (on failure nothing is appended). + Status TryTypedShred(const GenericVariant& variant, VariantValueType variant_type, Node* node, + bool* shredded); + + std::shared_ptr schema_; + std::unique_ptr root_builder_; + Node root_; +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_test.cpp b/src/paimon/common/data/variant/variant_test.cpp new file mode 100644 index 00000000..f35f5d8e --- /dev/null +++ b/src/paimon/common/data/variant/variant_test.cpp @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/data/variant.h" + +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +class VariantPublicApiTest : public ::testing::Test { + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(VariantPublicApiTest, FromJsonAndAccessors) { + ASSERT_OK_AND_ASSIGN(auto variant, + Variant::FromJson("{\"age\": 35, \"city\": \"Hangzhou\"}", pool_)); + ASSERT_GT(variant->Value().size(), 0); + ASSERT_GT(variant->Metadata().size(), 0); + ASSERT_EQ(variant->SizeInBytes(), + static_cast(variant->Value().size() + variant->Metadata().size())); + ASSERT_OK_AND_ASSIGN(std::string json, variant->ToJson()); + ASSERT_EQ(json, "{\"age\":35,\"city\":\"Hangzhou\"}"); + + ASSERT_OK_AND_ASSIGN( + auto rebuilt, + Variant::Create(variant->Value().data(), variant->Value().size(), + variant->Metadata().data(), variant->Metadata().size(), pool_)); + ASSERT_OK_AND_ASSIGN(std::string rebuilt_json, rebuilt->ToJson()); + ASSERT_EQ(rebuilt_json, json); +} + +TEST_F(VariantPublicApiTest, VariantGet) { + ASSERT_OK_AND_ASSIGN(auto variant, + Variant::FromJson("{\"age\": 35, \"city\": \"Hangzhou\"}", pool_)); + VariantCastArgs cast_args; + cast_args.fail_on_error = false; + { + auto target = std::make_unique(); + ASSERT_TRUE(arrow::ExportField(arrow::Field("t", arrow::int64()), target.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::optional literal, + variant->VariantGet("$.age", target.get(), cast_args)); + ASSERT_TRUE(literal.has_value()); + ASSERT_EQ(literal->GetValue(), 35); + } + { + auto target = std::make_unique(); + ASSERT_TRUE(arrow::ExportField(arrow::Field("t", arrow::utf8()), target.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::optional literal, + variant->VariantGet("$.missing", target.get(), cast_args)); + ASSERT_FALSE(literal.has_value()); + } + ASSERT_OK_AND_ASSIGN(std::optional sub_json, variant->VariantGetJson("$")); + ASSERT_TRUE(sub_json.has_value()); + ASSERT_EQ(*sub_json, "{\"age\":35,\"city\":\"Hangzhou\"}"); + ASSERT_OK_AND_ASSIGN(std::optional missing_json, + variant->VariantGetJson("$.missing")); + ASSERT_FALSE(missing_json.has_value()); +} + +TEST_F(VariantPublicApiTest, ArrowField) { + ASSERT_OK_AND_ASSIGN(auto c_field, Variant::ArrowField("v", /*nullable=*/true)); + auto imported = arrow::ImportField(c_field.get()); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + std::shared_ptr field = imported.ValueOrDie(); + ASSERT_TRUE(VariantTypeUtils::IsVariantField(field)); + ASSERT_TRUE(field->nullable()); + ASSERT_TRUE(field->type()->Equals(VariantTypeUtils::UnshreddedStructType())); +} + +TEST_F(VariantPublicApiTest, VariantGetArrow) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr variant, + Variant::FromJson("{\"user\": {\"name\": \"Paimon\"}}", pool_)); + auto target_type = arrow::struct_({arrow::field("name", arrow::utf8())}); + auto target = std::make_unique(); + ASSERT_TRUE(arrow::ExportField(arrow::Field("t", target_type), target.get()).ok()); + VariantCastArgs cast_args; + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_array, + variant->VariantGetArrow("$.user", target.get(), cast_args)); + auto imported = arrow::ImportArray(c_array.get(), target_type); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + std::shared_ptr array = imported.ValueOrDie(); + ASSERT_EQ(array->length(), 1); + const auto& row = static_cast(*array); + ASSERT_EQ(static_cast(*row.field(0)).GetString(0), "Paimon"); +} + +} // namespace paimon::test diff --git a/src/paimon/common/data/variant/variant_type_utils.cpp b/src/paimon/common/data/variant/variant_type_utils.cpp new file mode 100644 index 00000000..6bb494e1 --- /dev/null +++ b/src/paimon/common/data/variant/variant_type_utils.cpp @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_type_utils.h" + +#include "arrow/api.h" +#include "fmt/format.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/types/data_field.h" + +namespace paimon { + +bool VariantTypeUtils::IsVariantField(const std::shared_ptr& field) { + if (field->type()->id() != arrow::Type::STRUCT) { + return false; + } + if (!field->HasMetadata()) { + return false; + } + return IsVariantMetadata(field->metadata()); +} + +bool VariantTypeUtils::IsUnshreddedVariantType(const std::shared_ptr& type) { + if (type == nullptr || type->id() != arrow::Type::STRUCT || type->num_fields() != 2) { + return false; + } + const auto& value_field = type->field(0); + const auto& metadata_field = type->field(1); + return value_field->name() == VariantDefs::kValueFieldName && + value_field->type()->id() == arrow::Type::BINARY && !value_field->nullable() && + metadata_field->name() == VariantDefs::kMetadataFieldName && + metadata_field->type()->id() == arrow::Type::BINARY && !metadata_field->nullable(); +} + +bool VariantTypeUtils::IsVariantMetadata( + const std::shared_ptr& metadata) { + if (!metadata) { + return false; + } + auto extension_name = metadata->Get(VariantDefs::kExtensionTypeKey); + return extension_name.ok() && *extension_name == VariantDefs::kExtensionTypeValue; +} + +std::shared_ptr VariantTypeUtils::UnshreddedStructType() { + auto value_field = + arrow::field(VariantDefs::kValueFieldName, arrow::binary(), /*nullable=*/false, + arrow::KeyValueMetadata::Make({DataField::FIELD_ID}, + {std::to_string(VariantDefs::kValueFieldId)})); + auto metadata_field = + arrow::field(VariantDefs::kMetadataFieldName, arrow::binary(), /*nullable=*/false, + arrow::KeyValueMetadata::Make( + {DataField::FIELD_ID}, {std::to_string(VariantDefs::kMetadataFieldId)})); + return arrow::struct_({value_field, metadata_field}); +} + +std::shared_ptr VariantTypeUtils::ToArrowField( + const std::string& field_name, bool nullable, + std::unordered_map metadata) { + metadata[VariantDefs::kExtensionTypeKey] = VariantDefs::kExtensionTypeValue; + return arrow::field(field_name, UnshreddedStructType(), nullable, + std::make_shared(metadata)); +} + +Status VariantTypeUtils::ValidateVariantShape(const std::shared_ptr& field) { + const auto& type = field->type(); + if (type->id() != arrow::Type::STRUCT) { + return Status::Invalid(fmt::format("Variant field '{}' must be a struct, but got {}", + field->name(), type->ToString())); + } + const auto& struct_type = std::static_pointer_cast(type); + if (struct_type->num_fields() != 2) { + return Status::Invalid( + fmt::format("Variant field '{}' must be a struct, " + "but got {}", + field->name(), type->ToString())); + } + const auto& value_field = struct_type->field(0); + const auto& metadata_field = struct_type->field(1); + if (value_field->name() != VariantDefs::kValueFieldName || + value_field->type()->id() != arrow::Type::BINARY || value_field->nullable() || + metadata_field->name() != VariantDefs::kMetadataFieldName || + metadata_field->type()->id() != arrow::Type::BINARY || metadata_field->nullable()) { + return Status::Invalid( + fmt::format("Variant field '{}' must be a struct, but got {}", + field->name(), type->ToString())); + } + return Status::OK(); +} + +bool VariantTypeUtils::ContainsVariantField(const std::shared_ptr& field) { + if (IsVariantField(field)) { + return true; + } + for (const auto& child : field->type()->fields()) { + if (ContainsVariantField(child)) { + return true; + } + } + return false; +} + +bool VariantTypeUtils::ContainsVariantField(const std::shared_ptr& schema) { + for (const auto& field : schema->fields()) { + if (ContainsVariantField(field)) { + return true; + } + } + return false; +} + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_type_utils.h b/src/paimon/common/data/variant/variant_type_utils.h new file mode 100644 index 00000000..89b8201e --- /dev/null +++ b/src/paimon/common/data/variant/variant_type_utils.h @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace arrow { +class DataType; +class Field; +class KeyValueMetadata; +class Schema; +} // namespace arrow + +namespace paimon { + +/// Utils for the Paimon Variant type, whose underlying Arrow representation is +/// `struct` marked with the +/// `paimon.extension.type = paimon.type.variant` field metadata (see `VariantDefs`). +class PAIMON_EXPORT VariantTypeUtils { + public: + VariantTypeUtils() = delete; + ~VariantTypeUtils() = delete; + + /// Whether the field is a Paimon Variant field (a struct with the variant metadata marker). + static bool IsVariantField(const std::shared_ptr& field); + + /// Whether `type` is the unshredded variant physical type + /// `struct` (field metadata ignored). + static bool IsUnshreddedVariantType(const std::shared_ptr& type); + + /// Whether the metadata carries the Paimon Variant extension type marker. + static bool IsVariantMetadata(const std::shared_ptr& metadata); + + /// The unshredded physical Arrow type of a variant field: + /// `struct` with paimon field ids 0/1 on + /// the children. + static std::shared_ptr UnshreddedStructType(); + + /// Creates a Variant Arrow field with the variant metadata marker. + static std::shared_ptr ToArrowField( + const std::string& field_name, bool nullable = true, + std::unordered_map metadata = {}); + + /// Validates that a variant-marked field has the expected physical shape: + /// `struct`. + static Status ValidateVariantShape(const std::shared_ptr& field); + + /// Whether the schema contains a variant field, at the top level or nested inside structs, + /// arrays or maps. + static bool ContainsVariantField(const std::shared_ptr& schema); + + /// Whether the field contains a variant field, itself included, at any nesting level. + static bool ContainsVariantField(const std::shared_ptr& field); +}; + +} // namespace paimon diff --git a/src/paimon/common/data/variant/variant_type_utils_test.cpp b/src/paimon/common/data/variant/variant_type_utils_test.cpp new file mode 100644 index 00000000..ba6fc4ac --- /dev/null +++ b/src/paimon/common/data/variant/variant_type_utils_test.cpp @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/data/variant/variant_type_utils.h" + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/utils/field_type_utils.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(VariantTypeUtilsTest, ToArrowFieldAndDetection) { + auto field = VariantTypeUtils::ToArrowField("v"); + ASSERT_EQ(field->name(), "v"); + ASSERT_TRUE(field->nullable()); + ASSERT_TRUE(VariantTypeUtils::IsVariantField(field)); + ASSERT_TRUE(VariantTypeUtils::IsVariantMetadata(field->metadata())); + ASSERT_OK(VariantTypeUtils::ValidateVariantShape(field)); + + auto struct_type = std::static_pointer_cast(field->type()); + ASSERT_EQ(struct_type->num_fields(), 2); + ASSERT_EQ(struct_type->field(0)->name(), VariantDefs::kValueFieldName); + ASSERT_EQ(struct_type->field(0)->type()->id(), arrow::Type::BINARY); + ASSERT_FALSE(struct_type->field(0)->nullable()); + ASSERT_EQ(struct_type->field(1)->name(), VariantDefs::kMetadataFieldName); + ASSERT_EQ(struct_type->field(1)->type()->id(), arrow::Type::BINARY); + ASSERT_FALSE(struct_type->field(1)->nullable()); + // The children carry paimon field ids 0/1 (mapped to parquet field ids on write). + ASSERT_EQ(struct_type->field(0)->metadata()->Get("paimon.id").ValueOr(""), "0"); + ASSERT_EQ(struct_type->field(1)->metadata()->Get("paimon.id").ValueOr(""), "1"); + + auto plain_struct = arrow::field("s", VariantTypeUtils::UnshreddedStructType()); + ASSERT_FALSE(VariantTypeUtils::IsVariantField(plain_struct)); + std::unordered_map metadata = { + {VariantDefs::kExtensionTypeKey, VariantDefs::kExtensionTypeValue}}; + auto marked_binary = arrow::field("b", arrow::binary(), true, + std::make_shared(metadata)); + ASSERT_FALSE(VariantTypeUtils::IsVariantField(marked_binary)); +} + +TEST(VariantTypeUtilsTest, ValidateVariantShapeRejectsWrongShape) { + std::unordered_map metadata = { + {VariantDefs::kExtensionTypeKey, VariantDefs::kExtensionTypeValue}}; + auto arrow_metadata = std::make_shared(metadata); + auto one_child = arrow::field( + "v", arrow::struct_({arrow::field("value", arrow::binary(), false)}), true, arrow_metadata); + ASSERT_NOK(VariantTypeUtils::ValidateVariantShape(one_child)); + auto nullable_children = + arrow::field("v", + arrow::struct_({arrow::field("value", arrow::binary(), true), + arrow::field("metadata", arrow::binary(), true)}), + true, arrow_metadata); + ASSERT_NOK(VariantTypeUtils::ValidateVariantShape(nullable_children)); + auto wrong_type = + arrow::field("v", + arrow::struct_({arrow::field("value", arrow::utf8(), false), + arrow::field("metadata", arrow::binary(), false)}), + true, arrow_metadata); + ASSERT_NOK(VariantTypeUtils::ValidateVariantShape(wrong_type)); +} + +TEST(VariantTypeUtilsTest, ContainsVariantField) { + auto variant_field = VariantTypeUtils::ToArrowField("v"); + auto plain = arrow::schema({arrow::field("a", arrow::int32())}); + ASSERT_FALSE(VariantTypeUtils::ContainsVariantField(plain)); + auto top_level = arrow::schema({arrow::field("a", arrow::int32()), variant_field}); + ASSERT_TRUE(VariantTypeUtils::ContainsVariantField(top_level)); + auto nested = + arrow::schema({arrow::field("row", arrow::struct_({arrow::field("inner", arrow::int32()), + VariantTypeUtils::ToArrowField("v")}))}); + ASSERT_TRUE(VariantTypeUtils::ContainsVariantField(nested)); + auto in_list = + arrow::schema({arrow::field("l", arrow::list(VariantTypeUtils::ToArrowField("item")))}); + ASSERT_TRUE(VariantTypeUtils::ContainsVariantField(in_list)); +} + +TEST(VariantTypeUtilsTest, FieldTypeConversion) { + auto variant_field = VariantTypeUtils::ToArrowField("v"); + ASSERT_OK_AND_ASSIGN(FieldType field_type, FieldTypeUtils::ConvertToFieldType(variant_field)); + ASSERT_EQ(field_type, FieldType::VARIANT); + ASSERT_EQ(FieldTypeUtils::FieldTypeToString(FieldType::VARIANT), "VARIANT"); + auto plain_struct = arrow::field("s", VariantTypeUtils::UnshreddedStructType()); + ASSERT_OK_AND_ASSIGN(FieldType plain_type, FieldTypeUtils::ConvertToFieldType(plain_struct)); + ASSERT_EQ(plain_type, FieldType::STRUCT); +} + +} // namespace paimon::test diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 24a78acc..605b0c0b 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -104,6 +104,15 @@ const char Options::MAP_STORAGE_LAYOUT[] = "map.storage-layout"; const char Options::MAP_SHARED_SHREDDING_MAX_COLUMNS[] = "map.shared-shredding.max-columns"; const char Options::MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY[] = "map.shared-shredding.column-placement-policy"; +const char Options::VARIANT_SHREDDING_SCHEMA[] = "variant.shreddingSchema"; +const char Options::PARQUET_VARIANT_SHREDDING_SCHEMA[] = "parquet.variant.shreddingSchema"; +const char Options::VARIANT_INFER_SHREDDING_SCHEMA[] = "variant.inferShreddingSchema"; +const char Options::VARIANT_SHREDDING_MAX_SCHEMA_WIDTH[] = "variant.shredding.maxSchemaWidth"; +const char Options::VARIANT_SHREDDING_MAX_SCHEMA_DEPTH[] = "variant.shredding.maxSchemaDepth"; +const char Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO[] = + "variant.shredding.minFieldCardinalityRatio"; +const char Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW[] = + "variant.shredding.maxInferBufferRow"; const char Options::BLOB_AS_DESCRIPTOR[] = "blob-as-descriptor"; const char Options::BLOB_FIELD[] = "blob-field"; const char Options::BLOB_DESCRIPTOR_FIELD[] = "blob-descriptor-field"; diff --git a/src/paimon/common/types/array_type.h b/src/paimon/common/types/array_type.h index d03c0da9..cecd5bc2 100644 --- a/src/paimon/common/types/array_type.h +++ b/src/paimon/common/types/array_type.h @@ -46,8 +46,10 @@ class ArrayType : public DataType { auto type = arrow::internal::checked_cast(type_.get()); auto value_field = type->value_field(); + // The element metadata is load-bearing: it is what marks an extension type such as + // VARIANT. std::shared_ptr data_type = - DataType::Create(value_field->type(), value_field->nullable(), /*metadata=*/nullptr); + DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata()); obj.AddMember(rapidjson::StringRef("element"), RapidJsonUtil::SerializeValue(*data_type, allocator).Move(), *allocator); return obj; diff --git a/src/paimon/common/types/data_type.cpp b/src/paimon/common/types/data_type.cpp index 7b71f53d..623d4ca2 100644 --- a/src/paimon/common/types/data_type.cpp +++ b/src/paimon/common/types/data_type.cpp @@ -26,6 +26,7 @@ #include "arrow/util/checked_cast.h" #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/array_type.h" #include "paimon/common/types/map_type.h" #include "paimon/common/types/row_type.h" @@ -52,6 +53,11 @@ std::unique_ptr DataType::Create( case arrow::Type::type::LIST: return std::make_unique(type, nullable, metadata); case arrow::Type::type::STRUCT: + if (VariantTypeUtils::IsVariantMetadata(metadata)) { + // A variant field is physically a struct but is a scalar + // VARIANT type in the paimon type system, not a ROW type. + return std::unique_ptr(new DataType(type, nullable, metadata)); + } return std::make_unique(type, nullable, metadata); default: return std::unique_ptr(new DataType(type, nullable, metadata)); @@ -119,16 +125,23 @@ std::string DataType::DataTypeToString(const std::shared_ptr& t arrow::internal::checked_pointer_cast(type); return TimestampToString(timestamp_type); } + case arrow::Type::type::STRUCT: { + if (VariantTypeUtils::IsVariantMetadata(metadata_)) { + return "VARIANT"; + } + break; + } case arrow::Type::type::LARGE_BINARY: { // TODO(xinyu): change binary to large binary? if (BlobUtils::IsBlobMetadata(metadata_)) { return "BLOB"; } - [[fallthrough]]; + break; } default: - throw std::invalid_argument(fmt::format("unknown type {}", type->ToString())); + break; } + throw std::invalid_argument(fmt::format("unknown type {}", type->ToString())); } } // namespace paimon diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index 8efe7b70..950308a5 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -31,6 +31,7 @@ #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/rapidjson_util.h" @@ -83,9 +84,16 @@ struct Token { std::string value; }; +// Extension type attributes of a parsed atomic type. BLOB and VARIANT parse to plain arrow +// types (large_binary / struct) and need field-level metadata markers applied by the caller. +struct AtomicTypeAttributes { + bool is_blob = false; + bool is_variant = false; +}; + // nullptr is returned in the case of parsing failed Result> ParseAtomicType(const std::string& str, bool* nullable, - bool* is_blob); + AtomicTypeAttributes* attributes); std::vector Tokenize(const std::string& chars); bool IsWhitespace(char character); bool IsDelimiter(char character); @@ -139,6 +147,7 @@ enum class Keyword : int32_t { MAP, ROW, BLOB, + VARIANT, // NULL is keyword in c++ NULL_, RAW, @@ -187,6 +196,7 @@ const std::map& Keywords() { {"MAP", Keyword::MAP}, {"ROW", Keyword::ROW}, {"BLOB", Keyword::BLOB}, + {"VARIANT", Keyword::VARIANT}, {"NULL", Keyword::NULL_}, {"RAW", Keyword::RAW}, {"LEGACY", Keyword::LEGACY}, @@ -199,7 +209,8 @@ class TokenParser { TokenParser(const std::string& input_string, const std::vector& tokens) : input_string_(input_string), tokens_(tokens) {} - Result> ParseTokens(bool* nullable, bool* is_blob); + Result> ParseTokens(bool* nullable, + AtomicTypeAttributes* attributes); private: inline const Token& GetToken() const { @@ -228,9 +239,9 @@ class TokenParser { bool HasNextToken(const std::vector& types) const; bool HasNextToken(const std::vector& keywords) const; Result ParseNullability(); - Result> ParseTypeWithNullability(bool* nullable, - bool* is_blob); - Result> ParseTypeByKeyword(bool* is_blob); + Result> ParseTypeWithNullability( + bool* nullable, AtomicTypeAttributes* attributes); + Result> ParseTypeByKeyword(AtomicTypeAttributes* attributes); Result ParseStringLength(); template Result> ParseStringType(); @@ -248,11 +259,11 @@ class TokenParser { }; Result> ParseAtomicType(const std::string& str, bool* nullable, - bool* is_blob) { + AtomicTypeAttributes* attributes) { try { std::vector tokens = Tokenize(str); TokenParser converter(str, tokens); - return converter.ParseTokens(nullable, is_blob); + return converter.ParseTokens(nullable, attributes); } catch (...) { return Status::Invalid("parse atomic type failed."); } @@ -374,9 +385,10 @@ int32_t ConsumeIdentifier(const std::string& chars, int32_t cursor, std::ostring return cursor - 1; } -Result> TokenParser::ParseTokens(bool* nullable, bool* is_blob) { +Result> TokenParser::ParseTokens( + bool* nullable, AtomicTypeAttributes* attributes) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr type, - ParseTypeWithNullability(nullable, is_blob)); + ParseTypeWithNullability(nullable, attributes)); if (HasRemainingTokens()) { PAIMON_RETURN_NOT_OK(NextToken()); return Status::Invalid(fmt::format("Unexpected token: {}", GetToken().value)); @@ -455,9 +467,10 @@ Result TokenParser::ParseNullability() { return true; } -Result> TokenParser::ParseTypeWithNullability(bool* nullable, - bool* is_blob) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_type, ParseTypeByKeyword(is_blob)); +Result> TokenParser::ParseTypeWithNullability( + bool* nullable, AtomicTypeAttributes* attributes) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_type, + ParseTypeByKeyword(attributes)); PAIMON_ASSIGN_OR_RAISE(*nullable, ParseNullability()); // special case: suffix notation for ARRAY types if (HasNextToken({Keyword::ARRAY}) || HasNextToken({Keyword::MULTISET})) { @@ -466,7 +479,8 @@ Result> TokenParser::ParseTypeWithNullability(b return data_type; } -Result> TokenParser::ParseTypeByKeyword(bool* is_blob) { +Result> TokenParser::ParseTypeByKeyword( + AtomicTypeAttributes* attributes) { PAIMON_RETURN_NOT_OK(NextToken(TokenType::KEYWORD)); switch (TokenAsKeyword()) { case Keyword::CHAR: @@ -478,9 +492,13 @@ Result> TokenParser::ParseTypeByKeyword(bool* i case Keyword::BYTES: return arrow::binary(); case Keyword::BLOB: { - *is_blob = true; + attributes->is_blob = true; return arrow::large_binary(); } + case Keyword::VARIANT: { + attributes->is_variant = true; + return VariantTypeUtils::UnshreddedStructType(); + } case Keyword::STRING: return arrow::utf8(); case Keyword::BOOLEAN: @@ -615,11 +633,13 @@ Result> DataTypeJsonParser::ParseType( Result> DataTypeJsonParser::ParseAtomicTypeField( const std::string& name, const rapidjson::Value& type_json_value) { bool nullable = true; - bool is_blob = false; + AtomicTypeAttributes attributes; PAIMON_ASSIGN_OR_RAISE(std::shared_ptr type, - ParseAtomicType(type_json_value.GetString(), &nullable, &is_blob)); - if (is_blob) { + ParseAtomicType(type_json_value.GetString(), &nullable, &attributes)); + if (attributes.is_blob) { return BlobUtils::ToArrowField(name, nullable); + } else if (attributes.is_variant) { + return VariantTypeUtils::ToArrowField(name, nullable); } else { return arrow::field(name, type, nullable); } diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index fc0cdbfa..bbb4fb52 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -23,6 +23,7 @@ #include #include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" @@ -138,6 +139,17 @@ TEST(DataTypeJsonParserTest, ParseTypeAtomicTypeSuccess) { ASSERT_TRUE(field->type()->Equals(test_case.second)); } + // VARIANT parses to a variant-marked struct field. + for (const char* variant_str : {"VARIANT", "VARIANT NOT NULL"}) { + rapidjson::Document doc; + rapidjson::Value value(variant_str, doc.GetAllocator()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr field, + DataTypeJsonParser::ParseType("variant_field", value)); + ASSERT_TRUE(VariantTypeUtils::IsVariantField(field)); + ASSERT_EQ(field->nullable(), std::string(variant_str) == "VARIANT"); + ASSERT_TRUE(field->type()->Equals(VariantTypeUtils::UnshreddedStructType())); + } + // Invalid case { rapidjson::Document invalid_doc; diff --git a/src/paimon/common/types/data_type_test.cpp b/src/paimon/common/types/data_type_test.cpp index 137cf854..9568c3ff 100644 --- a/src/paimon/common/types/data_type_test.cpp +++ b/src/paimon/common/types/data_type_test.cpp @@ -24,7 +24,10 @@ #include "arrow/api.h" #include "gtest/gtest.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" +#include "paimon/common/utils/rapidjson_util.h" namespace paimon::test { @@ -65,6 +68,19 @@ TEST(DataTypeTest, DataTypeToString) { DataType blob_type(blob_field->type(), blob_field->nullable(), blob_field->metadata()); ASSERT_EQ(blob_type.DataTypeToString(blob_field->type()), "BLOB"); } + { + std::shared_ptr variant_field = VariantTypeUtils::ToArrowField("f3_variant"); + DataType variant_type(variant_field->type(), variant_field->nullable(), + variant_field->metadata()); + ASSERT_EQ(variant_type.DataTypeToString(variant_field->type()), "VARIANT"); + // A variant field is a scalar VARIANT type, not a ROW type. + auto created = DataType::Create(variant_field->type(), variant_field->nullable(), + variant_field->metadata()); + rapidjson::Document doc; + auto json_value = created->ToJson(&doc.GetAllocator()); + ASSERT_TRUE(json_value.IsString()); + ASSERT_EQ(std::string(json_value.GetString()), "VARIANT"); + } ASSERT_EQ(dummy_data_type.DataTypeToString(arrow::date32()), "DATE"); auto decimal_type1 = arrow::decimal128(10, 2); @@ -98,4 +114,38 @@ TEST(DataTypeTest, DataTypeToString) { ASSERT_THROW(dummy_data_type.DataTypeToString(unknown_type), std::invalid_argument); } +TEST(DataTypeTest, NestedTypeSerializationUsesChildMetadata) { + // ARRAY and MAP must carry their children's metadata into the serialized type, because that + // is what marks an extension type. Dropping it serialized a VARIANT as its physical + // `struct` ROW, whose fixed child ids 0/1 then broke reading the schema back. + auto to_json = [](const std::shared_ptr& field) { + auto data_type = DataType::Create(field->type(), field->nullable(), field->metadata()); + rapidjson::Document doc; + auto value = data_type->ToJson(&doc.GetAllocator()); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return std::string(buffer.GetString()); + }; + + auto variant_field = VariantTypeUtils::ToArrowField("element"); + auto array_field = arrow::field("arr", arrow::list(variant_field)); + ASSERT_EQ(to_json(array_field), R"({"type":"ARRAY","element":"VARIANT"})"); + + auto map_field = + arrow::field("m", arrow::map(arrow::utf8(), VariantTypeUtils::ToArrowField("value"))); + ASSERT_EQ(to_json(map_field), R"({"type":"MAP","key":"STRING NOT NULL","value":"VARIANT"})"); + + // BLOB is marked the same way and was degraded to its physical BYTES the same way. + auto blob_array = arrow::field("b", arrow::list(BlobUtils::ToArrowField("element", true))); + ASSERT_EQ(to_json(blob_array), R"({"type":"ARRAY","element":"BLOB"})"); + + // Child metadata only ever selects an extension type, so ordinary elements are unaffected by + // it being carried over. + auto plain_child = arrow::field("element", arrow::int32(), /*nullable=*/true, + arrow::KeyValueMetadata::Make({DataField::FIELD_ID}, {"7"})); + ASSERT_EQ(to_json(arrow::field("a", arrow::list(plain_child))), + R"({"type":"ARRAY","element":"INT"})"); +} + } // namespace paimon::test diff --git a/src/paimon/common/types/map_type.h b/src/paimon/common/types/map_type.h index 7749dfca..29fd8bbd 100644 --- a/src/paimon/common/types/map_type.h +++ b/src/paimon/common/types/map_type.h @@ -52,13 +52,15 @@ class MapType : public DataType { *allocator); auto type = arrow::internal::checked_cast(type_.get()); auto key_field = type->key_field(); + // The key and value metadata is load-bearing: it is what marks an extension type such as + // VARIANT. std::shared_ptr key_data_type = - DataType::Create(key_field->type(), key_field->nullable(), /*metadata=*/nullptr); + DataType::Create(key_field->type(), key_field->nullable(), key_field->metadata()); obj.AddMember(rapidjson::StringRef("key"), RapidJsonUtil::SerializeValue(*key_data_type, allocator).Move(), *allocator); auto value_field = type->item_field(); std::shared_ptr value_data_type = - DataType::Create(value_field->type(), value_field->nullable(), /*metadata=*/nullptr); + DataType::Create(value_field->type(), value_field->nullable(), value_field->metadata()); obj.AddMember(rapidjson::StringRef("value"), RapidJsonUtil::SerializeValue(*value_data_type, allocator).Move(), *allocator); diff --git a/src/paimon/common/utils/field_type_utils.h b/src/paimon/common/utils/field_type_utils.h index 77da7db8..72246769 100644 --- a/src/paimon/common/utils/field_type_utils.h +++ b/src/paimon/common/utils/field_type_utils.h @@ -26,6 +26,7 @@ #include "arrow/api.h" #include "arrow/type_fwd.h" #include "fmt/format.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/defs.h" #include "paimon/result.h" #include "paimon/status.h" @@ -49,6 +50,15 @@ class FieldTypeUtils { (type == FieldType::BIGINT); } + /// Converts an arrow field to a `FieldType`, disambiguating metadata-marked extension types + /// (a VARIANT field is physically a STRUCT with the variant metadata marker). + static Result ConvertToFieldType(const std::shared_ptr& field) { + if (VariantTypeUtils::IsVariantField(field)) { + return FieldType::VARIANT; + } + return ConvertToFieldType(field->type()->id()); + } + static Result ConvertToFieldType(const arrow::Type::type& arrow_type) { switch (arrow_type) { case arrow::Type::type::BOOL: @@ -123,6 +133,8 @@ class FieldTypeUtils { return "MAP"; case FieldType::STRUCT: return "STRUCT"; + case FieldType::VARIANT: + return "VARIANT"; default: return "UNKNOWN, type id:" + std::to_string(static_cast(type)); } diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index a4de60e8..833f540f 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -28,6 +28,7 @@ #include "arrow/c/helpers.h" #include "arrow/type.h" #include "arrow/util/key_value_metadata.h" +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/types/row_kind.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -196,10 +197,11 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWrit AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetDataFileWriterFactory( const std::shared_ptr& schema, const std::optional>& write_cols) const { - if (shredding_context_) { + if (auto plan_factory = ShreddingWritePlanFactories::SelectActive( + options_, schema, shredding_context_, memory_pool_)) { return std::make_shared( options_, schema_id_, schema, write_cols, seq_num_counter_, FileSource::Append(), - path_factory_, shredding_context_, memory_pool_); + path_factory_, plan_factory, memory_pool_); } return std::make_shared(options_, schema_id_, schema, write_cols, seq_num_counter_, FileSource::Append(), diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 9e261fc3..f2b93d92 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -449,6 +449,11 @@ struct CoreOptions::Impl { bool row_tracking_enabled = false; bool row_tracking_partition_group_on_commit = true; bool data_evolution_enabled = false; + bool variant_infer_shredding_schema = false; + int32_t variant_shredding_max_schema_width = 300; + int32_t variant_shredding_max_schema_depth = 50; + double variant_shredding_min_field_cardinality_ratio = 0.1; + int32_t variant_shredding_max_infer_buffer_row = 4096; bool blob_view_resolve_enabled = true; bool blob_as_descriptor = false; std::optional blob_split_by_file_size; @@ -848,6 +853,51 @@ struct CoreOptions::Impl { } // Parse lookup configurations: compact mode, bloom filter, remote file, cache, compression. + Status ParseVariantOptions(const ConfigParser& parser) { + // Parse variant.inferShreddingSchema - infer the shredding schema from sampled rows + PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_INFER_SHREDDING_SCHEMA, + &variant_infer_shredding_schema)); + // Parse variant.shredding.maxSchemaWidth - max number of shredded fields, default 300 + PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_SHREDDING_MAX_SCHEMA_WIDTH, + &variant_shredding_max_schema_width)); + // Parse variant.shredding.maxSchemaDepth - max shredded nesting depth, default 50 + PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_SHREDDING_MAX_SCHEMA_DEPTH, + &variant_shredding_max_schema_depth)); + // Parse variant.shredding.minFieldCardinalityRatio - min occurrence ratio for a field to + // be shredded, default 0.1 + PAIMON_RETURN_NOT_OK( + parser.Parse(Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO, + &variant_shredding_min_field_cardinality_ratio)); + // Parse variant.shredding.maxInferBufferRow - rows buffered per file for inference, + // default 4096 + PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW, + &variant_shredding_max_infer_buffer_row)); + if (variant_shredding_max_schema_width <= 0) { + return Status::Invalid(fmt::format( + "The option '{}' should be positive, while input is {}", + Options::VARIANT_SHREDDING_MAX_SCHEMA_WIDTH, variant_shredding_max_schema_width)); + } + if (variant_shredding_max_schema_depth <= 0) { + return Status::Invalid(fmt::format( + "The option '{}' should be positive, while input is {}", + Options::VARIANT_SHREDDING_MAX_SCHEMA_DEPTH, variant_shredding_max_schema_depth)); + } + if (variant_shredding_min_field_cardinality_ratio < 0.0 || + variant_shredding_min_field_cardinality_ratio > 1.0) { + return Status::Invalid( + fmt::format("The option '{}' should be in the range [0, 1], while input is {}", + Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO, + variant_shredding_min_field_cardinality_ratio)); + } + if (variant_shredding_max_infer_buffer_row <= 0) { + return Status::Invalid( + fmt::format("The option '{}' should be positive, while input is {}", + Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW, + variant_shredding_max_infer_buffer_row)); + } + return Status::OK(); + } + Status ParseLookupOptions(const ConfigParser& parser) { // Parse force-lookup - whether to force lookup for compaction, default false PAIMON_RETURN_NOT_OK(parser.Parse(Options::FORCE_LOOKUP, &force_lookup)); @@ -922,6 +972,7 @@ Result CoreOptions::FromMap( PAIMON_RETURN_NOT_OK(impl->ParseIndexOptions(parser)); PAIMON_RETURN_NOT_OK(impl->ParseCompactionOptions(parser)); PAIMON_RETURN_NOT_OK(impl->ParseLookupOptions(parser)); + PAIMON_RETURN_NOT_OK(impl->ParseVariantOptions(parser)); return options; } @@ -1276,6 +1327,37 @@ Result CoreOptions::FieldCollectAggDistinct(const std::string& field_name) return distinct; } +std::optional CoreOptions::GetVariantShreddingSchema() const { + auto it = impl_->raw_options.find(Options::VARIANT_SHREDDING_SCHEMA); + if (it == impl_->raw_options.end()) { + it = impl_->raw_options.find(Options::PARQUET_VARIANT_SHREDDING_SCHEMA); + } + if (it == impl_->raw_options.end() || it->second.empty()) { + return std::nullopt; + } + return it->second; +} + +bool CoreOptions::VariantInferShreddingSchemaEnabled() const { + return impl_->variant_infer_shredding_schema; +} + +int32_t CoreOptions::GetVariantShreddingMaxSchemaWidth() const { + return impl_->variant_shredding_max_schema_width; +} + +int32_t CoreOptions::GetVariantShreddingMaxSchemaDepth() const { + return impl_->variant_shredding_max_schema_depth; +} + +double CoreOptions::GetVariantShreddingMinFieldCardinalityRatio() const { + return impl_->variant_shredding_min_field_cardinality_ratio; +} + +int32_t CoreOptions::GetVariantShreddingMaxInferBufferRow() const { + return impl_->variant_shredding_max_infer_buffer_row; +} + Result CoreOptions::GetMapStorageLayout(const std::string& field_name) const { std::string key = std::string(Options::FIELDS_PREFIX) + "." + field_name + "." + std::string(Options::MAP_STORAGE_LAYOUT); diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index db6e7604..050685ce 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -147,6 +147,15 @@ class PAIMON_EXPORT CoreOptions { Result GetMapSharedShreddingColumnPlacementPolicy( const std::string& field_name) const; + /// The configured variant shredding schema JSON, if any (falls back to + /// "parquet.variant.shreddingSchema"). + std::optional GetVariantShreddingSchema() const; + bool VariantInferShreddingSchemaEnabled() const; + int32_t GetVariantShreddingMaxSchemaWidth() const; + int32_t GetVariantShreddingMaxSchemaDepth() const; + double GetVariantShreddingMinFieldCardinalityRatio() const; + int32_t GetVariantShreddingMaxInferBufferRow() const; + bool DeletionVectorsEnabled() const; bool DeletionVectorsBitmap64() const; int64_t DeletionVectorTargetFileSize() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 8101e359..e22989c4 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -1068,4 +1068,54 @@ TEST(CoreOptionsTest, TestMapStorageLayout) { } } +TEST(CoreOptionsTest, TestVariantOptions) { + { + // Defaults. + ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({})); + ASSERT_EQ(options.GetVariantShreddingSchema(), std::nullopt); + ASSERT_FALSE(options.VariantInferShreddingSchemaEnabled()); + ASSERT_EQ(options.GetVariantShreddingMaxSchemaWidth(), 300); + ASSERT_EQ(options.GetVariantShreddingMaxSchemaDepth(), 50); + ASSERT_DOUBLE_EQ(options.GetVariantShreddingMinFieldCardinalityRatio(), 0.1); + ASSERT_EQ(options.GetVariantShreddingMaxInferBufferRow(), 4096); + } + { + // Configured values. + ASSERT_OK_AND_ASSIGN( + CoreOptions options, + CoreOptions::FromMap({{"variant.shreddingSchema", "{\"type\": \"ROW\"}"}, + {"variant.inferShreddingSchema", "true"}, + {"variant.shredding.maxSchemaWidth", "20"}, + {"variant.shredding.maxSchemaDepth", "5"}, + {"variant.shredding.minFieldCardinalityRatio", "0.25"}, + {"variant.shredding.maxInferBufferRow", "128"}})); + ASSERT_EQ(options.GetVariantShreddingSchema(), "{\"type\": \"ROW\"}"); + ASSERT_TRUE(options.VariantInferShreddingSchemaEnabled()); + ASSERT_EQ(options.GetVariantShreddingMaxSchemaWidth(), 20); + ASSERT_EQ(options.GetVariantShreddingMaxSchemaDepth(), 5); + ASSERT_DOUBLE_EQ(options.GetVariantShreddingMinFieldCardinalityRatio(), 0.25); + ASSERT_EQ(options.GetVariantShreddingMaxInferBufferRow(), 128); + } + { + // The legacy parquet-prefixed key is a fallback for the shredding schema. + ASSERT_OK_AND_ASSIGN(CoreOptions options, + CoreOptions::FromMap({{"parquet.variant.shreddingSchema", "{}"}})); + ASSERT_EQ(options.GetVariantShreddingSchema(), "{}"); + } + // Invalid values fail when the options are parsed, not when they are used. + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.inferShreddingSchema", "not_a_bool"}}), + "variant.inferShreddingSchema"); + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxSchemaWidth", "abc"}}), + "variant.shredding.maxSchemaWidth"); + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxSchemaWidth", "0"}}), + "should be positive"); + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxSchemaDepth", "-1"}}), + "should be positive"); + ASSERT_NOK_WITH_MSG( + CoreOptions::FromMap({{"variant.shredding.minFieldCardinalityRatio", "1.5"}}), + "should be in the range [0, 1]"); + ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxInferBufferRow", "0"}}), + "should be positive"); +} + } // namespace paimon::test diff --git a/src/paimon/core/io/infer_shredding_file_writer.h b/src/paimon/core/io/infer_shredding_file_writer.h new file mode 100644 index 00000000..4ce7925a --- /dev/null +++ b/src/paimon/core/io/infer_shredding_file_writer.h @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" +#include "paimon/core/io/single_file_writer.h" +#include "paimon/core/key_value.h" + +namespace paimon { + +/// A file writer that infers the shredding write plan from the first rows of the file. Incoming +/// batches are buffered until `ShreddingWritePlanFactory::InferBufferRowCount` rows have been +/// collected (or the writer is closed); the buffered batches are then sampled to create the +/// batch converter, the actual file writer is created with the resulting physical schema, and +/// the buffered batches are replayed into it. The file never rolls while buffering. +template +class InferShreddingFileWriter : public SingleFileWriter { + public: + using CreateInnerFn = std::function>>( + const std::shared_ptr&)>; + + InferShreddingFileWriter(const std::shared_ptr& logical_schema, + const std::shared_ptr& plan_factory, + const std::string& file_format_identifier, CreateInnerFn create_inner) + : SingleFileWriter(/*compression=*/"", std::function()), + logical_type_(arrow::struct_(logical_schema->fields())), + plan_factory_(plan_factory), + file_format_identifier_(file_format_identifier), + create_inner_(std::move(create_inner)) {} + + Status Write(T record) override { + if (plan_finalized_) { + return inner_->Write(std::move(record)); + } + PAIMON_RETURN_NOT_OK(Buffer(std::move(record))); + if (buffered_rows_ >= plan_factory_->InferBufferRowCount()) { + return FinalizePlanAndFlush(); + } + return Status::OK(); + } + + Status Close() override { + if (!plan_finalized_) { + PAIMON_RETURN_NOT_OK(FinalizePlanAndFlush()); + } + return inner_->Close(); + } + + Result GetResult() override { + if (!inner_) { + return Status::Invalid("Cannot access the result unless the writer is closed."); + } + return inner_->GetResult(); + } + + Result ReachTargetSize(bool suggested_check, int64_t target_size) override { + if (!plan_finalized_) { + // Never roll the file while rows are being buffered for inference. + return false; + } + return inner_->ReachTargetSize(suggested_check, target_size); + } + + Result::AbortExecutor> GetAbortExecutor() const override { + if (!inner_) { + return Status::Invalid("Writer should be closed!"); + } + return inner_->GetAbortExecutor(); + } + + std::string GetPath() const override { + return inner_ ? inner_->GetPath() : ""; + } + + void Abort() override { + if (inner_) { + inner_->Abort(); + } + } + + int64_t RecordCount() const override { + return plan_finalized_ ? inner_->RecordCount() : buffered_rows_; + } + + std::shared_ptr GetMetrics() const override { + return inner_ ? inner_->GetMetrics() : nullptr; + } + + private: + struct BufferedBatch { + std::shared_ptr batch; + // Holds the non-batch part of a KeyValueBatch record; unused for plain batches. + KeyValueBatch record_template; + }; + + Status Buffer(T record) { + BufferedBatch buffered; + ::ArrowArray* c_batch; + if constexpr (std::is_same_v) { + c_batch = record; + } else { + static_assert(std::is_same_v, + "InferShreddingFileWriter supports ::ArrowArray* and KeyValueBatch"); + buffered.record_template = std::move(record); + c_batch = buffered.record_template.batch.get(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(buffered.batch, + arrow::ImportArray(c_batch, logical_type_)); + buffered_rows_ += buffered.batch->length(); + buffered_batches_.push_back(std::move(buffered)); + return Status::OK(); + } + + Status FinalizePlanAndFlush() { + std::vector> samples; + samples.reserve(buffered_batches_.size()); + for (const auto& buffered : buffered_batches_) { + samples.push_back(buffered.batch); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, + plan_factory_->CreateConverter(file_format_identifier_, samples)); + PAIMON_ASSIGN_OR_RAISE(inner_, create_inner_(converter)); + plan_finalized_ = true; + for (auto& buffered : buffered_batches_) { + if constexpr (std::is_same_v) { + ::ArrowArray c_batch; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*buffered.batch, &c_batch)); + PAIMON_RETURN_NOT_OK(inner_->Write(&c_batch)); + } else { + KeyValueBatch record = std::move(buffered.record_template); + record.batch = std::make_unique<::ArrowArray>(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*buffered.batch, record.batch.get())); + PAIMON_RETURN_NOT_OK(inner_->Write(std::move(record))); + } + } + buffered_batches_.clear(); + return Status::OK(); + } + + std::shared_ptr logical_type_; + std::shared_ptr plan_factory_; + std::string file_format_identifier_; + CreateInnerFn create_inner_; + + std::vector buffered_batches_; + int64_t buffered_rows_ = 0; + bool plan_finalized_ = false; + std::unique_ptr> inner_; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/infer_shredding_file_writer_test.cpp b/src/paimon/core/io/infer_shredding_file_writer_test.cpp new file mode 100644 index 00000000..fe98eb97 --- /dev/null +++ b/src/paimon/core/io/infer_shredding_file_writer_test.cpp @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/io/infer_shredding_file_writer.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/core_options.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/variant_test_data.h" + +namespace paimon::test { + +namespace { + +/// A file-less writer standing in for the actual data file writer: it applies the shredding +/// conversion like the injected converter lambda would and collects the written batches. +class CollectingFileWriter : public SingleFileWriter<::ArrowArray*, std::shared_ptr> { + public: + CollectingFileWriter(const std::shared_ptr& converter, + const std::shared_ptr& logical_type, + std::vector>* sink) + : SingleFileWriter("", std::function()), + converter_(converter), + logical_type_(logical_type), + sink_(sink) {} + + Status Write(::ArrowArray* record) override { + record_count_ += record->length; + std::shared_ptr array; + if (converter_) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowArray> physical, + converter_->Convert(record)); + auto physical_type = arrow::struct_(converter_->GetPhysicalSchema()->fields()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(array, + arrow::ImportArray(physical.get(), physical_type)); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(array, arrow::ImportArray(record, logical_type_)); + } + sink_->push_back(std::move(array)); + return Status::OK(); + } + + Status Close() override { + closed = true; + return Status::OK(); + } + + Result> GetResult() override { + return std::shared_ptr(nullptr); + } + + Result ReachTargetSize(bool suggested_check, int64_t target_size) override { + return false; + } + + Result GetAbortExecutor() const override { + return AbortExecutor(nullptr, ""); + } + + void Abort() override {} + + int64_t RecordCount() const override { + return record_count_; + } + + bool closed = false; + + private: + std::shared_ptr converter_; + std::shared_ptr logical_type_; + std::vector>* sink_; + int64_t record_count_ = 0; +}; + +} // namespace + +class InferShreddingFileWriterTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("v"))}; + schema_ = DataField::ConvertDataFieldsToArrowSchema(fields); + logical_type_ = arrow::struct_(schema_->fields()); + } + + std::shared_ptr BuildBatch(const std::vector& jsons) { + EXPECT_OK_AND_ASSIGN( + std::shared_ptr batch, + VariantTestData::BuildVariantBatch(schema_->field(0), schema_->field(1), jsons, pool_)); + return batch; + } + + Status WriteBatch(InferShreddingFileWriter<::ArrowArray*, std::shared_ptr>* w, + const std::vector& jsons) { + std::shared_ptr array = BuildBatch(jsons); + ::ArrowArray c_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &c_array)); + return w->Write(&c_array); + } + + std::unique_ptr>> + MakeWriter(int32_t buffer_rows) { + std::map option_map = { + {"variant.inferShreddingSchema", "true"}, + {"variant.shredding.maxInferBufferRow", std::to_string(buffer_rows)}, + // Keep the manifest format resolvable in test binaries without the avro plugin. + {"manifest.format", "parquet"}}; + EXPECT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap(option_map)); + auto plan_factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_); + auto create_inner = [this](const std::shared_ptr& converter) + -> Result< + std::unique_ptr>>> { + captured_converters_.push_back(converter); + auto writer = std::make_unique(converter, logical_type_, &sink_); + inner_ = writer.get(); + return std::unique_ptr>>( + std::move(writer)); + }; + return std::make_unique< + InferShreddingFileWriter<::ArrowArray*, std::shared_ptr>>( + schema_, plan_factory, "parquet", create_inner); + } + + protected: + std::shared_ptr pool_; + std::shared_ptr schema_; + std::shared_ptr logical_type_; + // The converters own the arrow pool backing the collected arrays; keep them declared first + // so the arrays are destroyed before the pool. + std::vector> captured_converters_; + std::vector> sink_; + CollectingFileWriter* inner_ = nullptr; +}; + +TEST_F(InferShreddingFileWriterTest, BuffersUntilThresholdThenReplays) { + auto writer = MakeWriter(/*buffer_rows=*/4); + // Never rolls while buffering. + ASSERT_OK_AND_ASSIGN(bool reach, writer->ReachTargetSize(true, 1)); + ASSERT_FALSE(reach); + + ASSERT_OK(WriteBatch( + writer.get(), {R"({"age": 35, "city": "Chicago"})", R"({"age": 25, "city": "Hangzhou"})"})); + ASSERT_TRUE(sink_.empty()); + ASSERT_EQ(writer->RecordCount(), 2); + + // Crossing the row threshold finalizes the plan and replays the buffered batches. + ASSERT_OK(WriteBatch( + writer.get(), {R"({"age": 18, "city": "Beijing"})", R"({"age": 60, "city": "Shanghai"})"})); + ASSERT_EQ(sink_.size(), 2); + ASSERT_EQ(captured_converters_.size(), 1); + ASSERT_NE(captured_converters_[0], nullptr); + const auto& physical_type = static_cast(*sink_[0]->type()); + const auto& variant_physical = + static_cast(*physical_type.GetFieldByName("v")->type()); + ASSERT_NE(variant_physical.GetFieldByName("typed_value"), nullptr); + + // Subsequent writes stream through the finalized writer directly. + ASSERT_OK(WriteBatch(writer.get(), {R"({"age": 1, "city": "Suzhou"})"})); + ASSERT_EQ(sink_.size(), 3); + ASSERT_EQ(writer->RecordCount(), 5); + + ASSERT_OK(writer->Close()); + ASSERT_TRUE(inner_->closed); +} + +TEST_F(InferShreddingFileWriterTest, CloseFlushesPartialBuffer) { + auto writer = MakeWriter(/*buffer_rows=*/100); + ASSERT_OK(WriteBatch(writer.get(), {R"({"age": 35, "city": "Chicago"})"})); + ASSERT_TRUE(sink_.empty()); + ASSERT_OK(writer->Close()); + ASSERT_EQ(sink_.size(), 1); + ASSERT_EQ(captured_converters_.size(), 1); + ASSERT_NE(captured_converters_[0], nullptr); + ASSERT_TRUE(inner_->closed); +} + +TEST_F(InferShreddingFileWriterTest, EmptyFileFallsBackToLogicalSchema) { + auto writer = MakeWriter(/*buffer_rows=*/4); + ASSERT_OK(writer->Close()); + // With no samples there is no useful shredding schema; the writer is created without a + // converter. + ASSERT_EQ(captured_converters_.size(), 1); + ASSERT_EQ(captured_converters_[0], nullptr); + ASSERT_TRUE(sink_.empty()); + ASSERT_TRUE(inner_->closed); +} + +} // namespace paimon::test diff --git a/src/paimon/core/io/rolling_file_writer.h b/src/paimon/core/io/rolling_file_writer.h index 71d04cc3..ac53bead 100644 --- a/src/paimon/core/io/rolling_file_writer.h +++ b/src/paimon/core/io/rolling_file_writer.h @@ -156,8 +156,10 @@ Status RollingFileWriter::CloseCurrentWriter() { if (current_writer_ == nullptr) { return Status::OK(); } - std::shared_ptr current_metrics = current_writer_->GetMetrics(); PAIMON_RETURN_NOT_OK(current_writer_->Close()); + // Read the metrics after Close(): writers that create their inner writer lazily (e.g. + // inferred shredding) only expose metrics once closed. + std::shared_ptr current_metrics = current_writer_->GetMetrics(); PAIMON_ASSIGN_OR_RAISE(auto abort_executor, current_writer_->GetAbortExecutor()); closed_writers_.push_back(abort_executor); PAIMON_ASSIGN_OR_RAISE(R result, current_writer_->GetResult()); diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp index 67bfef68..5e9b5718 100644 --- a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp +++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp @@ -22,13 +22,11 @@ #include #include "arrow/c/helpers.h" -#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" -#include "paimon/common/data/shredding/map_shared_shredding_context.h" -#include "paimon/common/data/shredding/map_shared_shredding_utils.h" -#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/io/data_file_writer.h" +#include "paimon/core/io/infer_shredding_file_writer.h" +#include "paimon/format/file_format.h" #include "paimon/fs/file_system.h" namespace paimon { @@ -39,21 +37,40 @@ ShreddingAppendDataFileWriterFactory::ShreddingAppendDataFileWriterFactory( const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, FileSource file_source, const std::shared_ptr& path_factory, - const std::shared_ptr& shredding_context, + const std::shared_ptr& plan_factory, const std::shared_ptr& pool) : AppendDataFileWriterFactory(options, schema_id, write_schema, write_cols, seq_num_counter, file_source, path_factory, pool), - shredding_context_(shredding_context) {} + plan_factory_(plan_factory) {} Result>>> ShreddingAppendDataFileWriterFactory::CreateWriter() const { - if (!shredding_context_) { - return Status::Invalid("Shared-shredding append writer requires a shredding context."); + if (!plan_factory_) { + return Status::Invalid("Shredding append writer requires a write-plan factory."); + } + const std::string format_identifier = options_.GetFileFormat()->Identifier(); + if (plan_factory_->ShouldInferWritePlan()) { + auto create_inner = [this](const std::shared_ptr& converter) { + return CreateShreddedWriter(converter); + }; + return std::make_unique< + InferShreddingFileWriter<::ArrowArray*, std::shared_ptr>>( + write_schema_, plan_factory_, format_identifier, std::move(create_inner)); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr converter, + plan_factory_->CreateConverter(format_identifier, /*sample_batches=*/{})); + return CreateShreddedWriter(converter); +} + +Result>>> +ShreddingAppendDataFileWriterFactory::CreateShreddedWriter( + const std::shared_ptr& converter) const { + if (converter == nullptr) { + // No conversion is useful for this file; fall back to the plain writer. + return AppendDataFileWriterFactory::CreateWriter(); } std::shared_ptr seq_num_counter = ResolveSeqNumCounter(); - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, - MapSharedShreddingBatchConverter::Create( - write_schema_, shredding_context_, options_, pool_)); std::shared_ptr file_schema = converter->GetPhysicalSchema(); std::function batch_converter = [converter](::ArrowArray* input, ::ArrowArray* output) -> Status { @@ -70,9 +87,11 @@ ShreddingAppendDataFileWriterFactory::CreateWriter() const { pool_); PAIMON_RETURN_NOT_OK( writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); - writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( - converter, MapSharedShreddingDefine::kDefaultDictCompression, shredding_context_, - file_schema)); + ShreddingWritePlanFactory::MetadataFinalizer finalizer = + plan_factory_->CreateMetadataFinalizer(converter); + if (finalizer) { + writer->SetMetadataFinalizer(std::move(finalizer)); + } return std::unique_ptr>>( std::move(writer)); } diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.h b/src/paimon/core/io/shredding_append_data_file_writer_factory.h index 3c46a0be..a69d9e5d 100644 --- a/src/paimon/core/io/shredding_append_data_file_writer_factory.h +++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.h @@ -24,15 +24,18 @@ #include #include +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" #include "paimon/core/io/append_data_file_writer_factory.h" namespace paimon { class DataFilePathFactory; class LongCounter; -class MapSharedShreddingContext; class MemoryPool; +/// Creates append data file writers that rewrite logical batches into a physical (shredded) +/// layout as planned by a `ShreddingWritePlanFactory` (MAP shared-shredding or VARIANT +/// shredding, configured or inferred). class ShreddingAppendDataFileWriterFactory : public AppendDataFileWriterFactory { public: ShreddingAppendDataFileWriterFactory( @@ -41,14 +44,17 @@ class ShreddingAppendDataFileWriterFactory : public AppendDataFileWriterFactory const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, FileSource file_source, const std::shared_ptr& path_factory, - const std::shared_ptr& shredding_context, + const std::shared_ptr& plan_factory, const std::shared_ptr& pool); Result>>> CreateWriter() const override; private: - std::shared_ptr shredding_context_; + Result>>> + CreateShreddedWriter(const std::shared_ptr& converter) const; + + std::shared_ptr plan_factory_; }; } // namespace paimon diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp index ce502fb6..10c7f8b2 100644 --- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp +++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp @@ -19,16 +19,12 @@ #include "paimon/core/io/shredding_key_value_data_file_writer_factory.h" -#include #include #include "arrow/c/helpers.h" -#include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h" -#include "paimon/common/data/shredding/map_shared_shredding_context.h" -#include "paimon/common/data/shredding/map_shared_shredding_utils.h" -#include "paimon/common/data/shredding/map_shredding_defs.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_path_factory.h" +#include "paimon/core/io/infer_shredding_file_writer.h" #include "paimon/core/io/key_value_data_file_writer.h" #include "paimon/format/file_format.h" #include "paimon/fs/file_system.h" @@ -40,20 +36,40 @@ ShreddingKeyValueDataFileWriterFactory::ShreddingKeyValueDataFileWriterFactory( const std::shared_ptr& write_schema, int32_t level, FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, bool create_stats_extractor, - const std::shared_ptr& shredding_context, + const std::shared_ptr& plan_factory, const std::shared_ptr& pool) : KeyValueDataFileWriterFactory(options, schema_id, write_schema, level, file_source, primary_keys, path_factory, create_stats_extractor, pool), - shredding_context_(shredding_context) {} + plan_factory_(plan_factory) {} Result>>> ShreddingKeyValueDataFileWriterFactory::CreateWriter() const { - if (!shredding_context_) { - return Status::Invalid("Shared-shredding key-value writer requires a shredding context."); + if (!plan_factory_) { + return Status::Invalid("Shredding key-value writer requires a write-plan factory."); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter, - MapSharedShreddingBatchConverter::Create( - write_schema_, shredding_context_, options_, pool_)); + const std::string format_identifier = options_.GetWriteFileFormat(level_)->Identifier(); + if (plan_factory_->ShouldInferWritePlan()) { + auto create_inner = [this](const std::shared_ptr& converter) { + return CreateShreddedWriter(converter); + }; + return std::make_unique< + InferShreddingFileWriter>>( + write_schema_, plan_factory_, format_identifier, std::move(create_inner)); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr converter, + plan_factory_->CreateConverter(format_identifier, /*sample_batches=*/{})); + return CreateShreddedWriter(converter); +} + +Result>>> +ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter( + const std::shared_ptr& converter) const { + if (converter == nullptr) { + // No conversion is useful for this file; fall back to the plain writer. + return KeyValueDataFileWriterFactory::CreateWriter(); + } + auto format = options_.GetWriteFileFormat(level_); std::shared_ptr file_schema = converter->GetPhysicalSchema(); std::function batch_converter = [converter](KeyValueBatch key_value_batch, ::ArrowArray* array) -> Status { @@ -62,8 +78,6 @@ ShreddingKeyValueDataFileWriterFactory::CreateWriter() const { ArrowArrayMove(physical.get(), array); return Status::OK(); }; - - auto format = options_.GetWriteFileFormat(level_); PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, file_schema, create_stats_extractor_)); auto writer = std::make_unique( @@ -72,9 +86,11 @@ ShreddingKeyValueDataFileWriterFactory::CreateWriter() const { path_factory_->IsExternalPath(), pool_); PAIMON_RETURN_NOT_OK( writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder)); - writer->SetMetadataFinalizer(MapSharedShreddingUtils::BuildMetadataFinalizer( - converter, MapSharedShreddingDefine::kDefaultDictCompression, shredding_context_, - file_schema)); + ShreddingWritePlanFactory::MetadataFinalizer finalizer = + plan_factory_->CreateMetadataFinalizer(converter); + if (finalizer) { + writer->SetMetadataFinalizer(std::move(finalizer)); + } return std::unique_ptr>>( std::move(writer)); } diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h index 2e2d654f..fb967588 100644 --- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h +++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.h @@ -23,14 +23,17 @@ #include #include +#include "paimon/common/data/shredding/shredding_write_plan_factory.h" #include "paimon/core/io/key_value_data_file_writer_factory.h" namespace paimon { class DataFilePathFactory; -class MapSharedShreddingContext; class MemoryPool; +/// Creates key-value data file writers that rewrite logical batches into a physical (shredded) +/// layout as planned by a `ShreddingWritePlanFactory` (MAP shared-shredding or VARIANT +/// shredding, configured or inferred). class ShreddingKeyValueDataFileWriterFactory : public KeyValueDataFileWriterFactory { public: ShreddingKeyValueDataFileWriterFactory( @@ -38,14 +41,17 @@ class ShreddingKeyValueDataFileWriterFactory : public KeyValueDataFileWriterFact const std::shared_ptr& write_schema, int32_t level, FileSource file_source, const std::vector& primary_keys, const std::shared_ptr& path_factory, bool create_stats_extractor, - const std::shared_ptr& shredding_context, + const std::shared_ptr& plan_factory, const std::shared_ptr& pool); Result>>> CreateWriter() const override; private: - std::shared_ptr shredding_context_; + Result>>> + CreateShreddedWriter(const std::shared_ptr& converter) const; + + std::shared_ptr plan_factory_; }; } // namespace paimon diff --git a/src/paimon/core/io/single_file_writer.h b/src/paimon/core/io/single_file_writer.h index c085729c..a92b827a 100644 --- a/src/paimon/core/io/single_file_writer.h +++ b/src/paimon/core/io/single_file_writer.h @@ -106,16 +106,16 @@ class SingleFileWriter : public FileWriter { return nullptr; } - Result ReachTargetSize(bool suggested_check, int64_t target_size); + virtual Result ReachTargetSize(bool suggested_check, int64_t target_size); - Result GetAbortExecutor() const { + virtual Result GetAbortExecutor() const { if (closed_ == false) { return Status::Invalid("Writer should be closed!"); } return AbortExecutor(fs_, path_); } - std::string GetPath() const { + virtual std::string GetPath() const { return path_; } diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp index f3f59b82..d17d7a0e 100644 --- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp +++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp @@ -23,6 +23,7 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/io/key_value_data_file_writer_factory.h" @@ -154,11 +155,12 @@ MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory, CreateDataFilePathFactory(format->Identifier())); std::shared_ptr>> factory; - if (shredding_context_) { + if (auto plan_factory = ShreddingWritePlanFactories::SelectActive(options_, write_schema_, + shredding_context_, pool_)) { factory = std::make_shared( options_, schema_id_, write_schema_, level, FileSource::Compact(), trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, - shredding_context_, pool_); + plan_factory, pool_); } else { factory = std::make_shared( options_, schema_id_, write_schema_, level, FileSource::Compact(), diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp index f6a3b424..7db94de1 100644 --- a/src/paimon/core/mergetree/merge_tree_writer.cpp +++ b/src/paimon/core/mergetree/merge_tree_writer.cpp @@ -27,6 +27,7 @@ #include "arrow/api.h" #include "arrow/c/abi.h" #include "arrow/c/helpers.h" +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -323,11 +324,12 @@ Result MergeTreeWriter::DrainIncrement() { std::unique_ptr>> MergeTreeWriter::CreateRollingRowWriter() const { std::shared_ptr>> factory; - if (shredding_context_) { + if (auto plan_factory = ShreddingWritePlanFactories::SelectActive(options_, write_schema_, + shredding_context_, pool_)) { factory = std::make_shared( options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, - shredding_context_, pool_); + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, plan_factory, + pool_); } else { factory = std::make_shared( options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index a55e1440..92e521bf 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -29,6 +29,9 @@ #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/shredding/map_shared_shredding_file_reader.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/shredding/shredding_file_reader.h" +#include "paimon/common/data/variant/variant_shredding_read_plan_factory.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/reader/delegating_prefetch_reader.h" #include "paimon/common/reader/predicate_batch_reader.h" #include "paimon/common/reader/prefetch_file_batch_reader_impl.h" @@ -201,6 +204,8 @@ Result> AbstractSplitRead::CreateFieldMappingRe std::move(file_reader), read_schema)); file_reader = std::move(shared_shredding_result.first); skip_map_selected_keys_filter_field_ids = std::move(shared_shredding_result.second); + PAIMON_ASSIGN_OR_RAISE( + file_reader, ApplyVariantShreddingReaderIfNeeded(std::move(file_reader), read_schema)); } if (NeedCompleteRowTrackingFields(options_.RowTrackingEnabled(), read_schema)) { file_reader = std::make_unique( @@ -279,6 +284,35 @@ AbstractSplitRead::ApplySharedShreddingReaderIfNeeded( return std::make_pair(std::move(file_reader), std::move(handled_shared_shredding_field_ids)); } +Result> AbstractSplitRead::ApplyVariantShreddingReaderIfNeeded( + std::unique_ptr&& file_reader, + const std::shared_ptr& read_schema) const { + bool has_variant_field = false; + for (const auto& read_field : read_schema->fields()) { + // Variant columns may be nested inside struct columns; a variant-access projection also + // matches because it carries the variant extension marker itself. + if (VariantTypeUtils::ContainsVariantField(read_field)) { + has_variant_field = true; + break; + } + } + if (!has_variant_field) { + return std::move(file_reader); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, + file_reader->GetFileSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_arrow_schema, + arrow::ImportSchema(file_schema.get())); + std::map> plans; + PAIMON_ASSIGN_OR_RAISE(plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_arrow_schema, pool_)); + if (!plans.empty()) { + file_reader = + std::make_unique(std::move(file_reader), std::move(plans), pool_); + } + return std::move(file_reader); +} + Result> AbstractSplitRead::ProjectFieldsForRowTrackingAndDataEvolution( const std::shared_ptr& data_schema, const std::optional>& write_cols) { diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index dfbe6891..d39232a5 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -115,6 +115,13 @@ class AbstractSplitRead : public SplitRead { ApplySharedShreddingReaderIfNeeded(std::unique_ptr&& file_reader, const std::shared_ptr& read_schema) const; + /// Wraps the reader with a `ShreddingFileReader` when any read variant column needs + /// reassembly or path extraction; a plain read of an unshredded variant column is passed + /// through untouched. + Result> ApplyVariantShreddingReaderIfNeeded( + std::unique_ptr&& file_reader, + const std::shared_ptr& read_schema) const; + static bool NeedCompleteRowTrackingFields(bool row_tracking_enabled, const std::shared_ptr& read_schema); diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp index ed3401ce..b021e64b 100644 --- a/src/paimon/core/operation/append_only_file_store_write.cpp +++ b/src/paimon/core/operation/append_only_file_store_write.cpp @@ -24,6 +24,7 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "paimon/common/data/binary_row.h" +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/core/append/append_only_writer.h" @@ -229,10 +230,11 @@ AppendOnlyFileStoreWrite::WriterFactory AppendOnlyFileStoreWrite::GetDataFileWri const std::vector>& to_compact, const std::shared_ptr& shredding_context) const { auto seq_num_counter = std::make_shared(to_compact[0]->min_sequence_number); - if (shredding_context) { + if (auto plan_factory = + ShreddingWritePlanFactories::SelectActive(options_, schema, shredding_context, pool_)) { return std::make_shared( options_, table_schema_->Id(), schema, write_cols, seq_num_counter, - FileSource::Compact(), data_file_path_factory, shredding_context, pool_); + FileSource::Compact(), data_file_path_factory, plan_factory, pool_); } return std::make_shared( options_, table_schema_->Id(), schema, write_cols, seq_num_counter, FileSource::Compact(), diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index 31ca2f9c..ffb2328c 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -58,7 +58,8 @@ struct DeletionFile; /// splits)->(BlobViewResolvingBatchReader)->(CompleteIndexScoreBatchReader)-> /// CompleteRowKindBatchReader->(PredicateBatchReader) /// ->ConcatBatchReader across files->DataEvolutionFileReader->(ConcatBatchReader across blob files) -/// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader)->(MapSharedShreddingFileReader) +/// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader)->(ShreddingFileReader) +/// ->(MapSharedShreddingFileReader) /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader /// /// diff --git a/src/paimon/core/operation/internal_read_context.cpp b/src/paimon/core/operation/internal_read_context.cpp index 4d5ad411..e8886b42 100644 --- a/src/paimon/core/operation/internal_read_context.cpp +++ b/src/paimon/core/operation/internal_read_context.cpp @@ -25,6 +25,8 @@ #include "arrow/c/abi.h" #include "arrow/c/bridge.h" #include "fmt/format.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" @@ -40,6 +42,14 @@ Result> InternalReadContext::AlignReadFieldWithTab const std::shared_ptr& table_field) { static const std::vector kReadMetadataWhitelist = {DataField::MAP_SELECTED_KEYS}; + if (VariantTypeUtils::IsVariantField(table_field) && + VariantAccessUtils::IsVariantAccessType(read_field->type())) { + // A variant column may be read as a variant-access projection: a struct whose children + // each carry a `__VARIANT_METADATA` description. Keep the projection type (including + // the children's descriptions) on the aligned field. + return table_field->WithType(read_field->type()); + } + if (read_field->type()->id() != table_field->type()->id()) { return Status::Invalid(fmt::format( "Read schema field '{}' type {} does not match table field type {}", read_field->name(), diff --git a/src/paimon/core/operation/merge_file_split_read.h b/src/paimon/core/operation/merge_file_split_read.h index 45bcddca..895bb573 100644 --- a/src/paimon/core/operation/merge_file_split_read.h +++ b/src/paimon/core/operation/merge_file_split_read.h @@ -73,7 +73,8 @@ class MergeFunctionWrapper; /// ->ConcatBatchReader across no overlapped /// files->KeyValueProjectionReader/AsyncKeyValueProjectionReader /// ->DropDeleteReader->SortMergeReader->ConcatKeyValueRecordReader->KeyValueDataFileRecordReader -/// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(MapSharedShreddingFileReader) +/// ->FieldMappingReader->(ApplyDeletionVectorBatchReader)->(ShreddingFileReader) +/// ->(MapSharedShreddingFileReader) /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader class MergeFileSplitRead : public AbstractSplitRead { public: diff --git a/src/paimon/core/operation/raw_file_split_read.h b/src/paimon/core/operation/raw_file_split_read.h index ba5035f6..1580cd84 100644 --- a/src/paimon/core/operation/raw_file_split_read.h +++ b/src/paimon/core/operation/raw_file_split_read.h @@ -55,7 +55,8 @@ struct DeletionFile; /// splits)->CompleteRowKindBatchReader->(PredicateBatchReader) /// ->ConcatBatchReader across /// files->FieldMappingReader->(ApplyBitmapIndexBatchReader)->(CompleteRowTrackingFieldsBatchReader) -/// ->(MapSharedShreddingFileReader)->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader +/// ->(ShreddingFileReader)->(MapSharedShreddingFileReader)->(DelegatingPrefetchReader) +/// ->(PrefetchFileBatchReader)->FormatReader class RawFileSplitRead : public AbstractSplitRead { public: diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp index 7159e03b..208a0a29 100644 --- a/src/paimon/core/postpone/postpone_bucket_writer.cpp +++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp @@ -32,6 +32,7 @@ #include "arrow/scalar.h" #include "arrow/util/checked_cast.h" #include "fmt/format.h" +#include "paimon/common/data/shredding/shredding_write_plan_factories.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" @@ -263,11 +264,12 @@ PostponeBucketWriter::PrepareMinMaxKey( std::unique_ptr>> PostponeBucketWriter::CreateRollingRowWriter() const { std::shared_ptr>> factory; - if (shredding_context_) { + if (auto plan_factory = ShreddingWritePlanFactories::SelectActive(options_, write_schema_, + shredding_context_, pool_)) { factory = std::make_shared( options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), - trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, - shredding_context_, pool_); + trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, plan_factory, + pool_); } else { factory = std::make_shared( options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(), diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index 22be2de4..60869d56 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -26,6 +26,8 @@ #include "arrow/util/checked_cast.h" #include "fmt/format.h" #include "paimon/common/data/blob_utils.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/decimal_utils.h" #include "paimon/common/utils/string_utils.h" @@ -119,6 +121,11 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId( break; } case arrow::Type::type::STRUCT: { + if (VariantTypeUtils::IsVariantMetadata(key_value_metadata)) { + // A variant struct is a leaf type: its value/metadata children carry fixed + // paimon field ids 0/1 which must not join the global field id uniqueness check. + break; + } arrow::FieldVector sub_fields = arrow::internal::checked_cast(type.get())->fields(); for (const auto& sub_field : sub_fields) { @@ -185,6 +192,16 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& break; } case arrow::Type::type::STRUCT: { + if (VariantTypeUtils::IsVariantField(field)) { + if (VariantAccessUtils::IsVariantAccessType(field->type())) { + // A variant column read as a variant-access projection keeps the variant + // marker but replaces the type with the projection struct; its children are + // cast targets validated by the variant read plans. + break; + } + PAIMON_RETURN_NOT_OK(VariantTypeUtils::ValidateVariantShape(field)); + break; + } arrow::FieldVector arrow_fields = arrow::internal::checked_cast(*field->type()).fields(); for (const auto& sub_field : arrow_fields) { diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp index f1ec0e5d..4fed03da 100644 --- a/src/paimon/core/schema/arrow_schema_validator_test.cpp +++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp @@ -24,6 +24,9 @@ #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/testing/utils/testharness.h" @@ -273,6 +276,47 @@ TEST(ArrowSchemaValidatorTest, ValidateDataTypeWithFieldId) { } } +TEST(ArrowSchemaValidatorTest, ValidateVariantField) { + // A variant field validates as a leaf: its fixed child ids 0/1 do not join the global field + // id uniqueness check, even when top-level fields use ids 0/1. + { + std::vector fields = {DataField(0, arrow::field("f0", arrow::utf8())), + DataField(1, VariantTypeUtils::ToArrowField("v1")), + DataField(2, VariantTypeUtils::ToArrowField("v2"))}; + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields); + ASSERT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema)) + << ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema).ToString(); + } + // A variant-access projection (variant marker + description-carrying children) validates + // as a leaf instead of being shape-checked. + { + auto age_child = + arrow::field("0", arrow::int64(), true, + arrow::KeyValueMetadata::Make( + {DataField::DESCRIPTION}, + {VariantAccessUtils::BuildVariantMetadata("$.age", true, "UTC")})); + std::unordered_map metadata = { + {VariantDefs::kExtensionTypeKey, VariantDefs::kExtensionTypeValue}}; + auto access_field = arrow::field("v", arrow::struct_({age_child}), true, + std::make_shared(metadata)); + auto arrow_schema = arrow::schema({access_field}); + ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow_schema)) + << ArrowSchemaValidator::ValidateSchema(*arrow_schema).ToString(); + } + // A variant-marked struct with the wrong physical shape is rejected. + { + std::unordered_map metadata = { + {VariantDefs::kExtensionTypeKey, VariantDefs::kExtensionTypeValue}}; + auto bad_variant = + arrow::field("v", + arrow::struct_({arrow::field("value", arrow::binary(), true), + arrow::field("metadata", arrow::binary(), true)}), + true, std::make_shared(metadata)); + auto arrow_schema = arrow::schema({bad_variant}); + ASSERT_NOK(ArrowSchemaValidator::ValidateSchema(*arrow_schema)); + } +} + TEST(ArrowSchemaValidatorTest, ContainTimestampWithTimezone) { auto timezone = DateTimeUtils::GetLocalTimezoneName(); { diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index c27e609f..ecb83b61 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -34,6 +34,7 @@ #include "fmt/ranges.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/shredding/map_shared_shredding_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/object_utils.h" @@ -534,6 +535,12 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema, if (layout != MapStorageLayout::SHARED_SHREDDING) { continue; } + for (const auto& field : schema.Fields()) { + if (VariantTypeUtils::ContainsVariantField(field.ArrowField())) { + return Status::Invalid( + "MAP shared-shredding currently cannot be used with Variant fields."); + } + } // Column configured with shared-shredding must be MAP if (!MapSharedShreddingUtils::IsShreddingKeyMap(field_type)) { return Status::Invalid( diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 1e089cb9..ec434261 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -28,6 +28,7 @@ #include "arrow/c/bridge.h" #include "arrow/util/checked_cast.h" #include "fmt/format.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/field_type_utils.h" @@ -96,6 +97,11 @@ Result> TableSchema::AssignFieldIdsRecursively( } auto type = field->type(); if (type->id() == arrow::Type::STRUCT) { + if (VariantTypeUtils::IsVariantField(field)) { + // A variant struct is a leaf type: its value/metadata children keep their fixed + // paimon field ids 0/1 (mapped to parquet field ids on write). + return metadata ? field->WithMergedMetadata(metadata) : field; + } auto struct_type = arrow::internal::checked_pointer_cast(field->type()); arrow::FieldVector new_childs; for (const auto& child : struct_type->fields()) { @@ -190,7 +196,7 @@ std::vector TableSchema::FieldNames() const { Result TableSchema::GetFieldType(const std::string& field_name) const { PAIMON_ASSIGN_OR_RAISE(DataField field, GetField(field_name)); - return FieldTypeUtils::ConvertToFieldType(field.Type()->id()); + return FieldTypeUtils::ConvertToFieldType(field.ArrowField()); } Result TableSchema::GetField(const std::string& field_name) const { diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index dedb3bc9..0dcc74fe 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -24,6 +24,7 @@ #include "arrow/api.h" #include "arrow/util/checked_cast.h" #include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/fs/local/local_file_system.h" @@ -136,6 +137,20 @@ TEST_F(TableSchemaTest, TestCreateWithAllFieldsHaveFieldId) { ASSERT_EQ(table_schema->Options(), options); } +TEST_F(TableSchemaTest, TestGetFieldTypeForVariant) { + arrow::FieldVector fields = {arrow::field("id", arrow::int32()), + VariantTypeUtils::ToArrowField("v")}; + ASSERT_OK_AND_ASSIGN(auto table_schema, + TableSchema::Create(/*schema_id=*/0, arrow::schema(fields), + /*partition_keys=*/{}, /*primary_keys=*/{}, + /*options=*/{})); + ASSERT_OK_AND_ASSIGN(FieldType id_type, table_schema->GetFieldType("id")); + ASSERT_EQ(id_type, FieldType::INT); + // The variant marker must disambiguate the physical struct into FieldType::VARIANT. + ASSERT_OK_AND_ASSIGN(FieldType variant_type, table_schema->GetFieldType("v")); + ASSERT_EQ(variant_type, FieldType::VARIANT); +} + TEST_F(TableSchemaTest, TestInvalidCreate) { // partial fields have field id arrow::FieldVector fields = MakeArrowField(3); diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp index 8f73daa4..bd64ee14 100644 --- a/src/paimon/core/utils/nested_projection_utils.cpp +++ b/src/paimon/core/utils/nested_projection_utils.cpp @@ -31,6 +31,8 @@ #include "arrow/array/concatenate.h" #include "arrow/type.h" #include "fmt/format.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/status.h" @@ -84,6 +86,11 @@ Result NestedProjectionUtils::HasNestedSubfieldProjectionType( const std::shared_ptr& read_type) { switch (file_type->id()) { case arrow::Type::STRUCT: { + if (VariantAccessUtils::IsVariantAccessType(read_type)) { + // A variant-access projection is resolved by the variant read plans, not by + // nested subfield projection. + return false; + } if (read_type->id() != arrow::Type::STRUCT) { return Status::Invalid(fmt::format( "HasNestedSubfieldProjectionType requires same nested type kind, but file " @@ -144,6 +151,41 @@ Result NestedProjectionUtils::HasNestedSubfieldProjectionType( } } +namespace { + +/// Whether `read_type` is `data_type` with variant columns replaced by their variant-access +/// projections and nothing else changed. Such a read drops no field, so it is not a partial +/// projection of an enclosing repeated group and may pass through where a real one must fail. +bool IsVariantAccessSubstitution(const std::shared_ptr& read_type, + const std::shared_ptr& data_type) { + if (read_type->Equals(data_type)) { + return true; + } + if (VariantAccessUtils::IsVariantAccessType(read_type) && + VariantTypeUtils::IsUnshreddedVariantType(data_type)) { + return true; + } + // Any other difference in shape, including a dropped field, is a real projection. + if (read_type->id() != data_type->id() || read_type->num_fields() != data_type->num_fields()) { + return false; + } + for (int32_t i = 0; i < read_type->num_fields(); ++i) { + const std::shared_ptr& read_child = read_type->field(i); + const std::shared_ptr& data_child = data_type->field(i); + // LIST and MAP name their children by format convention, so only STRUCT is matched + // by name. + if (read_type->id() == arrow::Type::STRUCT && read_child->name() != data_child->name()) { + return false; + } + if (!IsVariantAccessSubstitution(read_child->type(), data_child->type())) { + return false; + } + } + return true; +} + +} // namespace + Result>> NestedProjectionUtils::PruneDataType( const std::shared_ptr& read_type, const std::shared_ptr& data_type) { @@ -154,6 +196,12 @@ Result>> NestedProjectionUtils::P switch (read_type->id()) { case arrow::Type::STRUCT: { + if (VariantAccessUtils::IsVariantAccessType(read_type) && + VariantTypeUtils::IsUnshreddedVariantType(data_type)) { + // A variant-access projection replaces the variant column type; pass it through + // so the read path extracts the described paths. + return std::optional>(read_type); + } arrow::FieldVector pruned_fields; for (const auto& read_child : read_type->fields()) { PAIMON_ASSIGN_OR_RAISE(int32_t read_child_id, GetPaimonFieldId(read_child)); @@ -187,6 +235,9 @@ Result>> NestedProjectionUtils::P return std::optional>(arrow::struct_(pruned_fields)); } case arrow::Type::LIST: { + if (IsVariantAccessSubstitution(read_type, data_type)) { + return std::optional>(read_type); + } // Keep behavior aligned with format readers: partial projection inside // LIST is unsupported and must fail fast. return Status::Invalid( @@ -195,6 +246,9 @@ Result>> NestedProjectionUtils::P data_type->ToString(), read_type->ToString())); } case arrow::Type::MAP: { + if (IsVariantAccessSubstitution(read_type, data_type)) { + return std::optional>(read_type); + } // Keep behavior aligned with format readers: partial projection inside // MAP is unsupported and must fail fast. return Status::Invalid(fmt::format( diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp index 26b45015..c91adff3 100644 --- a/src/paimon/core/utils/nested_projection_utils_test.cpp +++ b/src/paimon/core/utils/nested_projection_utils_test.cpp @@ -27,6 +27,8 @@ #include "arrow/memory_pool.h" #include "arrow/type.h" #include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_access_utils.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/types/data_field.h" #include "paimon/testing/utils/testharness.h" @@ -40,6 +42,17 @@ static std::shared_ptr MakeField(const std::string& name, return DataField::ConvertDataFieldToArrowField(data_field); } +// Helper: a variant-access projection, i.e. a struct whose children carry `__VARIANT_METADATA` +// descriptions. +static std::shared_ptr MakeVariantAccessType() { + auto child = arrow::field( + "0", arrow::int64(), /*nullable=*/true, + arrow::KeyValueMetadata::Make( + {DataField::DESCRIPTION}, + {VariantAccessUtils::BuildVariantMetadata("$.x", /*fail_on_error=*/false, "UTC")})); + return arrow::struct_({child}); +} + // ============== GetPaimonFieldId ============== TEST(NestedProjectionUtilsTest, GetPaimonFieldIdPresent) { @@ -162,6 +175,56 @@ TEST(NestedProjectionUtilsTest, PruneDataTypeMapWithStructValue) { "partial projection inside map"); } +TEST(NestedProjectionUtilsTest, PruneDataTypeListWithVariantAccessElement) { + // data: LIST, read: LIST + // Not a projection of the list itself, so it must pass through to the variant read plans. + auto data_type = arrow::list(arrow::field("element", VariantTypeUtils::UnshreddedStructType())); + auto read_type = arrow::list(arrow::field("element", MakeVariantAccessType())); + + ASSERT_OK_AND_ASSIGN(std::optional> pruned, + NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(pruned.has_value()); + ASSERT_TRUE(pruned.value()->Equals(*read_type)) << pruned.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeMapWithVariantAccessValue) { + auto data_type = arrow::map(arrow::utf8(), VariantTypeUtils::UnshreddedStructType()); + auto read_type = arrow::map(arrow::utf8(), MakeVariantAccessType()); + + ASSERT_OK_AND_ASSIGN(std::optional> pruned, + NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(pruned.has_value()); + ASSERT_TRUE(pruned.value()->Equals(*read_type)) << pruned.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListWithVariantAccessInsideStruct) { + // The variant sits one struct level below the list, next to a plain sibling that is kept. + auto data_inner = arrow::struct_({MakeField("v", VariantTypeUtils::UnshreddedStructType(), 10), + MakeField("t", arrow::utf8(), 11)}); + auto read_inner = arrow::struct_( + {MakeField("v", MakeVariantAccessType(), 10), MakeField("t", arrow::utf8(), 11)}); + auto data_type = arrow::list(arrow::field("element", data_inner)); + auto read_type = arrow::list(arrow::field("element", read_inner)); + + ASSERT_OK_AND_ASSIGN(std::optional> pruned, + NestedProjectionUtils::PruneDataType(read_type, data_type)); + ASSERT_TRUE(pruned.has_value()); + ASSERT_TRUE(pruned.value()->Equals(*read_type)) << pruned.value()->ToString(); +} + +TEST(NestedProjectionUtilsTest, PruneDataTypeListDroppingSiblingOfVariantStillFails) { + // Dropping the plain sibling is a real partial projection inside the list and must keep + // failing fast, variant access or not. + auto data_inner = arrow::struct_({MakeField("v", VariantTypeUtils::UnshreddedStructType(), 10), + MakeField("t", arrow::utf8(), 11)}); + auto read_inner = arrow::struct_({MakeField("v", MakeVariantAccessType(), 10)}); + auto data_type = arrow::list(arrow::field("element", data_inner)); + auto read_type = arrow::list(arrow::field("element", read_inner)); + + ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type), + "partial projection inside list"); +} + TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) { auto file_schema = arrow::schema({ MakeField("f0", arrow::int32(), 1), diff --git a/src/paimon/format/avro/avro_schema_converter.cpp b/src/paimon/format/avro/avro_schema_converter.cpp index d62bfc7e..bfdbe5e0 100644 --- a/src/paimon/format/avro/avro_schema_converter.cpp +++ b/src/paimon/format/avro/avro_schema_converter.cpp @@ -31,6 +31,7 @@ #include "avro/Types.hh" #include "avro/ValidSchema.hh" #include "fmt/format.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/format/avro/avro_file_format_factory.h" #include "paimon/format/avro/avro_utils.h" @@ -380,6 +381,9 @@ Result<::avro::Schema> AvroSchemaConverter::ArrowTypeToAvroSchema( Result<::avro::ValidSchema> AvroSchemaConverter::ArrowSchemaToAvroSchema( const std::shared_ptr& arrow_schema) { + if (VariantTypeUtils::ContainsVariantField(arrow_schema)) { + return Status::NotImplemented("Avro format does not support the VARIANT type"); + } // top level row name of avro record, the same as java paimon static const std::string kTopLevelRowName = "org.apache.paimon.avro.generated.record"; ::avro::RecordSchema record_schema(kTopLevelRowName); diff --git a/src/paimon/format/avro/avro_schema_converter_test.cpp b/src/paimon/format/avro/avro_schema_converter_test.cpp index d61e930e..e30de486 100644 --- a/src/paimon/format/avro/avro_schema_converter_test.cpp +++ b/src/paimon/format/avro/avro_schema_converter_test.cpp @@ -22,6 +22,7 @@ #include "avro/Compiler.hh" #include "avro/ValidSchema.hh" #include "gtest/gtest.h" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/utils/date_time_utils.h" #include "paimon/core/manifest/manifest_file_meta.h" #include "paimon/core/utils/versioned_object_serializer.h" @@ -246,4 +247,11 @@ TEST(AvroSchemaConverterTest, TestAvroSchemaToArrowDataTypeWithTimestampType) { ASSERT_TRUE(arrow_type->Equals(arrow::struct_(expected_fields))) << arrow_type->ToString(); } +TEST(AvroSchemaConverterTest, TestVariantNotSupported) { + auto arrow_schema = + arrow::schema({arrow::field("id", arrow::int32()), VariantTypeUtils::ToArrowField("v")}); + auto result = AvroSchemaConverter::ArrowSchemaToAvroSchema(arrow_schema); + ASSERT_TRUE(result.status().IsNotImplemented()) << result.status().ToString(); +} + } // namespace paimon::avro::test diff --git a/src/paimon/format/orc/orc_format_writer.cpp b/src/paimon/format/orc/orc_format_writer.cpp index 2be84b7d..1a394ca9 100644 --- a/src/paimon/format/orc/orc_format_writer.cpp +++ b/src/paimon/format/orc/orc_format_writer.cpp @@ -38,6 +38,7 @@ #include "orc/Type.hh" #include "orc/Vector.hh" #include "orc/Writer.hh" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/options/memory_size.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -80,6 +81,9 @@ Result> OrcFormatWriter::Create( const std::map& options, const std::string& compression, int32_t batch_size, const std::shared_ptr& pool) { assert(output_stream); + if (VariantTypeUtils::ContainsVariantField(arrow::schema(schema.fields()))) { + return Status::NotImplemented("ORC format does not support the VARIANT type"); + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::orc::Type> orc_type, OrcAdapter::GetOrcType(schema)); auto data_type = arrow::struct_(schema.fields()); try { diff --git a/src/paimon/format/orc/orc_format_writer_test.cpp b/src/paimon/format/orc/orc_format_writer_test.cpp index a278b7e0..0f531f52 100644 --- a/src/paimon/format/orc/orc_format_writer_test.cpp +++ b/src/paimon/format/orc/orc_format_writer_test.cpp @@ -39,6 +39,7 @@ #include "orc/Type.hh" #include "orc/Vector.hh" #include "orc/Writer.hh" +#include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/format/orc/orc_format_defs.h" #include "paimon/format/orc/orc_input_stream_impl.h" #include "paimon/format/orc/orc_metrics.h" @@ -297,6 +298,23 @@ TEST_F(OrcFormatWriterTest, TestPrepareWriterOptions) { "invalid config, do not support writing timestamp with timezone in legacy format"); } } +TEST_F(OrcFormatWriterTest, TestVariantNotSupported) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + std::string file_name = test_root + "/variant.orc"; + auto arrow_schema = + arrow::schema({arrow::field("id", arrow::int32()), VariantTypeUtils::ToArrowField("v")}); + std::map options = {}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + file_system_->Create(file_name, /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr output_stream, + OrcOutputStreamImpl::Create(out)); + auto result = OrcFormatWriter::Create(std::move(output_stream), *arrow_schema, options, + /*compression=*/"lz4", /*batch_size=*/16, pool_); + ASSERT_TRUE(result.status().IsNotImplemented()) << result.status().ToString(); +} + // TODO(liancheng.lsz): add tests for GetEstimateLength } // namespace paimon::orc::test diff --git a/src/paimon/format/parquet/CMakeLists.txt b/src/paimon/format/parquet/CMakeLists.txt index 8129902c..a1a566c0 100644 --- a/src/paimon/format/parquet/CMakeLists.txt +++ b/src/paimon/format/parquet/CMakeLists.txt @@ -63,6 +63,7 @@ if(PAIMON_BUILD_TESTS) predicate_converter_test.cpp predicate_pushdown_test.cpp column_index_filter_test.cpp + variant_parquet_test.cpp STATIC_LINK_LIBS paimon_shared test_utils_static diff --git a/src/paimon/format/parquet/variant_parquet_test.cpp b/src/paimon/format/parquet/variant_parquet_test.cpp new file mode 100644 index 00000000..2a7e521b --- /dev/null +++ b/src/paimon/format/parquet/variant_parquet_test.cpp @@ -0,0 +1,1053 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "arrow/io/file.h" +#include "gtest/gtest.h" +#include "paimon/common/data/shredding/shredding_file_reader.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_defs.h" +#include "paimon/common/data/variant/variant_schema.h" +#include "paimon/common/data/variant/variant_shredding_batch_converter.h" +#include "paimon/common/data/variant/variant_shredding_read_plan_factory.h" +#include "paimon/common/data/variant/variant_shredding_utils.h" +#include "paimon/common/data/variant/variant_shredding_write_plan.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/data/variant.h" +#include "paimon/format/parquet/parquet_field_id_converter.h" +#include "paimon/format/parquet/parquet_file_batch_reader.h" +#include "paimon/format/parquet/parquet_format_defs.h" +#include "paimon/format/parquet/parquet_format_writer.h" +#include "paimon/fs/local/local_file_system.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/variant_test_data.h" +#include "parquet/arrow/reader.h" +#include "parquet/file_reader.h" +#include "parquet/metadata.h" +#include "parquet/properties.h" +#include "parquet/schema.h" + +namespace paimon::parquet::test { + +class VariantParquetTest : public ::testing::Test { + public: + void SetUp() override { + dir_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + fs_ = std::make_shared(); + pool_ = GetDefaultPool(); + arrow_pool_ = GetArrowPool(pool_); + file_path_ = PathUtil::JoinPath(dir_->Str(), "variant.parquet"); + + std::vector fields = {DataField(1, arrow::field("id", arrow::int32())), + DataField(2, VariantTypeUtils::ToArrowField("v"))}; + paimon_schema_ = DataField::ConvertDataFieldsToArrowSchema(fields); + + // `[id, s: struct]`: a variant nested inside a ROW column, next to + // a plain sibling that must survive the physical substitution and the reassembly. + nested_sibling_field_ = arrow::field("t", arrow::utf8()); + std::vector nested_fields = { + DataField(1, arrow::field("id", arrow::int32())), + DataField(2, arrow::field("s", arrow::struct_({VariantTypeUtils::ToArrowField("nv"), + nested_sibling_field_})))}; + nested_schema_ = DataField::ConvertDataFieldsToArrowSchema(nested_fields); + nested_variant_field_ = nested_schema_->field(1)->type()->field(0); + + // `[id, arr: ARRAY]` and `[id, m: MAP]`: variants inside + // repeated groups, which are never shredded but can still be read as a projection. + list_element_field_ = VariantTypeUtils::ToArrowField("element"); + std::vector list_fields = { + DataField(1, arrow::field("id", arrow::int32())), + DataField(2, arrow::field("arr", arrow::list(list_element_field_)))}; + list_schema_ = DataField::ConvertDataFieldsToArrowSchema(list_fields); + list_element_field_ = list_schema_->field(1)->type()->field(0); + + map_item_field_ = VariantTypeUtils::ToArrowField("value"); + std::vector map_fields = { + DataField(1, arrow::field("id", arrow::int32())), + DataField(2, arrow::field("m", arrow::map(arrow::utf8(), map_item_field_)))}; + map_schema_ = DataField::ConvertDataFieldsToArrowSchema(map_fields); + map_item_field_ = map_schema_->field(1)->type()->field(0)->type()->field(1); + + // `[id, arr2: ARRAY>]`: a variant one struct level below a + // repeated group, where the read and file children must line up field by field. + std::vector list_struct_fields = { + DataField(1, arrow::field("id", arrow::int32())), + DataField( + 2, arrow::field("arr2", + arrow::list(arrow::field( + "element", arrow::struct_({VariantTypeUtils::ToArrowField("v"), + nested_sibling_field_})))))}; + list_struct_schema_ = DataField::ConvertDataFieldsToArrowSchema(list_struct_fields); + list_struct_variant_field_ = + list_struct_schema_->field(1)->type()->field(0)->type()->field(0); + } + + std::shared_ptr BuildArray(const std::vector& jsons) { + EXPECT_OK_AND_ASSIGN(std::shared_ptr batch, + paimon::test::VariantTestData::BuildVariantBatch( + paimon_schema_->field(0), paimon_schema_->field(1), jsons, pool_)); + return batch; + } + + // Writes one batch with the given logical schema through the production parquet write path + // (mapping paimon field ids to parquet field ids). + void WriteFile(const std::shared_ptr& schema, ArrowArray* c_array) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr write_schema, + ParquetFieldIdConverter::AddParquetIdsFromPaimonIds(schema)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr out, + fs_->Create(file_path_, /*overwrite=*/true)); + ::parquet::WriterProperties::Builder builder; + auto writer_properties = builder.build(); + ASSERT_OK_AND_ASSIGN( + auto format_writer, + ParquetFormatWriter::Create(out, write_schema, writer_properties, + DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_)); + ASSERT_OK(format_writer->AddBatch(c_array)); + ASSERT_OK(format_writer->Flush()); + ASSERT_OK(format_writer->Finish()); + ASSERT_OK(out->Flush()); + ASSERT_OK(out->Close()); + } + + void WriteFile(const std::shared_ptr& array) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*array, arrow_array.get()).ok()); + WriteFile(paimon_schema_, arrow_array.get()); + } + + // Writes `jsons` shredded according to the configured ROW-type shredding schema JSON. + void WriteShreddedFile(const std::vector& jsons, + const char* shredding_schema_json) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr plan, + VariantShreddingWritePlan::FromConfiguredSchema(paimon_schema_, shredding_schema_json)); + ASSERT_NE(plan, nullptr); + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + VariantShreddingBatchConverter::Create(plan, pool_)); + auto logical = BuildArray(jsons); + auto c_logical = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*logical, c_logical.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_physical, + converter->Convert(c_logical.get())); + WriteFile(converter->GetPhysicalSchema(), c_physical.get()); + } + + static std::string NestedSiblingValue(size_t row) { + return "t" + std::to_string(row); + } + + std::shared_ptr BuildIdColumn(size_t rows) { + arrow::Int32Builder id_builder; + for (size_t i = 0; i < rows; ++i) { + EXPECT_TRUE(id_builder.Append(static_cast(i)).ok()); + } + std::shared_ptr ids; + EXPECT_TRUE(id_builder.Finish(&ids).ok()); + return ids->data(); + } + + // Builds the offsets buffer of a repeated column, flattening its elements into `flat`. + std::shared_ptr BuildOffsetsBuffer( + const std::vector>& rows, std::vector* flat) { + arrow::Int32Builder offset_builder; + for (const auto& row : rows) { + EXPECT_TRUE(offset_builder.Append(static_cast(flat->size())).ok()); + flat->insert(flat->end(), row.begin(), row.end()); + } + EXPECT_TRUE(offset_builder.Append(static_cast(flat->size())).ok()); + std::shared_ptr offsets; + EXPECT_TRUE(offset_builder.Finish(&offsets).ok()); + return offsets->data()->buffers[1]; + } + + // Builds a `[id, arr: list]` batch: row `i` holds the variants of `rows[i]`. + // + // The repeated columns here are assembled at the `ArrayData` level on purpose: Arrow's + // `FromArrays` helpers `checked_cast` their arguments, which is a `dynamic_cast` in a debug + // build and fails across the test binary / libpaimon boundary for templated array classes. + std::shared_ptr BuildListArray( + const std::vector>& rows) { + std::vector flat; + auto offsets_buffer = BuildOffsetsBuffer(rows, &flat); + // The element variants are built as one flat batch that the offsets slice into rows. + auto elements = paimon::test::VariantTestData::BuildVariantBatch( + list_schema_->field(0), list_element_field_, flat, pool_); + EXPECT_TRUE(elements.ok()) << elements.status().ToString(); + auto list_data = arrow::ArrayData::Make( + list_schema_->field(1)->type(), static_cast(rows.size()), + {nullptr, offsets_buffer}, {elements.value()->field(1)->data()}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make( + arrow::struct_(list_schema_->fields()), static_cast(rows.size()), {nullptr}, + {BuildIdColumn(rows.size()), list_data}, /*null_count=*/0); + return std::make_shared(batch_data); + } + + // Writes `rows` into the unshredded `arr: list` column. + void WriteListFile(const std::vector>& rows) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildListArray(rows), arrow_array.get()).ok()); + WriteFile(list_schema_, arrow_array.get()); + } + + // Builds a `[id, m: map]` batch: row `i` maps `keys[i][k]` to the variant of + // `rows[i][k]`. + std::shared_ptr BuildMapArray( + const std::vector>& keys, + const std::vector>& rows) { + std::vector flat; + arrow::StringBuilder key_builder; + EXPECT_EQ(keys.size(), rows.size()); + for (size_t i = 0; i < keys.size() && i < rows.size(); ++i) { + EXPECT_EQ(keys[i].size(), rows[i].size()) << "row " << i; + for (const auto& key : keys[i]) { + EXPECT_TRUE(key_builder.Append(key).ok()); + } + } + auto offsets_buffer = BuildOffsetsBuffer(rows, &flat); + std::shared_ptr map_keys; + EXPECT_TRUE(key_builder.Finish(&map_keys).ok()); + auto elements = paimon::test::VariantTestData::BuildVariantBatch( + map_schema_->field(0), map_item_field_, flat, pool_); + EXPECT_TRUE(elements.ok()) << elements.status().ToString(); + // A map array is a list of `struct` entries. + auto entries_data = arrow::ArrayData::Make( + map_schema_->field(1)->type()->field(0)->type(), static_cast(flat.size()), + {nullptr}, {map_keys->data(), elements.value()->field(1)->data()}, /*null_count=*/0); + auto map_data = + arrow::ArrayData::Make(map_schema_->field(1)->type(), static_cast(rows.size()), + {nullptr, offsets_buffer}, {entries_data}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make( + arrow::struct_(map_schema_->fields()), static_cast(rows.size()), {nullptr}, + {BuildIdColumn(rows.size()), map_data}, /*null_count=*/0); + return std::make_shared(batch_data); + } + + // Writes `rows` into the unshredded `m: map` column. + void WriteMapFile(const std::vector>& keys, + const std::vector>& rows) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildMapArray(keys, rows), arrow_array.get()).ok()); + WriteFile(map_schema_, arrow_array.get()); + } + + // Builds a `[id, arr2: list>]` batch. The sibling of the + // `k`-th element over the whole column is `NestedSiblingValue(k)`. + std::shared_ptr BuildListStructArray( + const std::vector>& rows) { + std::vector flat; + auto offsets_buffer = BuildOffsetsBuffer(rows, &flat); + auto elements = paimon::test::VariantTestData::BuildVariantBatch( + list_struct_schema_->field(0), list_struct_variant_field_, flat, pool_); + EXPECT_TRUE(elements.ok()) << elements.status().ToString(); + arrow::StringBuilder sibling_builder; + for (size_t i = 0; i < flat.size(); ++i) { + EXPECT_TRUE(sibling_builder.Append(NestedSiblingValue(i)).ok()); + } + std::shared_ptr sibling; + EXPECT_TRUE(sibling_builder.Finish(&sibling).ok()); + auto element_data = arrow::ArrayData::Make( + list_struct_schema_->field(1)->type()->field(0)->type(), + static_cast(flat.size()), {nullptr}, + {elements.value()->field(1)->data(), sibling->data()}, /*null_count=*/0); + auto list_data = arrow::ArrayData::Make( + list_struct_schema_->field(1)->type(), static_cast(rows.size()), + {nullptr, offsets_buffer}, {element_data}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make( + arrow::struct_(list_struct_schema_->fields()), static_cast(rows.size()), + {nullptr}, {BuildIdColumn(rows.size()), list_data}, /*null_count=*/0); + return std::make_shared(batch_data); + } + + // Writes `rows` into the unshredded `arr2: list>` column. + void WriteListStructFile(const std::vector>& rows) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildListStructArray(rows), arrow_array.get()).ok()); + WriteFile(list_struct_schema_, arrow_array.get()); + } + + // Builds a `[id, s: struct]` batch holding the variant encodings of `jsons`. + std::shared_ptr BuildNestedArray(const std::vector& jsons) { + auto batch = paimon::test::VariantTestData::BuildVariantBatch( + nested_schema_->field(0), nested_variant_field_, jsons, pool_); + EXPECT_TRUE(batch.ok()) << batch.status().ToString(); + arrow::StringBuilder sibling_builder; + for (size_t i = 0; i < jsons.size(); ++i) { + EXPECT_TRUE(sibling_builder.Append(NestedSiblingValue(i)).ok()); + } + std::shared_ptr sibling; + EXPECT_TRUE(sibling_builder.Finish(&sibling).ok()); + auto struct_column = arrow::StructArray::Make( + {batch.value()->field(1), sibling}, {nested_variant_field_, nested_sibling_field_}); + EXPECT_TRUE(struct_column.ok()) << struct_column.status().ToString(); + auto nested = + arrow::StructArray::Make({batch.value()->field(0), struct_column.ValueOrDie()}, + {nested_schema_->field(0), nested_schema_->field(1)}); + EXPECT_TRUE(nested.ok()) << nested.status().ToString(); + return nested.ValueOrDie(); + } + + // Writes `jsons` into `s.nv` unshredded. + void WriteNestedFile(const std::vector& jsons) { + auto arrow_array = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildNestedArray(jsons), arrow_array.get()).ok()); + WriteFile(nested_schema_, arrow_array.get()); + } + + // Writes `jsons` into `s.nv` shredded by `shredding_type` (the nested variant is addressed by + // its field-index path `{1, 0}`). + void WriteShreddedNestedFile(const std::vector& jsons, + const std::shared_ptr& shredding_type) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, + VariantShreddingWritePlan::CreateFromPaths( + nested_schema_, {{std::vector{1, 0}, shredding_type}})); + ASSERT_NE(plan, nullptr); + ASSERT_OK_AND_ASSIGN(std::shared_ptr converter, + VariantShreddingBatchConverter::Create(plan, pool_)); + auto c_logical = std::make_unique(); + ASSERT_TRUE(arrow::ExportArray(*BuildNestedArray(jsons), c_logical.get()).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr c_physical, + converter->Convert(c_logical.get())); + WriteFile(converter->GetPhysicalSchema(), c_physical.get()); + } + + // Builds the read schema projecting `s.nv` as the given variant-access projection, keeping + // the plain sibling `s.t`. + std::shared_ptr BuildNestedAccessReadSchema( + const std::vector, std::string>>& accesses) { + auto access_field = BuildAccessField(accesses, "nv"); + auto read_struct = nested_schema_->field(1)->WithType( + arrow::struct_({access_field, nested_sibling_field_})); + return arrow::schema({nested_schema_->field(0), read_struct}); + } + + // Asserts that the plain sibling column of `s` round-tripped unchanged. + void ExpectNestedSibling(const std::shared_ptr& s_column) { + const auto& sibling = static_cast(*s_column->field(1)); + for (int64_t i = 0; i < s_column->length(); ++i) { + EXPECT_EQ(sibling.GetString(i), NestedSiblingValue(static_cast(i))); + } + } + + // Opens the written file and returns the reader plus its imported file schema. + void OpenFile(std::unique_ptr* file_reader, + std::shared_ptr* file_schema) { + ASSERT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_path_)); + auto length = fs_->GetFileStatus(file_path_).value()->GetLen(); + auto in_stream = + std::make_unique(std::move(input_stream), arrow_pool_, length); + std::map options = {}; + ASSERT_OK_AND_ASSIGN(auto parquet_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, + /*batch_size=*/1024, + /*file_metadata=*/nullptr, arrow_pool_)); + *file_reader = std::move(parquet_reader); + ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> c_file_schema, + (*file_reader)->GetFileSchema()); + auto imported = arrow::ImportSchema(c_file_schema.get()); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + *file_schema = imported.ValueOrDie(); + } + + // Builds a variant-access projection field for a variant column via the public builder. + std::shared_ptr BuildAccessField( + const std::vector, std::string>>& accesses, + const std::string& field_name = "v") { + VariantAccessBuilder builder; + for (const auto& [type, path] : accesses) { + auto c_target = std::make_unique(); + EXPECT_TRUE(arrow::ExportField(arrow::Field("t", type), c_target.get()).ok()); + EXPECT_OK(builder.AddField(c_target.get(), path, /*fail_on_error=*/false)); + } + auto c_field = builder.Build(field_name); + EXPECT_TRUE(c_field.ok()) << c_field.status().ToString(); + auto imported = arrow::ImportField(c_field.value().get()); + EXPECT_TRUE(imported.ok()) << imported.status().ToString(); + return imported.ValueOrDie(); + } + + // Reads the whole file through the shredding reader with the given read schema and returns + // the second (variant) column. + void ReadColumn(const std::shared_ptr& read_schema, + std::shared_ptr* column) { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_EQ(plans.size(), 1); + auto shredding_reader = + std::make_unique(std::move(file_reader), std::move(plans), pool_); + auto c_read_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok()); + ASSERT_OK(shredding_reader->SetReadSchema(c_read_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto batch_with_bitmap, shredding_reader->NextBatchWithBitmap()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch_with_bitmap)); + auto& [read_batch, bitmap] = batch_with_bitmap; + auto imported = arrow::ImportArray(read_batch.first.get(), read_batch.second.get()); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + auto result_struct = std::static_pointer_cast(imported.ValueOrDie()); + *column = result_struct->field(1); + shredding_reader->Close(); + // The assembled arrays borrow the reader's memory pool; keep the reader alive until the + // fixture is torn down (fixture members outlive test-body locals). + live_readers_.push_back(std::move(shredding_reader)); + } + + // `ReadColumn` for the cases whose second column is a struct. + void ReadVariantColumn(const std::shared_ptr& read_schema, + std::shared_ptr* v_column) { + std::shared_ptr column; + ReadColumn(read_schema, &column); + ASSERT_EQ(column->type_id(), arrow::Type::STRUCT); + *v_column = std::static_pointer_cast(column); + } + + protected: + std::unique_ptr dir_; + std::shared_ptr fs_; + std::shared_ptr pool_; + std::shared_ptr arrow_pool_; + std::string file_path_; + std::shared_ptr paimon_schema_; + std::shared_ptr nested_schema_; + std::shared_ptr nested_variant_field_; + std::shared_ptr nested_sibling_field_; + std::shared_ptr list_schema_; + std::shared_ptr list_element_field_; + std::shared_ptr map_schema_; + std::shared_ptr map_item_field_; + std::shared_ptr list_struct_schema_; + std::shared_ptr list_struct_variant_field_; + std::vector> live_readers_; +}; + +namespace { + +constexpr const char* kAgeCityShreddingSchema = R"({ + "type": "ROW", + "fields": [ { + "id": 0, + "name": "v", + "type": { + "type": "ROW", + "fields": [ + {"id": 1, "name": "age", "type": "INT"}, + {"id": 2, "name": "city", "type": "STRING"} + ] + } + } ] +})"; + +} // namespace + +TEST_F(VariantParquetTest, PhysicalLayoutMatchesJava) { + auto array = BuildArray({R"({"a": 1, "b": "hello"})", nullptr, "[1,2,3]"}); + WriteFile(array); + + // The on-disk layout must match the Java ParquetSchemaConverter: an (unannotated) group + // with two REQUIRED BINARY fields `value` (id 0) and `metadata` (id 1). The raw parquet + // reader is required because these parquet-level properties (repetition, physical types, + // field ids, the absence of a logical-type annotation) are not visible in the Arrow schema + // surfaced by the paimon reader. + auto file = arrow::io::ReadableFile::Open(file_path_, arrow_pool_.get()); + ASSERT_TRUE(file.ok()); + std::unique_ptr<::parquet::arrow::FileReader> reader; + auto status = ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &reader); + ASSERT_TRUE(status.ok()) << status.ToString(); + const ::parquet::SchemaDescriptor* schema = reader->parquet_reader()->metadata()->schema(); + ASSERT_EQ(schema->num_columns(), 3); + const auto* root = schema->group_node(); + ASSERT_EQ(root->field_count(), 2); + const auto& variant_group_node = root->field(1); + ASSERT_TRUE(variant_group_node->is_group()); + ASSERT_EQ(variant_group_node->name(), "v"); + ASSERT_EQ(variant_group_node->field_id(), 2); + ASSERT_EQ(variant_group_node->logical_type()->type(), ::parquet::LogicalType::Type::NONE); + const auto* variant_group = + static_cast(variant_group_node.get()); + ASSERT_EQ(variant_group->field_count(), 2); + const auto& value_node = variant_group->field(0); + ASSERT_EQ(value_node->name(), "value"); + ASSERT_TRUE(value_node->is_primitive()); + ASSERT_TRUE(value_node->is_required()); + ASSERT_EQ(value_node->field_id(), 0); + ASSERT_EQ( + static_cast(value_node.get())->physical_type(), + ::parquet::Type::BYTE_ARRAY); + const auto& metadata_node = variant_group->field(1); + ASSERT_EQ(metadata_node->name(), "metadata"); + ASSERT_TRUE(metadata_node->is_primitive()); + ASSERT_TRUE(metadata_node->is_required()); + ASSERT_EQ(metadata_node->field_id(), 1); +} + +TEST_F(VariantParquetTest, WriteAndReadRoundTrip) { + std::vector jsons = { + R"({"a": 1, "b": "hello"})", + nullptr, + "[1,2,3]", + "{\"nested\": {\"x\": [true, null, 1.5]}, \"s\": \"中文\"}", + "12345678901234", + "100.99", + }; + auto array = BuildArray(jsons); + WriteFile(array); + + { + // Sanity-check the raw file through the plain parquet-arrow reader: the struct child + // arrays must align with the logical rows. + auto file = arrow::io::ReadableFile::Open(file_path_, arrow_pool_.get()); + ASSERT_TRUE(file.ok()); + std::unique_ptr<::parquet::arrow::FileReader> raw_reader; + ASSERT_TRUE( + ::parquet::arrow::OpenFile(file.ValueOrDie(), arrow_pool_.get(), &raw_reader).ok()); + std::shared_ptr table; + ASSERT_TRUE(raw_reader->ReadTable(&table).ok()); + auto raw_variant = std::static_pointer_cast(table->column(1)->chunk(0)); + auto raw_value = std::static_pointer_cast(raw_variant->field(0)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("raw row " + std::to_string(i)); + if (jsons[i] != nullptr) { + ASSERT_FALSE(raw_variant->IsNull(i)); + ASSERT_GT(raw_value->GetView(i).size(), 0); + } else { + ASSERT_TRUE(raw_variant->IsNull(i)); + } + } + } + + ASSERT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_path_)); + auto length = fs_->GetFileStatus(file_path_).value()->GetLen(); + auto in_stream = + std::make_unique(std::move(input_stream), arrow_pool_, length); + std::map options = {}; + ASSERT_OK_AND_ASSIGN(auto batch_reader, + ParquetFileBatchReader::Create(std::move(in_stream), options, + /*batch_size=*/1024, + /*file_metadata=*/nullptr, arrow_pool_)); + auto c_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*paimon_schema_, c_schema.get()).ok()); + ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto result_chunked, + paimon::test::ReadResultCollector::CollectResult(batch_reader.get())); + batch_reader->Close(); + ASSERT_EQ(result_chunked->length(), static_cast(jsons.size())); + ASSERT_EQ(result_chunked->num_chunks(), 1); + auto result_struct = std::static_pointer_cast(result_chunked->chunk(0)); + + auto variant_column = std::static_pointer_cast(result_struct->field(1)); + ASSERT_EQ(variant_column->length(), static_cast(jsons.size())); + ASSERT_EQ(variant_column->field(0)->length(), variant_column->length()); + auto value_column = std::static_pointer_cast(variant_column->field(0)); + auto metadata_column = std::static_pointer_cast(variant_column->field(1)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + if (jsons[i] == nullptr) { + ASSERT_TRUE(variant_column->IsNull(i)); + continue; + } + ASSERT_FALSE(variant_column->IsNull(i)); + auto value_view = value_column->GetView(i); + auto metadata_view = metadata_column->GetView(i); + SCOPED_TRACE("value size " + std::to_string(value_view.size()) + ", metadata size " + + std::to_string(metadata_view.size())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value_view, metadata_view, pool_)); + ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(jsons[i], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, expected_json); + } +} + +TEST_F(VariantParquetTest, ShreddedWriteAndReadRoundTrip) { + std::vector jsons = { + R"({"age": 35, "city": "Hangzhou"})", + nullptr, + R"({"age": "not a number", "extra": [1, 2]})", + "[\"top level array\"]", + }; + WriteShreddedFile(jsons, kAgeCityShreddingSchema); + + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + auto file_variant_field = file_schema->GetFieldByName("v"); + ASSERT_NE(file_variant_field, nullptr); + ASSERT_TRUE(VariantShreddingUtils::IsShreddedFileType(file_variant_field->type())) + << file_variant_field->type()->ToString(); + file_reader->Close(); + } + + // Reading the column as a plain VARIANT reassembles every physical shape back to the + // original logical value. + std::shared_ptr variant_column; + ReadVariantColumn(paimon_schema_, &variant_column); + ASSERT_EQ(variant_column->length(), static_cast(jsons.size())); + auto value_column = std::static_pointer_cast(variant_column->field(0)); + auto metadata_column = std::static_pointer_cast(variant_column->field(1)); + for (size_t i = 0; i < jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + if (jsons[i] == nullptr) { + ASSERT_TRUE(variant_column->IsNull(i)); + continue; + } + ASSERT_FALSE(variant_column->IsNull(i)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(i), metadata_column->GetView(i), pool_)); + ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(jsons[i], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, expected_json); + } +} + +TEST_F(VariantParquetTest, VariantAccessReadMixedTypedAndBinary) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + WriteShreddedFile(jsons, kAgeCityShreddingSchema); + + auto access_field = BuildAccessField( + {{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}, {arrow::utf8(), "$.missing"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + + // The plan prunes `typed_value` to the requested keys and keeps `value` because `$.other` + // and `$.missing` are not shredded. + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_EQ(plans.size(), 1); + const auto& physical_type = + static_cast(*plans.at("v")->PhysicalField()->type()); + ASSERT_NE(physical_type.GetFieldByName(VariantDefs::kMetadataFieldName), nullptr); + ASSERT_NE(physical_type.GetFieldByName(VariantDefs::kValueFieldName), nullptr); + auto typed_value = physical_type.GetFieldByName(VariantDefs::kTypedValueFieldName); + ASSERT_NE(typed_value, nullptr); + const auto& typed_struct = static_cast(*typed_value->type()); + ASSERT_EQ(typed_struct.num_fields(), 1); + ASSERT_NE(typed_struct.GetFieldByName("age"), nullptr); + file_reader->Close(); + } + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + ASSERT_EQ(v_column->length(), 3); + const auto& age = static_cast(*v_column->field(0)); + const auto& other = static_cast(*v_column->field(1)); + const auto& missing = static_cast(*v_column->field(2)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(v_column->IsNull(2)); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); + ASSERT_TRUE(missing.IsNull(0)); + ASSERT_TRUE(missing.IsNull(1)); +} + +TEST_F(VariantParquetTest, VariantAccessReadTypedOnlyPrunesValue) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})"}; + WriteShreddedFile(jsons, kAgeCityShreddingSchema); + + auto access_field = BuildAccessField({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.city"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + + // All requested keys are shredded: neither `value` nor the unrequested typed keys are read. + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + const auto& physical_type = + static_cast(*plans.at("v")->PhysicalField()->type()); + ASSERT_EQ(physical_type.GetFieldByName(VariantDefs::kValueFieldName), nullptr); + auto typed_value = physical_type.GetFieldByName(VariantDefs::kTypedValueFieldName); + ASSERT_NE(typed_value, nullptr); + ASSERT_EQ(typed_value->type()->num_fields(), 2); + file_reader->Close(); + } + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + const auto& age = static_cast(*v_column->field(0)); + const auto& city = static_cast(*v_column->field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_EQ(city.GetString(0), "Chicago"); + // Row 1 has no "city" key: the shredded field is missing, which reads as null. + ASSERT_TRUE(city.IsNull(1)); +} + +TEST_F(VariantParquetTest, VariantAccessReadUnshreddedFile) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + WriteFile(BuildArray(jsons)); + + auto access_field = BuildAccessField({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + ASSERT_EQ(v_column->length(), 3); + const auto& age = static_cast(*v_column->field(0)); + const auto& other = static_cast(*v_column->field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(v_column->IsNull(2)); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); +} + +TEST_F(VariantParquetTest, VariantAccessReadSemicolonKey) { + // Object keys may contain the description delimiter; the description parser anchors on the + // trailing failOnError/timeZoneId tokens instead of splitting on every delimiter. + std::vector jsons = {R"({"a;b": 7})"}; + WriteFile(BuildArray(jsons)); + auto access_field = BuildAccessField({{arrow::int64(), "$['a;b']"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + ASSERT_EQ(static_cast(*v_column->field(0)).Value(0), 7); +} + +TEST_F(VariantParquetTest, VariantAccessReadVariantTarget) { + std::vector jsons = {R"({"user": {"name": "Paimon", "age": 1}})", + R"({"user": "flat"})"}; + WriteFile(BuildArray(jsons)); + + // A variant-marked target re-encodes the extracted sub-variant instead of casting it to a + // plain struct; the marker on the target field must survive AddField. + VariantAccessBuilder builder; + ASSERT_OK_AND_ASSIGN(auto variant_target, Variant::ArrowField("t")); + ASSERT_OK(builder.AddField(variant_target.get(), "$.user")); + ASSERT_OK_AND_ASSIGN(auto c_access_field, builder.Build("v")); + auto imported = arrow::ImportField(c_access_field.get()); + ASSERT_TRUE(imported.ok()) << imported.status().ToString(); + auto read_schema = arrow::schema({paimon_schema_->field(0), imported.ValueOrDie()}); + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + const auto& user = static_cast(*v_column->field(0)); + const auto& value_column = static_cast(*user.field(0)); + const auto& metadata_column = static_cast(*user.field(1)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr row0, + GenericVariant::Create(value_column.GetView(0), metadata_column.GetView(0), pool_)); + ASSERT_OK_AND_ASSIGN(std::string row0_json, row0->ToJson()); + ASSERT_EQ(row0_json, R"({"age":1,"name":"Paimon"})"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr row1, + GenericVariant::Create(value_column.GetView(1), metadata_column.GetView(1), pool_)); + ASSERT_OK_AND_ASSIGN(std::string row1_json, row1->ToJson()); + ASSERT_EQ(row1_json, R"("flat")"); +} + +TEST_F(VariantParquetTest, VariantAccessReadNestedPath) { + const char* nested_shredding_schema = R"({ + "type": "ROW", + "fields": [ { + "id": 0, + "name": "v", + "type": { + "type": "ROW", + "fields": [ { + "id": 1, + "name": "address", + "type": { + "type": "ROW", + "fields": [ {"id": 2, "name": "city", "type": "STRING"} ] + } + } ] + } + } ] + })"; + std::vector jsons = {R"({"address": {"city": "Hangzhou"}})", + R"({"address": "oops"})", R"({"address": {"zip": 12345}})"}; + WriteShreddedFile(jsons, nested_shredding_schema); + + auto access_field = BuildAccessField({{arrow::utf8(), "$.address.city"}}); + auto read_schema = arrow::schema({paimon_schema_->field(0), access_field}); + + std::shared_ptr v_column; + ReadVariantColumn(read_schema, &v_column); + const auto& city = static_cast(*v_column->field(0)); + ASSERT_EQ(city.GetString(0), "Hangzhou"); + // Row 1's address is not an object; row 2's address has no "city" key. + ASSERT_TRUE(city.IsNull(1)); + ASSERT_TRUE(city.IsNull(2)); +} + +TEST_F(VariantParquetTest, NestedVariantPlainReadReassembles) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", nullptr}; + WriteShreddedNestedFile(jsons, arrow::struct_({arrow::field("age", arrow::int32()), + arrow::field("city", arrow::utf8())})); + + // Read as a plain nested VARIANT: the shredded sub-columns are reassembled back into + // `struct`. + std::shared_ptr s_column; + ReadVariantColumn(nested_schema_, &s_column); + const auto& nv = static_cast(*s_column->field(0)); + ASSERT_TRUE(nv.type()->Equals(*nested_variant_field_->type())); + const auto& value_column = static_cast(*nv.field(0)); + const auto& metadata_column = static_cast(*nv.field(1)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr row0, + GenericVariant::Create(value_column.GetView(0), metadata_column.GetView(0), pool_)); + ASSERT_OK_AND_ASSIGN(std::string row0_json, row0->ToJson()); + ASSERT_EQ(row0_json, R"({"age":35,"city":"Chicago"})"); + ASSERT_TRUE(nv.IsNull(1)); + ExpectNestedSibling(s_column); +} + +TEST_F(VariantParquetTest, NestedVariantAccessReadShreddedFile) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + WriteShreddedNestedFile(jsons, arrow::struct_({arrow::field("age", arrow::int32()), + arrow::field("city", arrow::utf8())})); + + auto read_schema = + BuildNestedAccessReadSchema({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}); + + // A nested projection prunes the scan the same way a top-level one does: `typed_value` is + // narrowed to `age`, and `value` is kept because `$.other` is not shredded. + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_EQ(plans.size(), 1); + const auto& physical_struct = + static_cast(*plans.at("s")->PhysicalField()->type()); + auto physical_nv = physical_struct.GetFieldByName("nv"); + ASSERT_NE(physical_nv, nullptr); + const auto& physical_nv_type = static_cast(*physical_nv->type()); + ASSERT_NE(physical_nv_type.GetFieldByName(VariantDefs::kMetadataFieldName), nullptr); + ASSERT_NE(physical_nv_type.GetFieldByName(VariantDefs::kValueFieldName), nullptr); + auto typed_value = physical_nv_type.GetFieldByName(VariantDefs::kTypedValueFieldName); + ASSERT_NE(typed_value, nullptr); + const auto& typed_struct = static_cast(*typed_value->type()); + ASSERT_EQ(typed_struct.num_fields(), 1); + ASSERT_NE(typed_struct.GetFieldByName("age"), nullptr); + file_reader->Close(); + } + + std::shared_ptr s_column; + ReadVariantColumn(read_schema, &s_column); + ASSERT_EQ(s_column->length(), 3); + const auto& nv = static_cast(*s_column->field(0)); + const auto& age = static_cast(*nv.field(0)); + const auto& other = static_cast(*nv.field(1)); + // The extracted values must match the requested access struct, not `struct`. + ASSERT_EQ(nv.num_fields(), 2); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); + // Row 2's nested variant is null, so the whole projection is null. + ASSERT_TRUE(nv.IsNull(2)); + ExpectNestedSibling(s_column); +} + +TEST_F(VariantParquetTest, NestedVariantAccessReadUnshreddedFile) { + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + WriteNestedFile(jsons); + + // The nested column is stored unshredded: the paths are extracted from the `value` binary, + // which still requires a read plan (the projection type must never reach the format reader). + auto read_schema = + BuildNestedAccessReadSchema({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}); + std::shared_ptr s_column; + ReadVariantColumn(read_schema, &s_column); + ASSERT_EQ(s_column->length(), 3); + const auto& nv = static_cast(*s_column->field(0)); + const auto& age = static_cast(*nv.field(0)); + const auto& other = static_cast(*nv.field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); + ASSERT_TRUE(nv.IsNull(2)); + ExpectNestedSibling(s_column); +} + +TEST_F(VariantParquetTest, ListVariantPlainReadNeedsNoPlan) { + // A variant inside a repeated group is never shredded, so a plain read of it still needs no + // plan at all: the logical type is exactly what the file stores. + WriteListFile({{R"({"x": 1})", R"({"x": 2})"}, {R"({"x": 3})"}}); + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + list_schema_, file_schema, pool_)); + ASSERT_TRUE(plans.empty()); + file_reader->Close(); +} + +TEST_F(VariantParquetTest, ListVariantAccessRead) { + // A variant-access projection inside an ARRAY extracts the paths per element (as in Java's + // testReadNestedVariantInArray). The empty row and the null element cover the offsets and + // the element validity being carried over by the reassembly. + WriteListFile( + {{R"({"x": 1, "y": 2})", R"({"x": 3, "y": 4})"}, {}, {R"({"x": 5, "y": 6})", nullptr}}); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "element"); + auto read_schema = arrow::schema( + {list_schema_->field(0), list_schema_->field(1)->WithType(arrow::list(access_field))}); + + // The parquet reader rejects partial projection inside a repeated group, so the whole file + // subtree is pushed down and only reassembled back. + { + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_EQ(plans.size(), 1); + ASSERT_TRUE(plans.at("arr")->PhysicalField()->type()->Equals( + *file_schema->GetFieldByName("arr")->type())); + file_reader->Close(); + } + + std::shared_ptr arr_column; + ReadColumn(read_schema, &arr_column); + ASSERT_NE(arr_column, nullptr); + const auto& list = static_cast(*arr_column); + ASSERT_EQ(list.length(), 3); + ASSERT_EQ(list.value_length(0), 2); + ASSERT_EQ(list.value_length(1), 0); + ASSERT_EQ(list.value_length(2), 2); + const auto& elements = static_cast(*list.values()); + const auto& x = static_cast(*elements.field(0)); + ASSERT_EQ(x.Value(list.value_offset(0)), 1); + ASSERT_EQ(x.Value(list.value_offset(0) + 1), 3); + ASSERT_EQ(x.Value(list.value_offset(2)), 5); + // The null element's whole projection is null. + ASSERT_TRUE(elements.IsNull(list.value_offset(2) + 1)); +} + +TEST_F(VariantParquetTest, ListOfStructVariantAccessRead) { + // A variant one struct level below the ARRAY: the plan descends list -> struct -> variant + // and must leave the struct's plain sibling untouched. + WriteListStructFile({{R"({"x": 1, "y": 2})", R"({"x": 3})"}, {R"({"x": 5})"}}); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "v"); + auto element_field = + arrow::field("element", arrow::struct_({access_field, nested_sibling_field_})); + auto read_schema = + arrow::schema({list_struct_schema_->field(0), + list_struct_schema_->field(1)->WithType(arrow::list(element_field))}); + + std::shared_ptr arr_column; + ReadColumn(read_schema, &arr_column); + ASSERT_NE(arr_column, nullptr); + const auto& list = static_cast(*arr_column); + ASSERT_EQ(list.length(), 2); + const auto& elements = static_cast(*list.values()); + // Each element is `struct, t: STRING>`, so the extracted path sits one + // struct level below the element. + const auto& access = static_cast(*elements.field(0)); + const auto& x = static_cast(*access.field(0)); + const auto& sibling = static_cast(*elements.field(1)); + ASSERT_EQ(x.Value(0), 1); + ASSERT_EQ(x.Value(1), 3); + ASSERT_EQ(x.Value(2), 5); + for (int64_t i = 0; i < 3; ++i) { + EXPECT_EQ(sibling.GetString(i), NestedSiblingValue(static_cast(i))); + } +} + +TEST_F(VariantParquetTest, ListPartialProjectionNeedsNoPlan) { + // Projecting a subset of a struct inside a repeated group is unsupported, so no plan is + // built: the read fails in the reader instead of assembling a mistyped column. + WriteListStructFile({{R"({"x": 1})"}}); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "v"); + auto element_field = arrow::field("element", arrow::struct_({access_field})); + auto read_schema = + arrow::schema({list_struct_schema_->field(0), + list_struct_schema_->field(1)->WithType(arrow::list(element_field))}); + + std::unique_ptr file_reader; + std::shared_ptr file_schema; + OpenFile(&file_reader, &file_schema); + ASSERT_OK_AND_ASSIGN(auto plans, VariantShreddingReadPlanFactory::CreateReadPlans( + read_schema, file_schema, pool_)); + ASSERT_TRUE(plans.empty()); + file_reader->Close(); +} + +TEST_F(VariantParquetTest, MapVariantAccessRead) { + // The same for a variant value inside a MAP: the plan descends through the map entries + // struct and rewrites only the item child. + WriteMapFile({{"a", "b"}, {"c", "d"}}, + {{R"({"x": 1, "y": 2})", R"({"x": 3})"}, {R"({"x": 5, "y": 6})", nullptr}}); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "value"); + auto read_schema = + arrow::schema({map_schema_->field(0), + map_schema_->field(1)->WithType(arrow::map(arrow::utf8(), access_field))}); + + std::shared_ptr m_column; + ReadColumn(read_schema, &m_column); + ASSERT_NE(m_column, nullptr); + const auto& map = static_cast(*m_column); + ASSERT_EQ(map.length(), 2); + ASSERT_EQ(map.value_length(0), 2); + ASSERT_EQ(map.value_length(1), 2); + // The keys must survive untouched next to the rewritten values. + const auto& keys = static_cast(*map.keys()); + ASSERT_EQ(keys.GetString(map.value_offset(0)), "a"); + ASSERT_EQ(keys.GetString(map.value_offset(0) + 1), "b"); + ASSERT_EQ(keys.GetString(map.value_offset(1)), "c"); + ASSERT_EQ(keys.GetString(map.value_offset(1) + 1), "d"); + const auto& items = static_cast(*map.items()); + const auto& x = static_cast(*items.field(0)); + ASSERT_EQ(x.Value(map.value_offset(0)), 1); + ASSERT_EQ(x.Value(map.value_offset(0) + 1), 3); + ASSERT_EQ(x.Value(map.value_offset(1)), 5); + // A null map value projects to a null row. + ASSERT_TRUE(items.IsNull(map.value_offset(1) + 1)); +} + +} // namespace paimon::parquet::test diff --git a/src/paimon/testing/utils/test_helper.h b/src/paimon/testing/utils/test_helper.h index 5489d15f..429c3be9 100644 --- a/src/paimon/testing/utils/test_helper.h +++ b/src/paimon/testing/utils/test_helper.h @@ -25,6 +25,7 @@ #include #include "arrow/api.h" +#include "arrow/array/concatenate.h" #include "arrow/c/bridge.h" #include "arrow/ipc/api.h" #include "paimon/api.h" @@ -259,6 +260,40 @@ class TestHelper { return result_blobs; } + /// Reads all rows of the given splits and returns the raw result (including the leading + /// `_VALUE_KIND` column). Useful when the expected data cannot be expressed as JSON, e.g. + /// binary-encoded VARIANT columns. + Result> ReadResult( + const std::vector>& splits) { + return ReadResult(splits, /*read_schema=*/nullptr); + } + + /// Reads all rows of the given splits with an optional projected read schema. + Result> ReadResult( + const std::vector>& splits, + std::unique_ptr<::ArrowSchema> read_schema) { + ReadContextBuilder read_context_builder(table_path_); + read_context_builder.SetOptions(options_); + if (read_schema != nullptr) { + read_context_builder.SetReadSchema(std::move(read_schema)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(auto table_read, TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(auto batch_reader, table_read->CreateReader(splits)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr collected, + ReadResultCollector::CollectResult(batch_reader.get())); + if (collected->num_chunks() == 0) { + return collected; + } + // The collected batches borrow reader-owned buffers; copy them into the process pool + // while the reader is still alive so the returned result may outlive it. + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr copied, + arrow::Concatenate(collected->chunks(), arrow::default_memory_pool())); + return std::make_shared(copied); + } + Result ReadAndCheckResult(const std::shared_ptr& data_type, const std::vector>& splits, const std::string& expected_result) { diff --git a/src/paimon/testing/utils/variant_test_data.h b/src/paimon/testing/utils/variant_test_data.h new file mode 100644 index 00000000..e78b0cb9 --- /dev/null +++ b/src/paimon/testing/utils/variant_test_data.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "arrow/api.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" + +namespace paimon::test { + +/// Helpers for building variant test data. +class VariantTestData { + public: + VariantTestData() = delete; + ~VariantTestData() = delete; + + /// Builds a `[id, v]` struct array where `v` holds the variant encodings of `jsons` (nullptr + /// means a null variant) and `id` is a running int32 starting at `id_offset`. + /// `variant_field` must be a variant-marked field (see `VariantTypeUtils::ToArrowField`). + static Result> BuildVariantBatch( + const std::shared_ptr& id_field, + const std::shared_ptr& variant_field, const std::vector& jsons, + const std::shared_ptr& pool, int32_t id_offset = 0); +}; + +inline Result> VariantTestData::BuildVariantBatch( + const std::shared_ptr& id_field, + const std::shared_ptr& variant_field, const std::vector& jsons, + const std::shared_ptr& pool, int32_t id_offset) { + arrow::Int32Builder id_builder; + auto value_builder = std::make_shared(); + auto metadata_builder = std::make_shared(); + arrow::StructBuilder variant_builder(variant_field->type(), arrow::default_memory_pool(), + {value_builder, metadata_builder}); + for (size_t i = 0; i < jsons.size(); ++i) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(id_builder.Append(id_offset + static_cast(i))); + if (jsons[i] == nullptr) { + // StructBuilder::AppendNull appends empty values to the child builders itself. + PAIMON_RETURN_NOT_OK_FROM_ARROW(variant_builder.AppendNull()); + } else { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant, + GenericVariant::FromJson(jsons[i], pool)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(variant_builder.Append()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(value_builder->Append(variant->RawValue())); + PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata_builder->Append(variant->Metadata())); + } + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr id_array, id_builder.Finish()); + std::shared_ptr variant_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(variant_builder.Finish(&variant_array)); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr result, + arrow::StructArray::Make({id_array, variant_array}, {id_field, variant_field})); + return result; +} + +} // namespace paimon::test diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt index 051b1030..a5b81e46 100644 --- a/test/inte/CMakeLists.txt +++ b/test/inte/CMakeLists.txt @@ -22,6 +22,13 @@ if(PAIMON_BUILD_TESTS) test_utils_static ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(variant_table_inte_test + STATIC_LINK_LIBS + paimon_shared + ${TEST_STATIC_LINK_LIBS} + test_utils_static + ${GTEST_LINK_TOOLCHAIN}) + add_paimon_test(data_evolution_table_test STATIC_LINK_LIBS paimon_shared diff --git a/test/inte/variant_table_inte_test.cpp b/test/inte/variant_table_inte_test.cpp new file mode 100644 index 00000000..ac9d157c --- /dev/null +++ b/test/inte/variant_table_inte_test.cpp @@ -0,0 +1,484 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/data/variant/generic_variant.h" +#include "paimon/common/data/variant/variant_type_utils.h" +#include "paimon/common/factories/io_hook.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/data/variant.h" +#include "paimon/defs.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/record_batch.h" +#include "paimon/table/source/startup_mode.h" +#include "paimon/testing/utils/io_exception_helper.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/test_helper.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/variant_test_data.h" + +namespace paimon::test { + +// End-to-end tests for tables with a VARIANT column: create, write, commit, scan and read. +class VariantTableInteTest : public ::testing::Test { + public: + void SetUp() override { + dir_ = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir_); + test_dir_ = dir_->Str(); + pool_ = GetDefaultPool(); + fields_ = {arrow::field("id", arrow::int32()), VariantTypeUtils::ToArrowField("v")}; + schema_ = arrow::schema(fields_); + } + + void TearDown() override { + dir_.reset(); + } + + std::shared_ptr BuildArray(const std::vector& jsons, + int32_t id_offset = 0) { + auto result = + VariantTestData::BuildVariantBatch(fields_[0], fields_[1], jsons, pool_, id_offset); + EXPECT_TRUE(result.ok()) << result.status().ToString(); + return std::move(result).value(); + } + + Result> MakeBatch( + const std::shared_ptr& array) { + ::ArrowArray arrow_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, &arrow_array)); + RecordBatchBuilder batch_builder(&arrow_array); + return batch_builder.SetPartition({}).SetBucket(0).SetRowKinds({}).Finish(); + } + + // Reads all rows back and checks the variant column renders to `expected_jsons` (nullptr + // means a null variant). The read result carries a leading `_VALUE_KIND` column. + void ReadAndCheck(TestHelper* helper, const std::vector>& splits, + const std::vector& expected_ids, + const std::vector& expected_jsons) { + ASSERT_OK_AND_ASSIGN(auto result, helper->ReadResult(splits)); + ASSERT_EQ(result->num_chunks(), 1); + auto result_struct = std::static_pointer_cast(result->chunk(0)); + ASSERT_EQ(result_struct->length(), static_cast(expected_jsons.size())); + auto struct_type = std::static_pointer_cast(result_struct->type()); + int32_t id_index = struct_type->GetFieldIndex("id"); + int32_t variant_index = struct_type->GetFieldIndex("v"); + ASSERT_GE(id_index, 0); + ASSERT_GE(variant_index, 0); + auto id_column = + std::static_pointer_cast(result_struct->field(id_index)); + auto variant_column = + std::static_pointer_cast(result_struct->field(variant_index)); + auto value_column = std::static_pointer_cast(variant_column->field(0)); + auto metadata_column = + std::static_pointer_cast(variant_column->field(1)); + for (size_t i = 0; i < expected_jsons.size(); ++i) { + SCOPED_TRACE("row " + std::to_string(i)); + ASSERT_EQ(id_column->Value(i), expected_ids[i]); + if (expected_jsons[i] == nullptr) { + ASSERT_TRUE(variant_column->IsNull(i)); + continue; + } + ASSERT_FALSE(variant_column->IsNull(i)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr variant, + GenericVariant::Create(value_column->GetView(i), + metadata_column->GetView(i), pool_)); + ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr expected, + GenericVariant::FromJson(expected_jsons[i], pool_)); + ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson()); + ASSERT_EQ(actual_json, expected_json); + } + } + + // Builds a variant-access projection field via the public builder. + std::shared_ptr BuildAccessField( + const std::vector, std::string>>& accesses, + const std::string& field_name) { + VariantAccessBuilder builder; + for (const auto& [type, path] : accesses) { + auto target = std::make_unique(); + EXPECT_TRUE(arrow::ExportField(arrow::Field("t", type), target.get()).ok()); + EXPECT_OK(builder.AddField(target.get(), path, /*fail_on_error=*/false)); + } + auto c_field = builder.Build(field_name); + EXPECT_TRUE(c_field.ok()) << c_field.status().ToString(); + auto imported = arrow::ImportField(c_field.value().get()); + EXPECT_TRUE(imported.ok()) << imported.status().ToString(); + return imported.ValueOrDie(); + } + + // Reads `splits` back with `read_schema` projected and returns the single result chunk. + void ReadWithSchema(TestHelper* helper, const std::vector>& splits, + const std::shared_ptr& read_schema, + std::shared_ptr* result_struct) { + auto c_read_schema = std::make_unique(); + ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_read_schema.get()).ok()); + ASSERT_OK_AND_ASSIGN(auto result, helper->ReadResult(splits, std::move(c_read_schema))); + ASSERT_EQ(result->num_chunks(), 1); + *result_struct = std::static_pointer_cast(result->chunk(0)); + } + + protected: + std::string test_dir_; + std::unique_ptr dir_; + std::shared_ptr pool_; + arrow::FieldVector fields_; + std::shared_ptr schema_; +}; + +TEST_F(VariantTableInteTest, TestAppendTable) { + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + // The document set covers deep object/array alternation, escaped and unicode strings, + // wide integers, decimals, exponent doubles and empty containers. + std::vector jsons = { + R"({"age": 35, "city": "Hangzhou"})", + nullptr, + "[1, \"two\", 3.5, null, true]", + "{\"nested\": {\"x\": [1, 2]}, \"s\": \"中文\"}", + R"({ + "user": { + "id": 9007199254740993, + "name": "张三 \"quoted\" \\ / \b\f\n\r\t", + "tags": ["a", 1, 2.5, true, null, {"deep": [[1, [2, [3, [4]]]]]}], + "address": { + "city": "Hangzhou", + "geo": {"lat": 30.274085, "lng": 120.15507, "alt": -1.5e-3}, + "history": [ + {"year": 2020, "city": "Beijing"}, + {"year": 2021, "city": "Shanghai", "note": null} + ] + }, + "balance": 12345678901234567890.123456789, + "scores": [0.1, -0.0, 1e100, -1e-100] + }, + "empty_object": {}, + "empty_array": [], + "flags": [true, false, null] + })", + R"([{"a": [{"b": {"c": [null, {"d": 1}]}}]}, [], {}, "end"])", + R"({"unicode": "😀"})", + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(BuildArray(jsons))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + ReadAndCheck(helper.get(), splits, {0, 1, 2, 3, 4, 5, 6}, jsons); +} + +TEST_F(VariantTableInteTest, TestPrimaryKeyTable) { + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch_1, + MakeBatch(BuildArray({"{\"a\": 1}", "{\"b\": 2}", nullptr}, /*id_offset=*/0))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_1, + helper->WriteAndCommit(std::move(batch_1), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch_2, + MakeBatch(BuildArray({"{\"b\": \"updated\"}", "[42]"}, /*id_offset=*/1))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs_2, + helper->WriteAndCommit(std::move(batch_2), /*commit_identifier=*/1, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt)); + // The second batch overwrites ids 1 and 2, so the merged view holds three rows. + ReadAndCheck(helper.get(), splits, {0, 1, 2}, {"{\"a\": 1}", R"({"b": "updated"})", "[42]"}); +} + +TEST_F(VariantTableInteTest, TestVariantAccessRead) { + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(BuildArray(jsons))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + + auto access_field = + BuildAccessField({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}, "v"); + auto read_schema = arrow::schema({fields_[0], access_field}); + std::shared_ptr result_struct; + ReadWithSchema(helper.get(), splits, read_schema, &result_struct); + ASSERT_EQ(result_struct->length(), 3); + auto struct_type = std::static_pointer_cast(result_struct->type()); + auto v_column = std::static_pointer_cast( + result_struct->field(struct_type->GetFieldIndex("v"))); + const auto& age = static_cast(*v_column->field(0)); + const auto& other = static_cast(*v_column->field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(v_column->IsNull(2)); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); +} + +// The two tests below read a variant nested inside a ROW and inside an ARRAY column as a +// variant-access projection. Unlike the format-level tests they go through the whole table read +// path, where the read schema is resolved against the table schema before the read plans see it. +TEST_F(VariantTableInteTest, TestNestedRowVariantAccessRead) { + // Table: [id, s: ROW] + auto struct_field = arrow::field("s", arrow::struct_({VariantTypeUtils::ToArrowField("nv"), + arrow::field("t", arrow::utf8())})); + auto table_schema = arrow::schema({fields_[0], struct_field}); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + std::vector jsons = {R"({"age": 35, "city": "Chicago"})", + R"({"age": 25, "other": "Hello"})", nullptr}; + auto variant_batch = BuildArray(jsons); + arrow::StringBuilder sibling_builder; + for (size_t i = 0; i < jsons.size(); ++i) { + ASSERT_TRUE(sibling_builder.Append("t" + std::to_string(i)).ok()); + } + std::shared_ptr sibling; + ASSERT_TRUE(sibling_builder.Finish(&sibling).ok()); + auto struct_data = arrow::ArrayData::Make( + struct_field->type(), static_cast(jsons.size()), {nullptr}, + {variant_batch->field(1)->data(), sibling->data()}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make( + arrow::struct_({fields_[0], struct_field}), static_cast(jsons.size()), {nullptr}, + {variant_batch->field(0)->data(), struct_data}, /*null_count=*/0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch(std::make_shared(batch_data))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + + auto access_field = + BuildAccessField({{arrow::int64(), "$.age"}, {arrow::utf8(), "$.other"}}, "nv"); + auto read_schema = arrow::schema( + {fields_[0], + struct_field->WithType(arrow::struct_({access_field, struct_field->type()->field(1)}))}); + std::shared_ptr result_struct; + ReadWithSchema(helper.get(), splits, read_schema, &result_struct); + + auto struct_type = std::static_pointer_cast(result_struct->type()); + auto s_column = std::static_pointer_cast( + result_struct->field(struct_type->GetFieldIndex("s"))); + const auto& nv = static_cast(*s_column->field(0)); + const auto& age = static_cast(*nv.field(0)); + const auto& other = static_cast(*nv.field(1)); + const auto& kept_sibling = static_cast(*s_column->field(1)); + ASSERT_EQ(age.Value(0), 35); + ASSERT_EQ(age.Value(1), 25); + ASSERT_TRUE(nv.IsNull(2)); + ASSERT_TRUE(other.IsNull(0)); + ASSERT_EQ(other.GetString(1), "Hello"); + for (size_t i = 0; i < jsons.size(); ++i) { + EXPECT_EQ(kept_sibling.GetString(static_cast(i)), "t" + std::to_string(i)); + } +} + +TEST_F(VariantTableInteTest, TestArrayVariantAccessRead) { + // Table: [id, arr: ARRAY] + auto list_field = arrow::field("arr", arrow::list(VariantTypeUtils::ToArrowField("element"))); + auto table_schema = arrow::schema({fields_[0], list_field}); + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN(auto helper, + TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/false)); + + // Row 0 holds two elements, row 1 is empty and row 2 holds one. + std::vector flat = {R"({"x": 1, "y": 2})", R"({"x": 3})", R"({"x": 5})"}; + std::vector offsets = {0, 2, 2, 3}; + arrow::Int32Builder offset_builder; + ASSERT_TRUE(offset_builder.AppendValues(offsets).ok()); + std::shared_ptr offset_array; + ASSERT_TRUE(offset_builder.Finish(&offset_array).ok()); + auto elements = BuildArray(flat); + arrow::Int32Builder id_builder; + ASSERT_TRUE(id_builder.AppendValues({0, 1, 2}).ok()); + std::shared_ptr ids; + ASSERT_TRUE(id_builder.Finish(&ids).ok()); + auto list_data = arrow::ArrayData::Make(list_field->type(), /*length=*/3, + {nullptr, offset_array->data()->buffers[1]}, + {elements->field(1)->data()}, /*null_count=*/0); + auto batch_data = arrow::ArrayData::Make(arrow::struct_({fields_[0], list_field}), /*length=*/3, + {nullptr}, {ids->data(), list_data}, /*null_count=*/0); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, + MakeBatch(std::make_shared(batch_data))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + + auto access_field = BuildAccessField({{arrow::int64(), "$.x"}}, "element"); + auto read_schema = arrow::schema({fields_[0], list_field->WithType(arrow::list(access_field))}); + std::shared_ptr result_struct; + ReadWithSchema(helper.get(), splits, read_schema, &result_struct); + + auto struct_type = std::static_pointer_cast(result_struct->type()); + const auto& list = static_cast( + *result_struct->field(struct_type->GetFieldIndex("arr"))); + ASSERT_EQ(list.length(), 3); + ASSERT_EQ(list.value_length(0), 2); + ASSERT_EQ(list.value_length(1), 0); + ASSERT_EQ(list.value_length(2), 1); + const auto& x = static_cast( + *static_cast(*list.values()).field(0)); + ASSERT_EQ(x.Value(list.value_offset(0)), 1); + ASSERT_EQ(x.Value(list.value_offset(0) + 1), 3); + ASSERT_EQ(x.Value(list.value_offset(2)), 5); +} + +TEST_F(VariantTableInteTest, TestReadWithIOException) { + // Injects an IO error at every position of the scan+read path and verifies each failure + // surfaces as a clean error status. + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + std::vector jsons = {R"({"age": 35, "city": "Hangzhou"})", nullptr, "[1, 2, 3]"}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(BuildArray(jsons))); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + bool run_complete = false; + auto io_hook = IOHook::GetInstance(); + for (size_t i = 0; i < 500; i++) { + ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); + io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + Result>> splits = + helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false); + CHECK_HOOK_STATUS(splits.status(), i); + Result> read_result = + helper->ReadResult(splits.value()); + CHECK_HOOK_STATUS(read_result.status(), i); + run_complete = true; + // All IO succeeded before the injected position was reached: check the data. + io_hook->Clear(); + ReadAndCheck(helper.get(), splits.value(), {0, 1, 2}, jsons); + break; + } + ASSERT_TRUE(run_complete); +} + +TEST_F(VariantTableInteTest, TestWriteWithIOException) { + // Injects an IO error at every position of the create+write+commit path (on a fresh table + // directory per attempt) and verifies each failure surfaces as a clean error status. + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "parquet"}, + {Options::BUCKET, "-1"}, + }; + std::vector jsons = {R"({"age": 35, "city": "Hangzhou"})", nullptr}; + bool run_complete = false; + auto io_hook = IOHook::GetInstance(); + for (size_t i = 0; i < 500; i++) { + std::string table_dir = test_dir_ + fmt::format("/io_exception_{}", i); + ScopeGuard guard([&io_hook]() { io_hook->Clear(); }); + io_hook->Reset(i, IOHook::Mode::RETURN_ERROR); + Result> helper = + TestHelper::Create(table_dir, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false); + CHECK_HOOK_STATUS(helper.status(), i); + Result> batch = MakeBatch(BuildArray(jsons)); + CHECK_HOOK_STATUS(batch.status(), i); + Result>> commit_msgs = + helper.value()->WriteAndCommit(std::move(batch).value(), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt); + CHECK_HOOK_STATUS(commit_msgs.status(), i); + run_complete = true; + // All IO succeeded before the injected position was reached: check the data. + io_hook->Clear(); + ASSERT_OK_AND_ASSIGN(std::vector> splits, + helper.value()->NewScan(StartupMode::LatestFull(), + /*snapshot_id=*/std::nullopt, + /*is_streaming=*/false)); + ReadAndCheck(helper.value().get(), splits, {0, 1}, jsons); + break; + } + ASSERT_TRUE(run_complete); +} + +TEST_F(VariantTableInteTest, TestOrcFormatRejected) { + std::map options = { + {Options::MANIFEST_FORMAT, "avro"}, + {Options::FILE_FORMAT, "orc"}, + {Options::BUCKET, "-1"}, + }; + ASSERT_OK_AND_ASSIGN( + auto helper, TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, + /*primary_keys=*/{}, options, /*is_streaming_mode=*/false)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(BuildArray({"{\"a\": 1}"}))); + auto result = helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt); + ASSERT_FALSE(result.ok()); +} + +} // namespace paimon::test From 7c0f3d1b93b2c93fe66553428d54b069f599333c Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:10:40 +0800 Subject: [PATCH 110/138] build: fetch Boost from OSS and define network test macro globally * build: fetch Boost from OSS * build: define network test macro globally --- CMakeLists.txt | 4 +--- LICENSE | 10 ++++++++++ NOTICE | 5 +++++ src/paimon/CMakeLists.txt | 4 ---- test/inte/CMakeLists.txt | 4 ---- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb6ee880..05047055 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,9 +76,7 @@ if(PAIMON_ENABLE_JINDO) add_definitions(-DPAIMON_ENABLE_JINDO) endif() if(PAIMON_ENABLE_NETWORK_TESTS) - if(NOT PAIMON_BUILD_TESTS) - message(FATAL_ERROR "PAIMON_ENABLE_NETWORK_TESTS requires PAIMON_BUILD_TESTS=ON") - endif() + add_definitions(-DPAIMON_ENABLE_NETWORK_TESTS) endif() if(PAIMON_USE_CXX11_ABI) add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1) diff --git a/LICENSE b/LICENSE index dd128552..b454b24c 100644 --- a/LICENSE +++ b/LICENSE @@ -405,6 +405,16 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This product includes code from LucenePlusPlus. + +* LucenePlusPlus utility in src/paimon/global_index/lucene/ directory + +Copyright: 2009-2014 Alan Wright. +Home page: https://github.com/luceneplusplus/LucenePlusPlus +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + This product includes code derived from PyTorch TH simd.h. * SIMD detection code in third_party/roaring_bitmap/roaring.cpp diff --git a/NOTICE b/NOTICE index ed0d3839..a0284350 100644 --- a/NOTICE +++ b/NOTICE @@ -45,5 +45,10 @@ PalDB -------------------------------------------------------------------------------- +This product includes software from LucenePlusPlus project (Apache 2.0) +Copyright 2009-2014 Alan Wright. + +-------------------------------------------------------------------------------- + JindoSDK NextArch C++ Copyright 2024-present Alibaba Cloud. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 0551cabc..9f78a69e 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -853,8 +853,4 @@ if(PAIMON_BUILD_TESTS) EXTRA_INCLUDES ${JINDOSDK_INCLUDE_DIR}) - if(PAIMON_ENABLE_NETWORK_TESTS) - target_compile_definitions(paimon-fs-test PRIVATE PAIMON_ENABLE_NETWORK_TESTS) - endif() - endif() diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt index a5b81e46..d50ae178 100644 --- a/test/inte/CMakeLists.txt +++ b/test/inte/CMakeLists.txt @@ -49,10 +49,6 @@ if(PAIMON_BUILD_TESTS) ${TEST_STATIC_LINK_LIBS} test_utils_static ${GTEST_LINK_TOOLCHAIN}) - if(PAIMON_ENABLE_NETWORK_TESTS) - target_compile_definitions(paimon-write-and-read-inte-test - PRIVATE PAIMON_ENABLE_NETWORK_TESTS) - endif() add_paimon_test(clean_inte_test STATIC_LINK_LIBS From 998e435e2ceac4bab22aa038e4fc014eba50e75f Mon Sep 17 00:00:00 2001 From: Yonghao Fang Date: Wed, 22 Jul 2026 15:42:48 +0800 Subject: [PATCH 111/138] feat: add metrics storage-read-bytes for parquet and fix clean inte test --- .../arrow/arrow_input_stream_adapter.cpp | 36 +++++++++---- .../utils/arrow/arrow_input_stream_adapter.h | 12 ++++- .../utils/arrow/arrow_stream_adapter_test.cpp | 2 +- src/paimon/core/mergetree/spill_reader.cpp | 2 +- .../core/operation/file_store_commit_impl.cpp | 50 ++++++------------- .../core/operation/file_store_commit_impl.h | 7 +-- .../operation/file_store_commit_impl_test.cpp | 17 +++---- .../parquet/column_index_filter_test.cpp | 2 +- .../parquet/file_reader_wrapper_test.cpp | 2 +- .../page_filtered_row_group_reader_test.cpp | 34 +++++++------ .../parquet/parquet_file_batch_reader.cpp | 8 ++- .../parquet/parquet_file_batch_reader.h | 10 +++- .../parquet_file_batch_reader_test.cpp | 30 ++++++----- .../format/parquet/parquet_format_defs.h | 2 + .../format/parquet/parquet_reader_builder.h | 6 ++- .../parquet/parquet_stats_extractor.cpp | 2 +- .../parquet/predicate_pushdown_test.cpp | 5 +- .../format/parquet/variant_parquet_test.cpp | 16 +++--- test/inte/clean_inte_test.cpp | 2 +- 19 files changed, 134 insertions(+), 111 deletions(-) diff --git a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp index 1dab4e48..01429df6 100644 --- a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp +++ b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.cpp @@ -46,9 +46,12 @@ arrow::Status ValidateArrowIoRange(int64_t value, const char* name) { } // namespace ArrowInputStreamAdapter::ArrowInputStreamAdapter( - const std::shared_ptr& input_stream, - const std::shared_ptr& pool, int64_t file_size) - : input_stream_(input_stream), pool_(pool), file_size_(file_size) { + const std::shared_ptr& input_stream, int64_t file_size, + const std::shared_ptr& pool) + : input_stream_(input_stream), + pool_(pool), + file_size_(file_size), + storage_read_bytes_(std::make_shared>(0)) { assert(file_size >= 0); } @@ -66,6 +69,9 @@ arrow::Result ArrowInputStreamAdapter::Read(int64_t nbytes, void* out) if (!read_bytes.ok()) { return ToArrowStatus(read_bytes.status()); } + if (storage_read_bytes_) { + storage_read_bytes_->fetch_add(static_cast(read_bytes.value())); + } return read_bytes.value(); } @@ -87,6 +93,9 @@ arrow::Result ArrowInputStreamAdapter::ReadAt(int64_t position, int64_t if (!read_bytes.ok()) { return ToArrowStatus(read_bytes.status()); } + if (storage_read_bytes_) { + storage_read_bytes_->fetch_add(static_cast(read_bytes.value())); + } return read_bytes.value(); } @@ -122,14 +131,19 @@ arrow::Future> ArrowInputStreamAdapter::ReadAsync return fut; } std::shared_ptr buffer = std::move(buffer_result).ValueUnsafe(); - input_stream_->ReadAsync(reinterpret_cast(buffer->mutable_data()), nbytes, position, - [fut, buffer](Status callback_status) mutable { - if (callback_status.ok()) { - fut.MarkFinished(std::move(buffer)); - } else { - fut.MarkFinished(ToArrowStatus(callback_status)); - } - }); + std::shared_ptr> storage_read_bytes = storage_read_bytes_; + input_stream_->ReadAsync( + reinterpret_cast(buffer->mutable_data()), nbytes, position, + [fut, buffer, storage_read_bytes, nbytes](Status callback_status) mutable { + if (callback_status.ok()) { + if (storage_read_bytes) { + storage_read_bytes->fetch_add(static_cast(nbytes)); + } + fut.MarkFinished(std::move(buffer)); + } else { + fut.MarkFinished(ToArrowStatus(callback_status)); + } + }); return fut; } diff --git a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h index 134568b2..1c649108 100644 --- a/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h +++ b/src/paimon/common/utils/arrow/arrow_input_stream_adapter.h @@ -19,6 +19,7 @@ #pragma once +#include #include #include @@ -33,7 +34,7 @@ class InputStream; class PAIMON_EXPORT ArrowInputStreamAdapter : public arrow::io::RandomAccessFile { public: ArrowInputStreamAdapter(const std::shared_ptr& input_stream, - const std::shared_ptr& pool, int64_t file_size); + int64_t file_size, const std::shared_ptr& pool); ~ArrowInputStreamAdapter() override; // NOTE: In paimon file system definition, position + nbytes should not exceed file_size_. @@ -52,12 +53,21 @@ class PAIMON_EXPORT ArrowInputStreamAdapter : public arrow::io::RandomAccessFile } bool closed() const override; + // Accumulated bytes read from the underlying stream (storageReadBytes). The counter is owned + // by this adapter and initialized to 0; callers may retain the returned shared_ptr to read the + // value after the adapter is closed or destroyed. + const std::shared_ptr>& StorageReadBytes() const { + return storage_read_bytes_; + } + private: arrow::Status DoClose(); std::shared_ptr input_stream_; std::shared_ptr pool_; int64_t file_size_; + // Accumulates the number of bytes read from the underlying stream (storageReadBytes). + std::shared_ptr> storage_read_bytes_; bool closed_ = false; }; diff --git a/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp b/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp index 87c1e0b2..b5680607 100644 --- a/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_stream_adapter_test.cpp @@ -62,7 +62,7 @@ TEST(ArrowStreamAdapterTest, TestInputAndOutputStream) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, file_system->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); auto in_stream = - std::make_unique(in, GetArrowPool(GetDefaultPool()), length); + std::make_unique(in, length, GetArrowPool(GetDefaultPool())); ASSERT_EQ(in_stream->GetSize().ValueOrDie(), static_cast(data.length())); ASSERT_EQ(in_stream->Tell().ValueOrDie(), 0); ASSERT_FALSE(in_stream->closed()); diff --git a/src/paimon/core/mergetree/spill_reader.cpp b/src/paimon/core/mergetree/spill_reader.cpp index 4b5d3f23..e1ed9004 100644 --- a/src/paimon/core/mergetree/spill_reader.cpp +++ b/src/paimon/core/mergetree/spill_reader.cpp @@ -56,7 +56,7 @@ Status SpillReader::Open(const FileIOChannel::ID& channel_id) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, fs_->GetFileStatus(file_path)); int64_t file_len = file_status->GetLen(); arrow_input_stream_adapter_ = - std::make_shared(in_stream_, arrow_pool_, file_len); + std::make_shared(in_stream_, file_len, arrow_pool_); auto ipc_read_options = arrow::ipc::IpcReadOptions::Defaults(); ipc_read_options.memory_pool = arrow_pool_.get(); ipc_read_options.use_threads = use_threads_; diff --git a/src/paimon/core/operation/file_store_commit_impl.cpp b/src/paimon/core/operation/file_store_commit_impl.cpp index 0131e1a9..6342f9b3 100644 --- a/src/paimon/core/operation/file_store_commit_impl.cpp +++ b/src/paimon/core/operation/file_store_commit_impl.cpp @@ -895,7 +895,6 @@ Result FileStoreCommitImpl::TryCommit( Snapshot::CommitKind commit_kind, bool detect_conflicts) { int32_t retry_count = 0; int64_t start_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000; - std::optional retry_start_snapshot_id; while (true) { PAIMON_ASSIGN_OR_RAISE(std::optional latest_snapshot, snapshot_manager_->LatestSnapshot()); @@ -905,13 +904,10 @@ Result FileStoreCommitImpl::TryCommit( bool commit_success, TryCommitOnce(commit_changes->delta_files, commit_changes->changelog_files, commit_changes->index_entries, identifier, watermark, properties, - commit_kind, latest_snapshot, detect_conflicts, retry_start_snapshot_id)); + commit_kind, latest_snapshot, detect_conflicts)); if (commit_success) { break; } - retry_start_snapshot_id = latest_snapshot - ? std::optional(latest_snapshot.value().Id() + 1) - : std::optional(Snapshot::FIRST_SNAPSHOT_ID); int64_t current_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000; if (current_millis - start_millis > options_.GetCommitTimeout() || retry_count >= options_.GetCommitMaxRetries()) { @@ -926,26 +922,6 @@ Result FileStoreCommitImpl::TryCommit( return retry_count + 1; } -Result FileStoreCommitImpl::CheckCommitted(const std::optional& latest_snapshot, - std::optional retry_start_snapshot_id, - int64_t identifier, - const Snapshot::CommitKind& commit_kind) const { - if (!latest_snapshot || !retry_start_snapshot_id || - retry_start_snapshot_id.value() > latest_snapshot.value().Id()) { - return false; - } - - for (int64_t snapshot_id = retry_start_snapshot_id.value(); - snapshot_id <= latest_snapshot.value().Id(); ++snapshot_id) { - PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, snapshot_manager_->LoadSnapshot(snapshot_id)); - if (snapshot.CommitUser() == commit_user_ && snapshot.CommitIdentifier() == identifier && - snapshot.GetCommitKind() == commit_kind) { - return true; - } - } - return false; -} - Status FileStoreCommitImpl::CheckSameBucketFromSnapshot( const std::vector& delta_entries, const std::optional& latest_snapshot) const { @@ -1018,13 +994,7 @@ Result FileStoreCommitImpl::TryCommitOnce( const std::vector& index_entries, int64_t identifier, std::optional watermark, const std::map& properties, Snapshot::CommitKind commit_kind, const std::optional& latest_snapshot, - bool detect_conflicts, std::optional retry_start_snapshot_id) { - PAIMON_ASSIGN_OR_RAISE(bool committed, CheckCommitted(latest_snapshot, retry_start_snapshot_id, - identifier, commit_kind)); - if (committed) { - return true; - } - + bool detect_conflicts) { std::vector delta_files = delta_entries; int64_t start_millis = DateTimeUtils::GetCurrentUTCTimeUs() / 1000; @@ -1253,11 +1223,21 @@ Result FileStoreCommitImpl::TryCommitOnce( Result commit_result = CommitSnapshotImpl(new_snapshot, delta_statistics); if (!commit_result.ok()) { - // commit exception is uncertain; retry after checking whether this commit already exists. - PAIMON_LOG_WARN(logger_, "Retry commit for exception. %s", + // commit exception, not sure about the situation and should not clean up the files. + PAIMON_LOG_WARN(logger_, + "You need to call FilterAndCommit to retry commit for exception. %s", commit_result.status().ToString().c_str()); + + // To prevent the case where an atomic write times out but actually succeeds, + // retrying the commit could lead to the snapshot file being committed multiple times. + // Therefore, retries should be handled by the upper layer, + // which should call FilterAndCommit to avoid duplicate commits. + // Therefore, we should not trigger cleanup here, + // as it may delete meta files from a snapshot that was just written by ourselves, + // leading to an incomplete or corrupted snapshot. guard.Release(); - return false; + return Status::Invalid("You need to call FilterAndCommit to retry commit for exception. ", + commit_result.status().ToString()); } bool commit_success = commit_result.value(); if (commit_success) { diff --git a/src/paimon/core/operation/file_store_commit_impl.h b/src/paimon/core/operation/file_store_commit_impl.h index ac1dc873..c0c93c9a 100644 --- a/src/paimon/core/operation/file_store_commit_impl.h +++ b/src/paimon/core/operation/file_store_commit_impl.h @@ -199,8 +199,7 @@ class FileStoreCommitImpl : public FileStoreCommit { const std::map& properties, Snapshot::CommitKind commit_kind, const std::optional& latest_snapshot, - bool detect_conflicts, - std::optional retry_start_snapshot_id); + bool detect_conflicts); Result CommitSnapshotImpl(const Snapshot& new_snapshot, const std::vector& delta_statistics); @@ -214,10 +213,6 @@ class FileStoreCommitImpl : public FileStoreCommit { const std::optional& old_index_manifest, const std::optional& new_index_manifest); - Result CheckCommitted(const std::optional& latest_snapshot, - std::optional retry_start_snapshot_id, int64_t identifier, - const Snapshot::CommitKind& commit_kind) const; - Status CheckSameBucketFromSnapshot(const std::vector& delta_entries, const std::optional& latest_snapshot) const; diff --git a/src/paimon/core/operation/file_store_commit_impl_test.cpp b/src/paimon/core/operation/file_store_commit_impl_test.cpp index 6bbe7c54..405b8280 100644 --- a/src/paimon/core/operation/file_store_commit_impl_test.cpp +++ b/src/paimon/core/operation/file_store_commit_impl_test.cpp @@ -278,8 +278,8 @@ class FileStoreCommitImplTest : public testing::Test { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), - /*external_path=*/std::nullopt, - /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); } @@ -294,8 +294,8 @@ class FileStoreCommitImplTest : public testing::Test { /*creation_time=*/Timestamp(0, 0), /*delete_row_count=*/std::nullopt, /*embedded_index=*/nullptr, FileSource::Append(), - /*external_path=*/std::nullopt, - /*value_stats_cols=*/std::nullopt, /*first_row_id=*/std::nullopt, + /*value_stats_cols=*/std::nullopt, + /*external_path=*/std::nullopt, /*first_row_id=*/std::nullopt, /*write_cols=*/std::nullopt); } @@ -640,12 +640,7 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua "/orc/append_09.db/append_09/commit_messages/commit_messages-01", /*version=*/3); ASSERT_GT(msgs.size(), 0); - ASSERT_OK(commit->Commit(msgs, /*commit_identifier=*/1)); - std::shared_ptr metrics = commit->GetCommitMetrics(); - ASSERT_TRUE(metrics); - ASSERT_OK_AND_ASSIGN(uint64_t counter, - metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS)); - ASSERT_EQ(2u, counter); + ASSERT_NOK(commit->Commit(msgs, /*commit_identifier=*/1)); ASSERT_OK_AND_ASSIGN( bool exist, file_system_->Exists(PathUtil::JoinPath(table_path, "snapshot/snapshot-6"))); ASSERT_TRUE(exist); @@ -658,6 +653,8 @@ TEST_F(FileStoreCommitImplTest, TestCommitWithAtomicWriteSnapshotTimeoutAndActua .Finish()); ASSERT_OK_AND_ASSIGN(auto commit_2, FileStoreCommit::Create(std::move(commit_context_2))); + ASSERT_OK_AND_ASSIGN(int32_t num_committed, commit_2->FilterAndCommit({{1, msgs}})); + ASSERT_EQ(0, num_committed); std::string new_snapshot_7 = PathUtil::JoinPath(table_path, "snapshot/snapshot-7"); EXPECT_CALL(*mock_fs, AtomicStore(testing::StrEq(new_snapshot_7), testing::_)) .WillOnce(testing::Invoke([&](const std::string& path, const std::string& content) { diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp index b9003436..4c63ea0b 100644 --- a/src/paimon/format/parquet/column_index_filter_test.cpp +++ b/src/paimon/format/parquet/column_index_filter_test.cpp @@ -255,7 +255,7 @@ class ColumnIndexFilterTest : public ::testing::Test { // Open as raw ParquetFileReader ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name_)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); parquet_reader_ = ::parquet::ParquetFileReader::Open(in_stream); ASSERT_TRUE(parquet_reader_); diff --git a/src/paimon/format/parquet/file_reader_wrapper_test.cpp b/src/paimon/format/parquet/file_reader_wrapper_test.cpp index 59d71b3f..3ac7f62e 100644 --- a/src/paimon/format/parquet/file_reader_wrapper_test.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper_test.cpp @@ -121,7 +121,7 @@ class FileReaderWrapperTest : public ::testing::Test { const std::string& file_path, int64_t wrapper_batch_size = 0) { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr in, fs_->Open(file_path)); PAIMON_ASSIGN_OR_RAISE(int64_t file_length, in->Length()); - auto input_stream = std::make_unique(in, arrow_pool_, file_length); + auto input_stream = std::make_unique(in, file_length, arrow_pool_); ::parquet::arrow::FileReaderBuilder file_reader_builder; ::parquet::ReaderProperties reader_properties; reader_properties.enable_buffered_stream(); diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index 75bf7522..e0672fd5 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -118,13 +118,14 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { int32_t batch_size = 1024) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); std::map options; options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "true"; ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( std::move(in_stream), options, batch_size, - /*file_metadata=*/nullptr, arrow_pool_)); + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate, @@ -143,11 +144,12 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test { int32_t batch_size = 1024) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); - ASSERT_OK_AND_ASSIGN(auto batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), options, - batch_size, nullptr, arrow_pool_)); + ASSERT_OK_AND_ASSIGN( + auto batch_reader, + ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size, nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok()); ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate, bitmap)); @@ -543,7 +545,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesPartialMatch) { // Open as raw ParquetFileReader ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); ASSERT_TRUE(parquet_reader); @@ -570,7 +572,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesAllMatch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); // All rows match @@ -597,7 +599,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesNoMatch) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); RowRanges row_ranges; // empty @@ -617,7 +619,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiColumn) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); // Match page 5 only (rows 50-59) @@ -645,7 +647,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); RowRanges row_ranges; @@ -794,7 +796,7 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding) // Open the file and verify metadata confirms dictionary page presence ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream); ASSERT_TRUE(parquet_reader); @@ -1639,11 +1641,11 @@ TEST_F(PageFilteredRowGroupReaderTest, BitmapInvalidStrategyTest) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); - ASSERT_OK_AND_ASSIGN( - auto batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), options, 1024, nullptr, arrow_pool_)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, 1024, nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); auto read_schema = arrow::schema({arrow::field("val", arrow::int32())}); auto c_schema = std::make_unique(); diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 8c44572f..547a3c1e 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -67,18 +67,21 @@ namespace paimon::parquet { ParquetFileBatchReader::ParquetFileBatchReader( std::shared_ptr&& input_stream, std::unique_ptr&& reader, const std::map& options, - const std::shared_ptr& arrow_pool) + const std::shared_ptr& arrow_pool, + std::shared_ptr> storage_read_bytes) : options_(options), arrow_pool_(arrow_pool), input_stream_(std::move(input_stream)), reader_(std::move(reader)), metrics_(std::make_shared()), + storage_read_bytes_(std::move(storage_read_bytes)), logger_(Logger::GetLogger("ParquetFileBatchReader")) {} Result> ParquetFileBatchReader::Create( std::shared_ptr&& input_stream, const std::map& options, int32_t batch_size, std::shared_ptr<::parquet::FileMetaData> file_metadata, + std::shared_ptr> storage_read_bytes, const std::shared_ptr& pool) { try { assert(input_stream); @@ -100,7 +103,8 @@ Result> ParquetFileBatchReader::Create( FileReaderWrapper::Create(std::move(file_reader), static_cast(batch_size), pool)); auto parquet_file_batch_reader = std::unique_ptr( - new ParquetFileBatchReader(std::move(input_stream), std::move(reader), options, pool)); + new ParquetFileBatchReader(std::move(input_stream), std::move(reader), options, pool, + std::move(storage_read_bytes))); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> file_schema, parquet_file_batch_reader->GetFileSchema()); PAIMON_RETURN_NOT_OK(parquet_file_batch_reader->SetReadSchema( diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.h b/src/paimon/format/parquet/parquet_file_batch_reader.h index 308e7dfa..0cdecbfb 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.h +++ b/src/paimon/format/parquet/parquet_file_batch_reader.h @@ -20,6 +20,7 @@ #include +#include #include #include #include @@ -40,6 +41,7 @@ #include "paimon/common/metrics/metrics_impl.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/format/parquet/file_reader_wrapper.h" +#include "paimon/format/parquet/parquet_format_defs.h" #include "paimon/format/parquet/row_ranges.h" #include "paimon/format/parquet/target_row_group.h" #include "paimon/logging.h" @@ -73,6 +75,7 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { std::shared_ptr&& input_stream, const std::map& options, int32_t batch_size, std::shared_ptr<::parquet::FileMetaData> file_metadata, + std::shared_ptr> storage_read_bytes, const std::shared_ptr& pool); static Result<::parquet::ReaderProperties> CreateReaderProperties( @@ -132,6 +135,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { } std::shared_ptr GetReaderMetrics() const override { + uint64_t storage = storage_read_bytes_ ? storage_read_bytes_->load() : 0; + metrics_->SetCounter(ParquetMetrics::READ_STORAGE_BYTES, storage); return metrics_; } @@ -152,7 +157,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { ParquetFileBatchReader(std::shared_ptr&& input_stream, std::unique_ptr&& reader, const std::map& options, - const std::shared_ptr& arrow_pool); + const std::shared_ptr& arrow_pool, + std::shared_ptr> storage_read_bytes); static Result<::parquet::ArrowReaderProperties> CreateArrowReaderProperties( const std::shared_ptr& pool, @@ -253,6 +259,8 @@ class ParquetFileBatchReader : public PrefetchFileBatchReader { std::shared_ptr read_data_type_; std::shared_ptr metrics_; + // storageReadBytes counter shared with the underlying ArrowInputStreamAdapter. + std::shared_ptr> storage_read_bytes_; std::unique_ptr logger_; uint64_t read_rows_ = 0; diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp index 1c40b230..7a4328ff 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp @@ -18,9 +18,11 @@ #include "paimon/format/parquet/parquet_file_batch_reader.h" +#include #include #include #include +#include #include #include "arrow/api.h" @@ -202,12 +204,14 @@ class ParquetFileBatchReaderTest : public ::testing::Test, EXPECT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_name)); auto length = fs_->GetFileStatus(file_name).value()->GetLen(); auto in_stream = - std::make_unique(std::move(input_stream), pool_, length); + std::make_unique(std::move(input_stream), length, pool_); + auto storage_read_bytes = in_stream->StorageReadBytes(); std::map options; options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = enable_page_level_filter ? "true" : "false"; return PrepareParquetFileBatchReader(std::move(in_stream), options, read_schema, predicate, - selection_bitmap, batch_size); + selection_bitmap, batch_size, + std::move(storage_read_bytes)); } std::unique_ptr PrepareParquetFileBatchReader( @@ -215,11 +219,12 @@ class ParquetFileBatchReaderTest : public ::testing::Test, const std::map& options, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, - const std::optional& selection_bitmap, int32_t batch_size) const { - EXPECT_OK_AND_ASSIGN( - auto parquet_batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size, - /*file_metadata=*/nullptr, pool_)); + const std::optional& selection_bitmap, int32_t batch_size, + std::shared_ptr> storage_read_bytes = nullptr) const { + EXPECT_OK_AND_ASSIGN(auto parquet_batch_reader, + ParquetFileBatchReader::Create( + std::move(in_stream), options, batch_size, + /*file_metadata=*/nullptr, std::move(storage_read_bytes), pool_)); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); EXPECT_TRUE(arrow_status.ok()); @@ -381,11 +386,12 @@ TEST_F(ParquetFileBatchReaderTest, TestSetReadSchema) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, fs_->Open(file_name)); auto length = fs_->GetFileStatus(file_name).value()->GetLen(); auto in_stream = - std::make_unique(std::move(input_stream), pool_, length); + std::make_unique(std::move(input_stream), length, pool_); std::map options; ASSERT_OK_AND_ASSIGN(auto parquet_batch_reader, ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size_, - /*file_metadata=*/nullptr, pool_)); + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, pool_)); // test GetFileSchema() ASSERT_OK_AND_ASSIGN(auto c_file_schema, parquet_batch_reader->GetFileSchema()); auto arrow_file_schema = arrow::ImportSchema(c_file_schema.get()).ValueOrDie(); @@ -474,9 +480,9 @@ TEST_F(ParquetFileBatchReaderTest, TestNextBatchSimple) { // test metrics auto read_metrics = parquet_batch_reader->GetReaderMetrics(); ASSERT_TRUE(read_metrics); - // TODO(jinli.zjw): test metrics - // ASSERT_TRUE(read_metrics->GetCounter(ParquetMetrics::READ_BYTES) > 0); - // ASSERT_TRUE(read_metrics->GetCounter(ParquetMetrics::READ_RAW_BYTES) > 0); + ASSERT_OK_AND_ASSIGN(uint64_t storage_read_bytes, + read_metrics->GetCounter(ParquetMetrics::READ_STORAGE_BYTES)); + ASSERT_GT(storage_read_bytes, 0u); } } diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h index 5cb6f2da..cb171078 100644 --- a/src/paimon/format/parquet/parquet_format_defs.h +++ b/src/paimon/format/parquet/parquet_format_defs.h @@ -114,6 +114,8 @@ class ParquetMetrics { "parquet.read.row-groups.after-filter"; static inline const char READ_ROWS[] = "parquet.read.rows"; static inline const char READ_BATCH_COUNT[] = "parquet.read.batch-count"; + // Byte-level read metric. storage-read-bytes: physical bytes read from storage. + static inline const char READ_STORAGE_BYTES[] = "parquet.read.storage-read-bytes"; }; } // namespace paimon::parquet diff --git a/src/paimon/format/parquet/parquet_reader_builder.h b/src/paimon/format/parquet/parquet_reader_builder.h index 76e2fd9c..51976372 100644 --- a/src/paimon/format/parquet/parquet_reader_builder.h +++ b/src/paimon/format/parquet/parquet_reader_builder.h @@ -72,13 +72,15 @@ class ParquetReaderBuilder : public ReaderBuilder { } std::shared_ptr arrow_pool = GetArrowPool(pool_); auto unique_input_stream = - std::make_unique(path, arrow_pool, file_length); + std::make_unique(path, file_length, arrow_pool); + auto storage_read_bytes = unique_input_stream->StorageReadBytes(); std::shared_ptr input_stream( std::move(unique_input_stream)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<::parquet::FileMetaData> file_metadata, GetCachedParquetMetadata(input_stream, file_uri, arrow_pool)); return ParquetFileBatchReader::Create(std::move(input_stream), options_, batch_size_, - std::move(file_metadata), arrow_pool); + std::move(file_metadata), + std::move(storage_read_bytes), arrow_pool); } PAIMON_PARQUET_CATCH_AND_RETURN_STATUS("ParquetReaderBuilder::Build") } diff --git a/src/paimon/format/parquet/parquet_stats_extractor.cpp b/src/paimon/format/parquet/parquet_stats_extractor.cpp index f5789161..f1df8b6e 100644 --- a/src/paimon/format/parquet/parquet_stats_extractor.cpp +++ b/src/paimon/format/parquet/parquet_stats_extractor.cpp @@ -278,7 +278,7 @@ ParquetStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& fi PAIMON_ASSIGN_OR_RAISE(int64_t file_length, input_stream->Length()); std::shared_ptr parquet_memory_pool = GetArrowPool(pool); auto parquet_input_file = std::make_shared( - std::move(input_stream), parquet_memory_pool, file_length); + std::move(input_stream), file_length, parquet_memory_pool); ::parquet::ReaderProperties read_properties(parquet_memory_pool.get()); read_properties.enable_buffered_stream(); ::parquet::arrow::FileReaderBuilder file_reader_builder; diff --git a/src/paimon/format/parquet/predicate_pushdown_test.cpp b/src/paimon/format/parquet/predicate_pushdown_test.cpp index 64ed5a54..b63eaa38 100644 --- a/src/paimon/format/parquet/predicate_pushdown_test.cpp +++ b/src/paimon/format/parquet/predicate_pushdown_test.cpp @@ -107,14 +107,15 @@ class PredicatePushdownTest : public ::testing::Test { paimon::parquet::DEFAULT_PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT) { ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name_)); ASSERT_OK_AND_ASSIGN(int64_t length, in->Length()); - auto in_stream = std::make_shared(in, arrow_pool_, length); + auto in_stream = std::make_shared(in, length, arrow_pool_); std::map options; options[paimon::parquet::PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT] = std::to_string(predicate_node_count_limit); ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( std::move(in_stream), options, batch_size_, - /*file_metadata=*/nullptr, arrow_pool_)); + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); std::unique_ptr c_schema = std::make_unique(); auto arrow_status = arrow::ExportSchema(*read_schema, c_schema.get()); ASSERT_TRUE(arrow_status.ok()); diff --git a/src/paimon/format/parquet/variant_parquet_test.cpp b/src/paimon/format/parquet/variant_parquet_test.cpp index 2a7e521b..b2bf1693 100644 --- a/src/paimon/format/parquet/variant_parquet_test.cpp +++ b/src/paimon/format/parquet/variant_parquet_test.cpp @@ -363,12 +363,13 @@ class VariantParquetTest : public ::testing::Test { ASSERT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_path_)); auto length = fs_->GetFileStatus(file_path_).value()->GetLen(); auto in_stream = - std::make_unique(std::move(input_stream), arrow_pool_, length); + std::make_unique(std::move(input_stream), length, arrow_pool_); std::map options = {}; ASSERT_OK_AND_ASSIGN(auto parquet_reader, ParquetFileBatchReader::Create( std::move(in_stream), options, /*batch_size=*/1024, - /*file_metadata=*/nullptr, arrow_pool_)); + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); *file_reader = std::move(parquet_reader); ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> c_file_schema, (*file_reader)->GetFileSchema()); @@ -549,12 +550,13 @@ TEST_F(VariantParquetTest, WriteAndReadRoundTrip) { ASSERT_OK_AND_ASSIGN(auto input_stream, fs_->Open(file_path_)); auto length = fs_->GetFileStatus(file_path_).value()->GetLen(); auto in_stream = - std::make_unique(std::move(input_stream), arrow_pool_, length); + std::make_unique(std::move(input_stream), length, arrow_pool_); std::map options = {}; - ASSERT_OK_AND_ASSIGN(auto batch_reader, - ParquetFileBatchReader::Create(std::move(in_stream), options, - /*batch_size=*/1024, - /*file_metadata=*/nullptr, arrow_pool_)); + ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create( + std::move(in_stream), options, + /*batch_size=*/1024, + /*file_metadata=*/nullptr, + /*storage_read_bytes=*/nullptr, arrow_pool_)); auto c_schema = std::make_unique(); ASSERT_TRUE(arrow::ExportSchema(*paimon_schema_, c_schema.get()).ok()); ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, diff --git a/test/inte/clean_inte_test.cpp b/test/inte/clean_inte_test.cpp index b59969ea..536bdc74 100644 --- a/test/inte/clean_inte_test.cpp +++ b/test/inte/clean_inte_test.cpp @@ -407,7 +407,7 @@ TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshot) { ASSERT_EQ(3u, manifests[1].NumAddedFiles()); } -TEST_F(CleanInteTest, DISABLED_TestDropPartitionAndExpireSnapshotWithIOException) { +TEST_F(CleanInteTest, TestDropPartitionAndExpireSnapshotWithIOException) { auto string_field = arrow::field("f0", arrow::utf8()); auto int_field = arrow::field("f1", arrow::int32()); auto int_field1 = arrow::field("f2", arrow::int32()); From 598c8bcf0c5e941c454c6fed6f358d91451067e2 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Thu, 23 Jul 2026 09:59:44 +0800 Subject: [PATCH 112/138] feat(lumina): support Lumina tag filtering --- .../global_index/global_index_io_meta.h | 2 +- include/paimon/global_index/global_indexer.h | 4 + include/paimon/predicate/vector_search.h | 4 + .../bitmap/bitmap_global_index.cpp | 4 + .../global_index/bitmap/bitmap_global_index.h | 3 + .../btree/btree_global_indexer.cpp | 4 + .../global_index/btree/btree_global_indexer.h | 2 + .../rangebitmap/range_bitmap_global_index.cpp | 4 + .../rangebitmap/range_bitmap_global_index.h | 3 + .../reader/data_evolution_file_reader.cpp | 6 +- .../data_evolution_file_reader_test.cpp | 3 +- .../global_index/global_index_scan_impl.cpp | 2 +- .../global_index/global_index_write_task.cpp | 251 ++++++- .../lucene/lucene_global_index.cpp | 4 + .../global_index/lucene/lucene_global_index.h | 3 + .../lumina/lumina_global_index.cpp | 672 +++++++++++++++++- .../global_index/lumina/lumina_global_index.h | 47 ++ .../lumina/lumina_global_index_test.cpp | 457 +++++++++++- .../tantivy/tantivy_global_index.cpp | 4 + .../tantivy/tantivy_global_index.h | 3 + test/inte/global_index_test.cpp | 224 +++++- third_party/versions.txt | 4 +- 22 files changed, 1635 insertions(+), 75 deletions(-) diff --git a/include/paimon/global_index/global_index_io_meta.h b/include/paimon/global_index/global_index_io_meta.h index f4497b7e..10ace821 100644 --- a/include/paimon/global_index/global_index_io_meta.h +++ b/include/paimon/global_index/global_index_io_meta.h @@ -19,11 +19,11 @@ #pragma once +#include #include #include #include "paimon/memory/bytes.h" -#include "paimon/utils/range.h" namespace paimon { /// Metadata describing a single file entry in a global index. diff --git a/include/paimon/global_index/global_indexer.h b/include/paimon/global_index/global_indexer.h index 3cf5f2c0..4da6293f 100644 --- a/include/paimon/global_index/global_indexer.h +++ b/include/paimon/global_index/global_indexer.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -39,6 +40,9 @@ class PAIMON_EXPORT GlobalIndexer { public: virtual ~GlobalIndexer() = default; + /// Returns additional table fields required during index construction. + virtual Result>> GetExtraFieldNames() const = 0; + /// Creates a writer for building a global index on a specific field. /// /// @param field_name Name of the field to be indexed. diff --git a/include/paimon/predicate/vector_search.h b/include/paimon/predicate/vector_search.h index b77abd23..44852162 100644 --- a/include/paimon/predicate/vector_search.h +++ b/include/paimon/predicate/vector_search.h @@ -76,6 +76,10 @@ struct PAIMON_EXPORT VectorSearch { /// context-aware filtering at query time. /// @note All fields referenced in the predicate must have been materialized /// in the index during build to ensure availability. + /// @note For tag-based vector indexes, fields referenced by the predicate + /// must keep the same names and types as the fields used during index + /// construction. Indexes built with tag fields must not be reused across + /// schema evolution that renames or changes those tag fields. std::shared_ptr predicate; /// The distance metric to use for this query, if explicitly specified. /// If set, this value must match the distance type used by the index (e.g., EUCLIDEAN, COSINE). diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp b/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp index e0a8f54f..2975c90c 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp @@ -22,6 +22,10 @@ #include "paimon/common/global_index/wrap/file_index_writer_wrapper.h" namespace paimon { +Result>> BitmapGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> BitmapGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index.h b/src/paimon/common/global_index/bitmap/bitmap_global_index.h index 60b11fcb..7045dcb8 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index.h +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -32,6 +33,8 @@ class BitmapGlobalIndex : public GlobalIndexer { public: explicit BitmapGlobalIndex(const std::shared_ptr& index) : index_(index) {} + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index fd410089..29bffe9f 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -57,6 +57,10 @@ Result> BTreeGlobalIndexer::Create( return std::unique_ptr(new BTreeGlobalIndexer(cache_manager, options)); } +Result>> BTreeGlobalIndexer::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> BTreeGlobalIndexer::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.h b/src/paimon/common/global_index/btree/btree_global_indexer.h index d1f5e926..5568adba 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.h +++ b/src/paimon/common/global_index/btree/btree_global_indexer.h @@ -56,6 +56,8 @@ class BTreeGlobalIndexer : public GlobalIndexer { static Result> Create( const std::map& options); + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp index db3c756e..71f9f4bc 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp @@ -22,6 +22,10 @@ #include "paimon/common/global_index/wrap/file_index_writer_wrapper.h" namespace paimon { +Result>> RangeBitmapGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> RangeBitmapGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h index 7b8923e4..3e60c428 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -33,6 +34,8 @@ class RangeBitmapGlobalIndex : public GlobalIndexer { explicit RangeBitmapGlobalIndex(const std::shared_ptr& index) : index_(index) {} + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index 8e8fbbce..32eb5f96 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -26,6 +26,7 @@ #include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" + namespace paimon { Result> DataEvolutionFileReader::Create( std::vector>&& readers, @@ -168,11 +169,10 @@ Result> DataEvolutionFileReader::NextBatchForSingl if (concat_array_vec.empty()) { return std::shared_ptr(); } - if (concat_array_vec.size() == 1) { - // avoid data copy + if (concat_array_vec.size() == 1 && concat_array_vec[0]->offset() == 0) { + // Avoid data copy when the array is already normalized. return concat_array_vec[0]; } - // TODO(xinyu.lxy) remove data copy for efficiency PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, arrow::Concatenate(concat_array_vec, arrow_pool_.get())); assert(concat_array->length() == total_array_length); diff --git a/src/paimon/common/reader/data_evolution_file_reader_test.cpp b/src/paimon/common/reader/data_evolution_file_reader_test.cpp index 39afb15c..5f7e3a07 100644 --- a/src/paimon/common/reader/data_evolution_file_reader_test.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader_test.cpp @@ -33,8 +33,8 @@ #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" -namespace paimon::test { +namespace paimon::test { class DataEvolutionFileReaderTest : public ::testing::Test, public ::testing::WithParamInterface { public: @@ -120,6 +120,7 @@ class DataEvolutionFileReaderTest : public ::testing::Test, if (result_array == nullptr) { break; } + ASSERT_EQ(result_array->offset(), 0); result_array_vec.push_back(result_array); } ASSERT_EQ(result_array_vec.size(), diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index b40478e1..e4b3e7b7 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -23,6 +23,7 @@ #include #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" #include "paimon/common/global_index/offset_global_index_reader.h" #include "paimon/common/global_index/union_global_index_reader.h" #include "paimon/common/utils/scope_guard.h" @@ -188,7 +189,6 @@ Result>> GlobalIndexScanImpl::Cre if (row_range_index && !row_range_index->Intersects(range.from, range.to)) { continue; } - // TODO(xinyu.lxy): c_arrow_schema may contains additional associated fields. auto arrow_field = DataField::ConvertDataFieldToArrowField(field); auto arrow_schema = arrow::schema({arrow_field}); diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index 3208e87e..f0e35050 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -19,11 +19,18 @@ #include "paimon/global_index/global_index_write_task.h" +#include + +#include "arrow/array/array_nested.h" #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "arrow/type.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/casting/casting_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/global_index/global_index_file_manager.h" #include "paimon/core/io/data_increment.h" @@ -38,6 +45,17 @@ #include "paimon/table/source/table_read.h" namespace paimon { namespace { +Result> CreateGlobalIndexer(const std::string& index_type, + const CoreOptions& core_options) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(index_type, core_options.ToMap())); + if (!indexer) { + return Status::Invalid( + fmt::format("Unknown index type {}, may not registered", index_type)); + } + return indexer; +} + Result> CreateGlobalIndexFileManager( const std::string& table_path, const std::shared_ptr& table_schema, const CoreOptions& core_options, const std::shared_ptr& pool) { @@ -61,25 +79,81 @@ Result> CreateGlobalIndexFileManager( } Result> CreateGlobalIndexWriter( - const std::string& index_type, const DataField& field, + const GlobalIndexer& indexer, const DataField& field, + const std::vector& extra_fields, const std::shared_ptr& index_file_manager, - const CoreOptions& core_options, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, - GlobalIndexerFactory::Get(index_type, core_options.ToMap())); - if (!indexer) { - return Status::Invalid( - fmt::format("Unknown index type {}, may not registered", index_type)); + const std::shared_ptr& pool) { + arrow::FieldVector arrow_fields; + arrow_fields.reserve(extra_fields.size() + 1); + arrow_fields.push_back(DataField::ConvertDataFieldToArrowField(field)); + for (const auto& extra_field : extra_fields) { + arrow_fields.push_back(DataField::ConvertDataFieldToArrowField(extra_field)); } - // TODO(xinyu.lxy): may add additional fields to read for index write - auto arrow_field = DataField::ConvertDataFieldToArrowField(field); - auto arrow_schema = arrow::schema({arrow_field}); + auto arrow_schema = arrow::schema(arrow_fields); ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); - return indexer->CreateWriter(field.Name(), &c_arrow_schema, index_file_manager, pool); + ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); + return indexer.CreateWriter(field.Name(), &c_arrow_schema, index_file_manager, pool); +} + +Result> GetExtraFields(const TableSchema& table_schema, + const std::string& field_name, + const std::vector& extra_field_names) { + std::vector extra_fields; + extra_fields.reserve(extra_field_names.size()); + std::set dedup_field_names; + for (const auto& extra_field_name : extra_field_names) { + if (extra_field_name == field_name) { + return Status::Invalid(fmt::format( + "global index extra field {} must not be the indexed field", extra_field_name)); + } + if (!dedup_field_names.insert(extra_field_name).second) { + return Status::Invalid(fmt::format("global index extra field {} must not be duplicated", + extra_field_name)); + } + PAIMON_ASSIGN_OR_RAISE(DataField extra_field, table_schema.GetField(extra_field_name)); + extra_fields.push_back(extra_field); + } + return extra_fields; +} + +std::vector BuildReadFieldNames(const std::string& field_name, + const std::vector& extra_fields) { + std::vector read_field_names; + read_field_names.reserve(extra_fields.size() + 2); + read_field_names.push_back(field_name); + for (const auto& extra_field : extra_fields) { + read_field_names.push_back(extra_field.Name()); + } + read_field_names.push_back(SpecialFields::RowId().Name()); + return read_field_names; +} + +std::vector BuildWriterFieldNames(const std::string& field_name, + const std::vector& extra_fields) { + std::vector writer_field_names; + writer_field_names.reserve(extra_fields.size() + 1); + writer_field_names.push_back(field_name); + for (const auto& extra_field : extra_fields) { + writer_field_names.push_back(extra_field.Name()); + } + return writer_field_names; +} + +std::optional> GetExtraFieldIds(const std::vector& extra_fields) { + if (extra_fields.empty()) { + return std::nullopt; + } + std::vector extra_field_ids; + extra_field_ids.reserve(extra_fields.size()); + for (const auto& extra_field : extra_fields) { + extra_field_ids.push_back(extra_field.Id()); + } + return extra_field_ids; } Result> CreateBatchReader( - const std::string& table_path, const std::string& field_name, + const std::string& table_path, const std::vector& read_field_names, const std::shared_ptr& indexed_split, const CoreOptions& core_options, const std::shared_ptr& pool) { ReadContextBuilder read_context_builder(table_path); @@ -87,7 +161,7 @@ Result> CreateBatchReader( .WithFileSystem(core_options.GetFileSystem()) .EnablePrefetch(true) .WithMemoryPool(pool) - .SetReadFieldNames({field_name, SpecialFields::RowId().Name()}); + .SetReadFieldNames(read_field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, @@ -95,9 +169,99 @@ Result> CreateBatchReader( return table_read->CreateReader(indexed_split); } -Result> BuildIndex(const std::string& field_name, const Range& range, - BatchReader* batch_reader, - GlobalIndexWriter* global_index_writer) { +Result> CastDictionaryArrayToString( + const std::shared_ptr& array, arrow::MemoryPool* pool) { + arrow::Type::type type_id = array->type_id(); + if (type_id == arrow::Type::DICTIONARY) { + const auto* dictionary_type = + static_cast(array->type().get()); + arrow::Type::type value_type = dictionary_type->value_type()->id(); + if (value_type != arrow::Type::STRING && value_type != arrow::Type::LARGE_STRING) { + return Status::Invalid(fmt::format( + "GlobalIndexWriteTask cannot decode dictionary array with value type {}", + dictionary_type->value_type()->ToString())); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr casted_array, + CastingUtils::Cast(array, arrow::utf8(), arrow::compute::CastOptions::Safe(), pool)); + return casted_array; + } + if (type_id != arrow::Type::STRUCT && type_id != arrow::Type::MAP && + type_id != arrow::Type::LIST) { + return array; + } + + if (type_id == arrow::Type::STRUCT) { + std::shared_ptr struct_array = + std::static_pointer_cast(array); + arrow::ArrayVector children; + for (int32_t i = 0; i < struct_array->num_fields(); i++) { + std::shared_ptr child = struct_array->field(i); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr casted_child, + CastDictionaryArrayToString(child, pool)); + if (casted_child != child && children.empty()) { + children = struct_array->fields(); + } + if (!children.empty()) { + children[i] = std::move(casted_child); + } + } + if (children.empty()) { + return array; + } + std::vector field_names; + field_names.reserve(struct_array->num_fields()); + for (int32_t i = 0; i < struct_array->num_fields(); i++) { + field_names.push_back(struct_array->struct_type()->field(i)->name()); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr casted_array, + arrow::StructArray::Make(children, field_names, struct_array->null_bitmap(), + struct_array->null_count(), struct_array->offset())); + return casted_array; + } + + if (type_id == arrow::Type::MAP) { + std::shared_ptr map_array = + std::static_pointer_cast(array); + std::shared_ptr original_keys = map_array->keys(); + std::shared_ptr original_items = map_array->items(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr keys, + CastDictionaryArrayToString(original_keys, pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr items, + CastDictionaryArrayToString(original_items, pool)); + if (keys == original_keys && items == original_items) { + return array; + } + const auto* map_type = static_cast(map_array->type().get()); + std::shared_ptr casted_type = std::make_shared( + map_type->key_field()->WithType(keys->type()), + map_type->item_field()->WithType(items->type()), map_type->keys_sorted()); + return std::make_shared( + casted_type, map_array->length(), map_array->value_offsets(), keys, items, + map_array->null_bitmap(), map_array->null_count(), map_array->offset()); + } + + std::shared_ptr list_array = + std::static_pointer_cast(array); + std::shared_ptr original_values = list_array->values(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr values, + CastDictionaryArrayToString(original_values, pool)); + if (values == original_values) { + return array; + } + const auto* list_type = static_cast(list_array->type().get()); + std::shared_ptr casted_type = + arrow::list(list_type->value_field()->WithType(values->type())); + return std::make_shared( + casted_type, list_array->length(), list_array->value_offsets(), values, + list_array->null_bitmap(), list_array->null_count(), list_array->offset()); +} + +Result> BuildIndex( + const std::string& field_name, const Range& range, + const std::vector& writer_field_names, BatchReader* batch_reader, + GlobalIndexWriter* global_index_writer, arrow::MemoryPool* arrow_pool) { while (true) { PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch read_batch, batch_reader->NextBatch()); if (BatchReader::IsEofBatch(read_batch)) { @@ -111,11 +275,6 @@ Result> BuildIndex(const std::string& field_name, return Status::Invalid( "array read from batch reader is not a struct array in GlobalIndexWriteTask"); } - auto indexed_array = struct_array->GetFieldByName(field_name); - if (!indexed_array) { - return Status::Invalid(fmt::format( - "read array does not contain {} field in GlobalIndexWriteTask", field_name)); - } auto row_id_array = struct_array->GetFieldByName(SpecialFields::RowId().Name()); auto typed_row_id_array = std::dynamic_pointer_cast(row_id_array); if (!typed_row_id_array) { @@ -134,8 +293,22 @@ Result> BuildIndex(const std::string& field_name, } relative_row_ids.push_back(row_id - range.from); } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr new_array, - arrow::StructArray::Make({indexed_array}, {field_name})); + std::vector> writer_arrays; + writer_arrays.reserve(writer_field_names.size()); + for (const auto& writer_field_name : writer_field_names) { + auto writer_array = struct_array->GetFieldByName(writer_field_name); + if (!writer_array) { + return Status::Invalid( + fmt::format("read array does not contain {} field in GlobalIndexWriteTask", + writer_field_name)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr decoded_writer_array, + CastDictionaryArrayToString(writer_array, arrow_pool)); + writer_arrays.push_back(std::move(decoded_writer_array)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr new_array, + arrow::StructArray::Make(writer_arrays, writer_field_names)); ::ArrowArray c_new_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*new_array, &c_new_array)); PAIMON_RETURN_NOT_OK( @@ -147,7 +320,8 @@ Result> BuildIndex(const std::string& field_name, Result> ToCommitMessage( const std::string& index_type, int32_t field_id, const Range& range, const std::vector& global_index_io_metas, const BinaryRow& partition, - int32_t bucket, const std::shared_ptr& file_manager) { + int32_t bucket, const std::shared_ptr& file_manager, + const std::optional>& extra_field_ids) { std::vector> index_file_metas; index_file_metas.reserve(global_index_io_metas.size()); bool is_external_path = file_manager->IsExternalPath(); @@ -160,8 +334,7 @@ Result> ToCommitMessage( index_file_metas.push_back(std::make_shared( index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, range.Count(), /*dv_ranges=*/std::nullopt, external_path, - GlobalIndexMeta(range.from, range.to, field_id, - /*extra_field_ids=*/std::nullopt, io_meta.metadata))); + GlobalIndexMeta(range.from, range.to, field_id, extra_field_ids, io_meta.metadata))); } DataIncrement data_increment(std::move(index_file_metas)); return std::make_shared(partition, bucket, @@ -185,6 +358,7 @@ Result> GlobalIndexWriteTask::WriteIndex( } const auto& range = ranges[0]; std::shared_ptr pool = memory_pool ? memory_pool : GetDefaultPool(); + std::unique_ptr arrow_pool = GetArrowPool(pool); // load schema PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(options, file_system)); @@ -202,6 +376,17 @@ Result> GlobalIndexWriteTask::WriteIndex( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(final_options, file_system)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + CreateGlobalIndexer(index_type, core_options)); + PAIMON_ASSIGN_OR_RAISE(std::optional> extra_field_names, + indexer->GetExtraFieldNames()); + PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(field_name)); + PAIMON_ASSIGN_OR_RAISE(std::vector extra_fields, + GetExtraFields(*table_schema, field_name, + extra_field_names.value_or(std::vector()))); + std::optional> extra_field_ids = GetExtraFieldIds(extra_fields); + std::vector writer_field_names = BuildWriterFieldNames(field_name, extra_fields); + std::vector read_field_names = BuildReadFieldNames(field_name, extra_fields); // create index file manager PAIMON_ASSIGN_OR_RAISE( @@ -211,13 +396,12 @@ Result> GlobalIndexWriteTask::WriteIndex( // create batch reader PAIMON_ASSIGN_OR_RAISE( std::unique_ptr batch_reader, - CreateBatchReader(table_path, field_name, indexed_split, core_options, pool)); + CreateBatchReader(table_path, read_field_names, indexed_split, core_options, pool)); // create global index writer - PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(field_name)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr global_index_writer, - CreateGlobalIndexWriter(index_type, field, index_file_manager, core_options, pool)); + CreateGlobalIndexWriter(*indexer, field, extra_fields, index_file_manager, pool)); ScopeGuard guard([&]() { global_index_writer.reset(); @@ -225,13 +409,14 @@ Result> GlobalIndexWriteTask::WriteIndex( }); // read from data split and write to index writer - PAIMON_ASSIGN_OR_RAISE( - std::vector global_index_io_metas, - BuildIndex(field_name, range, batch_reader.get(), global_index_writer.get())); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_io_metas, + BuildIndex(field_name, range, writer_field_names, batch_reader.get(), + global_index_writer.get(), arrow_pool.get())); // generate commit message return ToCommitMessage(index_type, field.Id(), range, global_index_io_metas, - data_split->Partition(), data_split->Bucket(), index_file_manager); + data_split->Partition(), data_split->Bucket(), index_file_manager, + extra_field_ids); } } // namespace paimon diff --git a/src/paimon/global_index/lucene/lucene_global_index.cpp b/src/paimon/global_index/lucene/lucene_global_index.cpp index fa54d377..03715bcc 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index.cpp @@ -37,6 +37,10 @@ namespace paimon::lucene { LuceneGlobalIndex::LuceneGlobalIndex(const std::map& options) : options_(OptionsUtils::FetchOptionsWithPrefix(kOptionKeyPrefix, options)) {} +Result>> LuceneGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> LuceneGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/global_index/lucene/lucene_global_index.h b/src/paimon/global_index/lucene/lucene_global_index.h index 9f033ba3..68857c79 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.h +++ b/src/paimon/global_index/lucene/lucene_global_index.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include @@ -33,6 +34,8 @@ class LuceneGlobalIndex : public GlobalIndexer { public: explicit LuceneGlobalIndex(const std::map& options); + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/global_index/lumina/lumina_global_index.cpp b/src/paimon/global_index/lumina/lumina_global_index.cpp index 9d0fcdd1..b27cdc7a 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index.cpp @@ -18,6 +18,10 @@ #include "paimon/global_index/lumina/lumina_global_index.h" +#include +#include +#include +#include #include #include "arrow/c/bridge.h" @@ -29,6 +33,7 @@ #include "lumina/core/Constants.h" #include "lumina/core/Status.h" #include "lumina/core/Types.h" +#include "lumina/extensions/experimental/BuildCombinedExtensionV0.h" #include "paimon/common/global_index/global_index_utils.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/rapidjson_util.h" @@ -37,6 +42,9 @@ #include "paimon/global_index/lumina/lumina_file_reader.h" #include "paimon/global_index/lumina/lumina_file_writer.h" #include "paimon/global_index/lumina/lumina_utils.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/leaf_predicate.h" +#include "rapidjson/document.h" namespace paimon::lumina { #define CHECK_NOT_NULL(pointer, error_msg) \ do { \ @@ -45,6 +53,502 @@ namespace paimon::lumina { } \ } while (0) +namespace { +using TagDimensionData = ::lumina::extensions::experimental::TagDimensionData; +using TagFilter = ::lumina::extensions::experimental::TagFilter; +using TagValue = ::lumina::extensions::experimental::TagValue; +using TagValues = ::lumina::extensions::experimental::TagValues; + +Result GetRequiredStringMember(const rapidjson::Value& obj, + const std::string& field_name, + const std::string& tag_label) { + auto iter = obj.FindMember(field_name.c_str()); + if (iter == obj.MemberEnd()) { + return Status::Invalid( + fmt::format("lumina tag_schema {} missing required field: {}", tag_label, field_name)); + } + if (!iter->value.IsString()) { + return Status::Invalid( + fmt::format("lumina tag_schema {} field {} must be string", tag_label, field_name)); + } + return std::string(iter->value.GetString(), iter->value.GetStringLength()); +} + +Result ParseTagField(const rapidjson::Value& obj, const std::string& tag_label) { + if (!obj.IsObject()) { + return Status::Invalid(fmt::format("lumina tag_schema {} must be object", tag_label)); + } + if (obj.MemberCount() != 3) { + return Status::Invalid(fmt::format( + "lumina tag_schema {} must have exactly 3 fields: key_name, type, value_type", + tag_label)); + } + + PAIMON_ASSIGN_OR_RAISE( + std::string key_name, + GetRequiredStringMember(obj, std::string(::lumina::core::kExtensionTagKName), tag_label)); + PAIMON_ASSIGN_OR_RAISE( + std::string type, + GetRequiredStringMember(obj, std::string(::lumina::core::kExtensionTagType), tag_label)); + PAIMON_ASSIGN_OR_RAISE( + std::string value_type, + GetRequiredStringMember(obj, std::string(::lumina::core::kExtensionTagVType), tag_label)); + + if (key_name.empty()) { + return Status::Invalid( + fmt::format("lumina tag_schema {} key_name must not be empty", tag_label)); + } + LuminaTagField::Type parsed_type; + if (type == std::string(::lumina::core::kExtensionTagTypeEnum)) { + parsed_type = LuminaTagField::Type::ENUM; + } else if (type == std::string(::lumina::core::kExtensionTagTypeRange)) { + parsed_type = LuminaTagField::Type::RANGE; + } else { + return Status::Invalid( + fmt::format("lumina tag_schema {} has unsupported type: {}", tag_label, type)); + } + + LuminaTagField::ValueType parsed_value_type; + if (value_type == std::string(::lumina::core::kExtensionTagVTypeInt32)) { + parsed_value_type = LuminaTagField::ValueType::INT32; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeInt64)) { + parsed_value_type = LuminaTagField::ValueType::INT64; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeFloat)) { + parsed_value_type = LuminaTagField::ValueType::FLOAT; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeDouble)) { + parsed_value_type = LuminaTagField::ValueType::DOUBLE; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeString)) { + parsed_value_type = LuminaTagField::ValueType::STRING; + } else { + return Status::Invalid(fmt::format("lumina tag_schema {} has unsupported value_type: {}", + tag_label, value_type)); + } + return LuminaTagField{key_name, parsed_type, parsed_value_type}; +} + +Status ValidateTagArrowType(const LuminaTagField& tag_field, + const std::shared_ptr& field_type) { + auto value_type = field_type; + if (auto list_type = std::dynamic_pointer_cast(field_type)) { + value_type = list_type->value_type(); + } + + bool compatible = false; + switch (tag_field.value_type) { + case LuminaTagField::ValueType::INT32: + compatible = value_type->id() == arrow::Type::INT8 || + value_type->id() == arrow::Type::INT16 || + value_type->id() == arrow::Type::INT32; + break; + case LuminaTagField::ValueType::INT64: + compatible = value_type->id() == arrow::Type::INT64; + break; + case LuminaTagField::ValueType::FLOAT: + compatible = value_type->id() == arrow::Type::FLOAT; + break; + case LuminaTagField::ValueType::DOUBLE: + compatible = value_type->id() == arrow::Type::DOUBLE; + break; + case LuminaTagField::ValueType::STRING: + compatible = value_type->id() == arrow::Type::STRING; + break; + } + if (!compatible) { + return Status::Invalid( + fmt::format("lumina tag field {} type {} is not compatible with tag_schema value_type", + tag_field.name, field_type->ToString())); + } + return Status::OK(); +} + +template +void AppendPrimitiveTagValue(const std::shared_ptr& array, int64_t index, + std::vector* values) { + values->push_back( + static_cast(static_cast(array.get())->Value(index))); +} + +template +Status AppendTagValue(const std::shared_ptr& array, int64_t index, + std::vector* values) { + if (array->IsNull(index)) { + return Status::OK(); + } + + auto validate_array_type = [&](arrow::Type::type expected_type, + const char* value_type_name) -> Status { + if (array->type_id() != expected_type) { + return Status::Invalid(fmt::format("lumina {} tag field has unsupported arrow type {}", + value_type_name, array->type()->ToString())); + } + return Status::OK(); + }; + + if constexpr (std::is_same_v) { + switch (array->type_id()) { + case arrow::Type::INT8: + AppendPrimitiveTagValue(array, index, values); + break; + case arrow::Type::INT16: + AppendPrimitiveTagValue(array, index, values); + break; + case arrow::Type::INT32: + AppendPrimitiveTagValue(array, index, values); + break; + default: + return Status::Invalid( + fmt::format("lumina integer tag field has unsupported arrow type {}", + array->type()->ToString())); + } + } else if constexpr (std::is_same_v) { + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::INT64, "int64")); + AppendPrimitiveTagValue(array, index, values); + } else if constexpr (std::is_same_v) { + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::FLOAT, "float")); + AppendPrimitiveTagValue(array, index, values); + } else if constexpr (std::is_same_v) { + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::DOUBLE, "double")); + AppendPrimitiveTagValue(array, index, values); + } else if constexpr (std::is_same_v) { + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::STRING, "string")); + auto string_array = static_cast(array.get()); + auto view = string_array->GetView(index); + values->emplace_back(view.data(), view.size()); + } else { + return Status::Invalid("lumina tag field has unsupported value type"); + } + return Status::OK(); +} + +template +Status ExtractTagValues(const std::shared_ptr& field_array, int64_t segment_start, + int64_t segment_len, std::vector>* values) { + values->resize(segment_len); + auto list_array = std::dynamic_pointer_cast(field_array); + if (list_array) { + auto child_values = list_array->values(); + for (int64_t i = 0; i < segment_len; i++) { + int64_t row = segment_start + i; + if (list_array->IsNull(row)) { + continue; + } + auto value_start = list_array->value_offset(row); + auto value_end = list_array->value_offset(row + 1); + auto& row_values = (*values)[i]; + row_values.reserve(value_end - value_start); + for (int64_t value_index = value_start; value_index < value_end; value_index++) { + PAIMON_RETURN_NOT_OK(AppendTagValue(child_values, value_index, &row_values)); + } + } + return Status::OK(); + } + + for (int64_t i = 0; i < segment_len; i++) { + PAIMON_RETURN_NOT_OK(AppendTagValue(field_array, segment_start + i, &(*values)[i])); + } + return Status::OK(); +} + +Result LiteralToTagValue(const Literal& literal) { + if (literal.IsNull()) { + return Status::Invalid("lumina tag predicate does not support null literal"); + } + switch (literal.GetType()) { + case FieldType::TINYINT: + return TagValue(static_cast(literal.GetValue())); + case FieldType::SMALLINT: + return TagValue(static_cast(literal.GetValue())); + case FieldType::INT: + return TagValue(literal.GetValue()); + case FieldType::BIGINT: + return TagValue(literal.GetValue()); + case FieldType::FLOAT: + return TagValue(literal.GetValue()); + case FieldType::DOUBLE: + return TagValue(literal.GetValue()); + case FieldType::STRING: + return TagValue(literal.GetValue()); + default: + return Status::Invalid( + fmt::format("lumina tag predicate does not support literal type {}", + static_cast(literal.GetType()))); + } +} + +Result GetSingleLiteral(const std::vector& literals, + const std::string& function_name) { + if (literals.size() != 1) { + return Status::Invalid( + fmt::format("lumina tag {} predicate requires one literal", function_name)); + } + return &literals[0]; +} + +Result LiteralsToTagValues(const std::vector& literals) { + if (literals.empty()) { + return Status::Invalid("lumina tag predicate IN requires at least one literal"); + } + + switch (literals[0].GetType()) { + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::BIGINT: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::FLOAT: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::DOUBLE: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::STRING: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(std::move(*typed_value)); + } + return TagValues(std::move(values)); + } + default: + return Status::Invalid( + fmt::format("lumina tag predicate IN does not support literal type {}", + static_cast(literals[0].GetType()))); + } +} + +} // namespace + +Result> LuminaIndexWriter::ExtractTagDataForSegment( + const std::shared_ptr& struct_array, + const std::vector& tag_fields, int64_t segment_start, int64_t segment_len) { + std::vector tag_dimensions_data; + tag_dimensions_data.reserve(tag_fields.size()); + for (const auto& tag_field : tag_fields) { + auto field_array = struct_array->GetFieldByName(tag_field.name); + CHECK_NOT_NULL(field_array, + fmt::format("lumina tag field {} not in input array", tag_field.name)); + + TagDimensionData tag_dimension_data; + tag_dimension_data.tagkName = tag_field.name; + switch (tag_field.value_type) { + case LuminaTagField::ValueType::INT32: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::INT64: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::FLOAT: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::DOUBLE: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::STRING: { + std::vector> values; + PAIMON_RETURN_NOT_OK(ExtractTagValues(field_array, segment_start, + segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } + } + tag_dimensions_data.push_back(std::move(tag_dimension_data)); + } + return tag_dimensions_data; +} + +Result> LuminaGlobalIndex::ParseTagSchema( + const std::map& lumina_options) { + auto iter = lumina_options.find(std::string(::lumina::core::kExtensionTagSchema)); + if (iter == lumina_options.end()) { + return std::vector(); + } + + rapidjson::Document document; + document.Parse(iter->second.c_str()); + if (document.HasParseError()) { + return Status::Invalid("lumina tag_schema must be a valid JSON string"); + } + + std::vector tag_fields; + if (document.IsArray()) { + if (document.Empty()) { + return Status::Invalid("lumina tag_schema must contain at least one tag definition"); + } + tag_fields.reserve(document.Size()); + for (rapidjson::SizeType i = 0; i < document.Size(); i++) { + PAIMON_ASSIGN_OR_RAISE(LuminaTagField field, + ParseTagField(document[i], fmt::format("tag[{}]", i))); + tag_fields.push_back(std::move(field)); + } + } else if (document.IsObject()) { + PAIMON_ASSIGN_OR_RAISE(LuminaTagField field, ParseTagField(document, "tag[0]")); + tag_fields.push_back(std::move(field)); + } else { + return Status::Invalid("lumina tag_schema must be an object or array of objects"); + } + + std::unordered_set seen_names; + for (const auto& field : tag_fields) { + if (!seen_names.insert(field.name).second) { + return Status::Invalid( + fmt::format("lumina tag_schema has duplicate key_name: {}", field.name)); + } + } + return tag_fields; +} + +Status LuminaGlobalIndex::ValidateTagFields(const arrow::StructType& struct_type, + const std::vector& tag_fields) { + for (const auto& tag_field : tag_fields) { + auto field = struct_type.GetFieldByName(tag_field.name); + CHECK_NOT_NULL( + field, fmt::format("lumina tag field {} not exist in arrow schema", tag_field.name)); + PAIMON_RETURN_NOT_OK(ValidateTagArrowType(tag_field, field->type())); + } + return Status::OK(); +} + +Result<::lumina::extensions::experimental::TagFilter> LuminaIndexReader::PredicateToTagFilter( + const std::shared_ptr& predicate) { + if (!predicate) { + return Status::Invalid("lumina tag predicate must not be null"); + } + + auto compound_predicate = std::dynamic_pointer_cast(predicate); + if (compound_predicate) { + std::vector<::lumina::extensions::experimental::TagFilter> children; + children.reserve(compound_predicate->Children().size()); + for (const auto& child : compound_predicate->Children()) { + PAIMON_ASSIGN_OR_RAISE(::lumina::extensions::experimental::TagFilter tag_filter, + PredicateToTagFilter(child)); + children.push_back(std::move(tag_filter)); + } + if (children.empty()) { + return Status::Invalid("lumina tag compound predicate must have at least one child"); + } + if (children.size() == 1) { + return std::move(children.front()); + } + switch (compound_predicate->GetFunction().GetType()) { + case Function::Type::AND: + return ::lumina::extensions::experimental::TagFilter::And(std::move(children)); + case Function::Type::OR: + return ::lumina::extensions::experimental::TagFilter::Or(std::move(children)); + default: + return Status::NotImplemented( + fmt::format("lumina tag predicate does not support compound function {}", + compound_predicate->GetFunction().ToString())); + } + } + + auto leaf_predicate = std::dynamic_pointer_cast(predicate); + if (!leaf_predicate) { + return Status::Invalid( + fmt::format("cannot cast predicate {} to CompoundPredicate or LeafPredicate", + predicate->ToString())); + } + + const auto& literals = leaf_predicate->Literals(); + const auto& field_name = leaf_predicate->FieldName(); + switch (leaf_predicate->GetFunction().GetType()) { + case Function::Type::EQUAL: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, GetSingleLiteral(literals, "equal")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Eq(field_name, std::move(value)); + } + case Function::Type::GREATER_THAN: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, + GetSingleLiteral(literals, "greater than")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Gt(field_name, std::move(value)); + } + case Function::Type::GREATER_OR_EQUAL: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, + GetSingleLiteral(literals, "greater or equal")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Gte(field_name, std::move(value)); + } + case Function::Type::LESS_THAN: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, GetSingleLiteral(literals, "less than")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Lt(field_name, std::move(value)); + } + case Function::Type::LESS_OR_EQUAL: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, + GetSingleLiteral(literals, "less or equal")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Lte(field_name, std::move(value)); + } + case Function::Type::IN: { + PAIMON_ASSIGN_OR_RAISE(TagValues values, LiteralsToTagValues(literals)); + return ::lumina::extensions::experimental::TagFilter::In(field_name, std::move(values)); + } + default: + return Status::NotImplemented( + fmt::format("lumina tag predicate does not support leaf function {}", + leaf_predicate->GetFunction().ToString())); + } +} + Result> LuminaGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, @@ -67,6 +571,8 @@ Result> LuminaGlobalIndex::CreateWriter( // check options auto lumina_options = OptionsUtils::FetchOptionsWithPrefix(LuminaDefines::kOptionKeyPrefix, options_); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_fields, ParseTagSchema(lumina_options)); + PAIMON_RETURN_NOT_OK(ValidateTagFields(*struct_type, tag_fields)); PAIMON_ASSIGN_OR_RAISE(uint32_t dimension, OptionsUtils::GetValueFromMap( lumina_options, std::string(::lumina::core::kDimension))); @@ -78,7 +584,7 @@ Result> LuminaGlobalIndex::CreateWriter( auto lumina_pool = std::make_shared(pool); return std::make_shared( field_name, arrow_type, dimension, file_writer, std::move(builder_options), - ::lumina::api::IOOptions(), lumina_options, lumina_pool); + ::lumina::api::IOOptions(), lumina_options, std::move(tag_fields), lumina_pool); } Result LuminaIndexReader::GetIndexInfo( @@ -113,7 +619,9 @@ Result LuminaIndexReader::GetIndexInfo( return Status::Invalid( fmt::format("invalid distance type {} for lumina", distance_type_str)); } - return LuminaIndexReader::IndexInfo({dimension, index_type, distance_type}); + bool has_tag = lumina_write_options.find(std::string(::lumina::core::kExtensionTagSchema)) != + lumina_write_options.end(); + return LuminaIndexReader::IndexInfo({dimension, index_type, distance_type, has_tag}); } Result> LuminaGlobalIndex::CreateReader( @@ -172,8 +680,30 @@ Result> LuminaGlobalIndex::CreateReader( } auto searcher_with_filter = std::make_unique<::lumina::extensions::SearchWithFilterExtension>(); PAIMON_RETURN_NOT_OK_FROM_LUMINA(searcher->Attach(*searcher_with_filter)); + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension> searcher_with_tag; + if (index_info.has_tag) { + searcher_with_tag = + std::make_unique<::lumina::extensions::experimental::SearchWithTagExtension>(); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(searcher->Attach(*searcher_with_tag)); + } return std::make_shared(index_info, std::move(searcher), - std::move(searcher_with_filter), lumina_pool); + std::move(searcher_with_filter), + std::move(searcher_with_tag), lumina_pool); +} + +Result>> LuminaGlobalIndex::GetExtraFieldNames() const { + auto lumina_options = + OptionsUtils::FetchOptionsWithPrefix(LuminaDefines::kOptionKeyPrefix, options_); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_fields, ParseTagSchema(lumina_options)); + if (tag_fields.empty()) { + return std::optional>(std::nullopt); + } + std::vector field_names; + field_names.reserve(tag_fields.size()); + for (const auto& tag_field : tag_fields) { + field_names.push_back(tag_field.name); + } + return std::optional>(std::move(field_names)); } class LuminaDataset : public ::lumina::api::Dataset { @@ -201,18 +731,67 @@ class LuminaDataset : public ::lumina::api::Dataset { } auto& value_array = array_vec_[cursor_]; int64_t value_array_length = value_array->length(); - int64_t element_count = value_array_length / dimension_; + int64_t batch_element_count = value_array_length / dimension_; const float* value_ptr = value_array->raw_values(); vector_buffer.resize(value_array_length); memcpy(vector_buffer.data(), value_ptr, sizeof(float) * value_array_length); - id_buffer.resize(element_count); + id_buffer.resize(batch_element_count); std::iota(id_buffer.begin(), id_buffer.end(), static_cast<::lumina::core::vector_id_t>(start_ids_[cursor_])); // release the array when copy to vector_buffer value_array.reset(); cursor_++; - return ::lumina::core::Result::Ok(static_cast(element_count)); + return ::lumina::core::Result::Ok(static_cast(batch_element_count)); + } + + private: + int64_t element_count_; + uint32_t dimension_; + std::vector> array_vec_; + std::vector start_ids_; + size_t cursor_ = 0; +}; + +class LuminaDatasetWithTag : public ::lumina::extensions::experimental::DatasetWithTag { + public: + LuminaDatasetWithTag(int64_t element_count, uint32_t dimension, + const std::vector>& array_vec, + const std::vector& start_ids, + const std::vector>& tag_data_vec) + : element_count_(element_count), + dimension_(dimension), + array_vec_(array_vec), + start_ids_(start_ids), + tag_data_vec_(tag_data_vec) {} + + uint32_t Dim() const noexcept override { + return dimension_; + } + uint64_t TotalSize() const noexcept override { + return element_count_; + } + + ::lumina::core::Result GetNextBatch( + std::vector& vector_buffer, std::vector<::lumina::core::vector_id_t>& id_buffer, + std::vector& tag_dimensions_data) noexcept override { + if (cursor_ >= array_vec_.size()) { + return ::lumina::core::Result::Ok(0); + } + auto& value_array = array_vec_[cursor_]; + int64_t value_array_length = value_array->length(); + int64_t batch_element_count = value_array_length / dimension_; + const float* value_ptr = value_array->raw_values(); + vector_buffer.resize(value_array_length); + memcpy(vector_buffer.data(), value_ptr, sizeof(float) * value_array_length); + id_buffer.resize(batch_element_count); + std::iota(id_buffer.begin(), id_buffer.end(), + static_cast<::lumina::core::vector_id_t>(start_ids_[cursor_])); + tag_dimensions_data = std::move(tag_data_vec_[cursor_]); + + value_array.reset(); + cursor_++; + return ::lumina::core::Result::Ok(static_cast(batch_element_count)); } private: @@ -220,17 +799,16 @@ class LuminaDataset : public ::lumina::api::Dataset { uint32_t dimension_; std::vector> array_vec_; std::vector start_ids_; + std::vector> tag_data_vec_; size_t cursor_ = 0; }; -LuminaIndexWriter::LuminaIndexWriter(const std::string& field_name, - const std::shared_ptr& arrow_type, - uint32_t dimension, - const std::shared_ptr& file_manager, - ::lumina::api::BuilderOptions&& builder_options, - ::lumina::api::IOOptions&& io_options, - const std::map& lumina_options, - const std::shared_ptr& pool) +LuminaIndexWriter::LuminaIndexWriter( + const std::string& field_name, const std::shared_ptr& arrow_type, + uint32_t dimension, const std::shared_ptr& file_manager, + ::lumina::api::BuilderOptions&& builder_options, ::lumina::api::IOOptions&& io_options, + const std::map& lumina_options, + std::vector&& tag_fields, const std::shared_ptr& pool) : pool_(pool), field_name_(field_name), arrow_type_(arrow_type), @@ -238,7 +816,8 @@ LuminaIndexWriter::LuminaIndexWriter(const std::string& field_name, file_manager_(file_manager), builder_options_(std::move(builder_options)), io_options_(std::move(io_options)), - lumina_options_(lumina_options) {} + lumina_options_(lumina_options), + tag_fields_(std::move(tag_fields)) {} Status LuminaIndexWriter::AddBatch(::ArrowArray* arrow_array, std::vector&& relative_row_ids) { @@ -285,11 +864,21 @@ Status LuminaIndexWriter::AddBatch(::ArrowArray* arrow_array, return Status::Invalid( "field value array in LuminaIndexWriter is invalid, must not null"); } - if (sliced_values->length() != segment_len * static_cast(dimension_)) { - return Status::Invalid(fmt::format( - "invalid input array in LuminaIndexWriter, length of field array [{}] " - "multiplied dimension [{}] must match length of field value array [{}]", - segment_len, dimension_, sliced_values->length())); + for (int64_t row = segment_start; row < segment_start + segment_len; row++) { + int64_t vector_length = + list_field_array->value_offset(row + 1) - list_field_array->value_offset(row); + if (vector_length != static_cast(dimension_)) { + return Status::Invalid(fmt::format( + "invalid input array in LuminaIndexWriter, vector at row [{}] has length " + "[{}], expected dimension [{}]", + row, vector_length, dimension_)); + } + } + if (!tag_fields_.empty()) { + PAIMON_ASSIGN_OR_RAISE(std::vector tag_data, + ExtractTagDataForSegment(struct_array, tag_fields_, + segment_start, segment_len)); + tag_data_vec_.push_back(std::move(tag_data)); } array_vec_.push_back(std::move(sliced_values)); array_start_ids_.push_back(count_ + segment_start); @@ -315,9 +904,19 @@ Result> LuminaIndexWriter::Finish() { PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.PretrainFrom(dataset1)); // insert data - LuminaDataset dataset2(indexed_count_, dimension_, array_vec_, array_start_ids_); - std::vector>().swap(array_vec_); - PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.InsertFrom(dataset2)); + if (tag_fields_.empty()) { + LuminaDataset dataset2(indexed_count_, dimension_, array_vec_, array_start_ids_); + std::vector>().swap(array_vec_); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.InsertFrom(dataset2)); + } else { + ::lumina::extensions::experimental::BuildWithTagExtension tag_extension; + PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.Attach(tag_extension)); + LuminaDatasetWithTag dataset2(indexed_count_, dimension_, array_vec_, array_start_ids_, + tag_data_vec_); + std::vector>().swap(array_vec_); + std::vector>().swap(tag_data_vec_); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(tag_extension.InsertFromWithTag(dataset2)); + } // dump index PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, @@ -340,17 +939,16 @@ LuminaIndexReader::LuminaIndexReader( const LuminaIndexReader::IndexInfo& index_info, std::unique_ptr<::lumina::api::LuminaSearcher>&& searcher, std::unique_ptr<::lumina::extensions::SearchWithFilterExtension>&& searcher_with_filter, + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension>&& searcher_with_tag, const std::shared_ptr& pool) : index_info_(index_info), pool_(pool), searcher_(std::move(searcher)), - searcher_with_filter_(std::move(searcher_with_filter)) {} + searcher_with_filter_(std::move(searcher_with_filter)), + searcher_with_tag_(std::move(searcher_with_tag)) {} Result> LuminaIndexReader::VisitVectorSearch( const std::shared_ptr& vector_search) { - if (vector_search->predicate) { - return Status::NotImplemented("lumina index not support predicate in VisitVectorSearch"); - } if (vector_search->distance_type && vector_search->distance_type.value() != index_info_.distance_type) { return Status::Invalid("distance type for index and search not match"); @@ -377,7 +975,25 @@ Result> LuminaIndexReader::VisitVectorS ::lumina::api::Query lumina_query(vector_search->query.data(), vector_search->query.size()); ::lumina::api::LuminaSearcher::SearchResult search_result; - if (!vector_search->pre_filter) { + if (vector_search->predicate) { + if (!searcher_with_tag_) { + return Status::Invalid("lumina index was not built with tag"); + } + PAIMON_ASSIGN_OR_RAISE(::lumina::extensions::experimental::TagFilter tag_filter, + PredicateToTagFilter(vector_search->predicate)); + if (!vector_search->pre_filter) { + PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( + search_result, searcher_with_tag_->SearchWithTag(lumina_query, tag_filter, + search_options, *pool_)); + } else { + auto lumina_filter = [filter = vector_search->pre_filter]( + ::lumina::core::vector_id_t id) -> bool { return filter(id); }; + PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( + search_result, + searcher_with_tag_->SearchWithTagAndFilter(lumina_query, tag_filter, lumina_filter, + search_options, *pool_)); + } + } else if (!vector_search->pre_filter) { PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA(search_result, searcher_->Search(lumina_query, search_options, *pool_)); } else { diff --git a/src/paimon/global_index/lumina/lumina_global_index.h b/src/paimon/global_index/lumina/lumina_global_index.h index 572361a0..c2c30475 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.h +++ b/src/paimon/global_index/lumina/lumina_global_index.h @@ -20,20 +20,44 @@ #include #include +#include #include #include +#include #include #include "arrow/api.h" #include "lumina/api/LuminaSearcher.h" #include "lumina/api/Options.h" #include "lumina/extensions/SearchWithFilterExtension.h" +#include "lumina/extensions/experimental/DatasetWithTag.h" +#include "lumina/extensions/experimental/SearchWithTagExtension.h" +#include "lumina/extensions/experimental/TagFilter.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/global_indexer.h" #include "paimon/global_index/lumina/lumina_memory_pool.h" #include "paimon/global_index/lumina/lumina_utils.h" namespace paimon::lumina { +struct LuminaTagField { + enum class Type { + ENUM, + RANGE, + }; + + enum class ValueType { + INT32, + INT64, + FLOAT, + DOUBLE, + STRING, + }; + + std::string name; + Type type; + ValueType value_type; +}; + /// @note When enabling the lumina global index in `paimon-cpp`, all configuration parameters /// specific to Lumina **must be prefixed with `lumina.`**. /// See `docs/reference/OptionsReference.md` in the Lumina release package for more options. @@ -63,6 +87,8 @@ class LuminaGlobalIndex : public GlobalIndexer { explicit LuminaGlobalIndex(const std::map& options) : options_(options) {} + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, @@ -74,6 +100,12 @@ class LuminaGlobalIndex : public GlobalIndexer { const std::shared_ptr& pool) const override; private: + static Result> ParseTagSchema( + const std::map& lumina_options); + + static Status ValidateTagFields(const arrow::StructType& struct_type, + const std::vector& tag_fields); + std::map options_; }; @@ -85,6 +117,7 @@ class LuminaIndexWriter : public GlobalIndexWriter { ::lumina::api::BuilderOptions&& builder_options, ::lumina::api::IOOptions&& io_options, const std::map& lumina_options, + std::vector&& tag_fields, const std::shared_ptr& pool); Status AddBatch(::ArrowArray* arrow_array, std::vector&& relative_row_ids) override; @@ -92,6 +125,11 @@ class LuminaIndexWriter : public GlobalIndexWriter { Result> Finish() override; private: + static Result> + ExtractTagDataForSegment(const std::shared_ptr& struct_array, + const std::vector& tag_fields, int64_t segment_start, + int64_t segment_len); + int64_t count_ = 0; int64_t indexed_count_ = 0; std::shared_ptr pool_; @@ -102,8 +140,10 @@ class LuminaIndexWriter : public GlobalIndexWriter { ::lumina::api::BuilderOptions builder_options_; ::lumina::api::IOOptions io_options_; std::map lumina_options_; + std::vector tag_fields_; std::vector> array_vec_; std::vector array_start_ids_; + std::vector> tag_data_vec_; }; class LuminaIndexReader : public GlobalIndexReader { @@ -112,11 +152,14 @@ class LuminaIndexReader : public GlobalIndexReader { uint32_t dimension; std::string index_type; VectorSearch::DistanceType distance_type; + bool has_tag; }; LuminaIndexReader( const IndexInfo& index_info, std::unique_ptr<::lumina::api::LuminaSearcher>&& searcher, std::unique_ptr<::lumina::extensions::SearchWithFilterExtension>&& searcher_with_filter, + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension>&& + searcher_with_tag, const std::shared_ptr& pool); ~LuminaIndexReader() override { @@ -203,9 +246,13 @@ class LuminaIndexReader : public GlobalIndexReader { static Result GetIndexInfo(const GlobalIndexIOMeta& io_meta); private: + static Result<::lumina::extensions::experimental::TagFilter> PredicateToTagFilter( + const std::shared_ptr& predicate); + LuminaIndexReader::IndexInfo index_info_; std::shared_ptr pool_; std::unique_ptr<::lumina::api::LuminaSearcher> searcher_; std::unique_ptr<::lumina::extensions::SearchWithFilterExtension> searcher_with_filter_; + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension> searcher_with_tag_; }; } // namespace paimon::lumina diff --git a/src/paimon/global_index/lumina/lumina_global_index_test.cpp b/src/paimon/global_index/lumina/lumina_global_index_test.cpp index 0f314776..2d54950a 100644 --- a/src/paimon/global_index/lumina/lumina_global_index_test.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index_test.cpp @@ -252,6 +252,443 @@ TEST_F(LuminaGlobalIndexTest, TestWithFilter) { } } +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagFilter) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"({"key_name":"color","type":"enum","value_type":"string"})"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8())}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], "cold"], + [[0.0, 1.0, 0.0, 1.0], "warm"], + [[1.0, 0.0, 1.0, 0.0], "cold"], + [[1.0, 1.0, 1.0, 1.0], "warm"] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "warm", 4)); + { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, {3l, 1l}, {0.01f, 2.01f}); + } + { + auto pre_filter = [](int64_t id) -> bool { return id < 3; }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr filtered_scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, pre_filter, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(filtered_scored_result, {1l}, {2.01f}); + } +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithMixedTagPredicates) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"float"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), + arrow::field("price", arrow::float32())}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], "cold", 5.0], + [[0.0, 1.0, 0.0, 1.0], "warm", 10.0], + [[1.0, 0.0, 1.0, 0.0], "warm", 20.0], + [[1.0, 1.0, 1.0, 1.0], "warm", 30.0] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + std::shared_ptr color_predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "warm", 4)); + std::shared_ptr low_price_predicate = PredicateBuilder::LessOrEqual( + /*field_index=*/2, /*field_name=*/"price", FieldType::FLOAT, Literal(10.0f)); + std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/2, /*field_name=*/"price", FieldType::FLOAT, Literal(30.0f)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr price_predicate, + PredicateBuilder::Or({low_price_predicate, high_price_predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({color_predicate, price_predicate})); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, {3l, 1l}, {0.01f, 2.01f}); +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithIntegerListTagFilter) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"category_ids","type":"enum","value_type":"int32"}])"; + + std::shared_ptr tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("category_ids", arrow::list(arrow::int32()))}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], [1, 2]], + [[0.0, 1.0, 0.0, 1.0], [3, 8]], + [[1.0, 0.0, 1.0, 0.0], [4, 5]], + [[1.0, 1.0, 1.0, 1.0], [6, 9]] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + std::shared_ptr predicate = PredicateBuilder::In( + /*field_index=*/1, /*field_name=*/"category_ids", FieldType::INT, {Literal(8), Literal(9)}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, {3l, 1l}, {0.01f, 2.01f}); +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithCompatibleTagArrowTypes) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"tag_i8","type":"enum","value_type":"int32"},)" + R"({"key_name":"tag_i16","type":"enum","value_type":"int32"},)" + R"({"key_name":"tag_i32","type":"range","value_type":"int32"},)" + R"({"key_name":"tag_i64","type":"enum","value_type":"int64"},)" + R"({"key_name":"tag_f32","type":"range","value_type":"float"},)" + R"({"key_name":"tag_f64","type":"enum","value_type":"double"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("tag_i8", arrow::int8()), + arrow::field("tag_i16", arrow::int16()), arrow::field("tag_i32", arrow::int32()), + arrow::field("tag_i64", arrow::int64()), arrow::field("tag_f32", arrow::float32()), + arrow::field("tag_f64", arrow::float64())}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], 1, 10, 100, 10000000001, 1.5, 1.25], + [[0.0, 1.0, 0.0, 1.0], 2, 20, 200, 10000000002, 2.5, 2.25], + [[1.0, 0.0, 1.0, 0.0], 3, 30, 300, 10000000003, 3.5, 3.25], + [[1.0, 1.0, 1.0, 1.0], 4, 40, 400, 10000000004, 4.5, 4.25] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::vector& expected_ids, + const std::vector& expected_scores) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, expected_ids, expected_scores); + }; + + search_and_check(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"tag_i8", + FieldType::TINYINT, Literal(static_cast(2))), + {1l}, {2.01f}); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"tag_i16", FieldType::SMALLINT, + Literal(static_cast(30))), + {2l}, {2.21f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"tag_i32", + FieldType::INT, Literal(250)), + {3l, 2l}, {0.01f, 2.21f}); + search_and_check(PredicateBuilder::Equal( + /*field_index=*/4, /*field_name=*/"tag_i64", FieldType::BIGINT, + Literal(static_cast(10000000003))), + {2l}, {2.21f}); + search_and_check(PredicateBuilder::In( + /*field_index=*/4, /*field_name=*/"tag_i64", FieldType::BIGINT, + {Literal(static_cast(10000000001)), + Literal(static_cast(10000000004))}), + {3l, 0l}, {0.01f, 4.21f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/5, /*field_name=*/"tag_f32", + FieldType::FLOAT, Literal(4.0f)), + {3l}, {0.01f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/6, /*field_name=*/"tag_f64", + FieldType::DOUBLE, Literal(3.25)), + {2l}, {2.21f}); + search_and_check(PredicateBuilder::In(/*field_index=*/6, /*field_name=*/"tag_f64", + FieldType::DOUBLE, {Literal(2.25), Literal(4.25)}), + {3l, 1l}, {0.01f, 2.01f}); +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagNullAndEmptyValues) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"labels","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"float"},)" + R"({"key_name":"scores","type":"enum","value_type":"float"},)" + R"({"key_name":"category","type":"enum","value_type":"int32"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), + arrow::field("labels", arrow::list(arrow::utf8())), + arrow::field("price", arrow::float32()), + arrow::field("scores", arrow::list(arrow::float32())), + arrow::field("category", arrow::int32()), + arrow::field("category_ids", arrow::list(arrow::int32()))}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], "red", ["hot"], 5.0, [0.25], 7, [1, 2]], + [null, "red", ["vip"], 8.0, [0.8], 7, [9]], + [[0.0, 1.0, 0.0, 1.0], null, null, null, null, null, null], + [[1.0, 0.0, 1.0, 0.0], " ", [], 20.0, [], 0, []], + [[1.0, 1.0, 1.0, 1.0], "blue", ["vip", null], 30.0, [null, 0.75], 3, [null, 9]] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 4))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + auto search_and_check_with_filter = + [&](VectorSearch::PreFilter pre_filter, const std::shared_ptr& predicate, + const std::vector& expected_ids, const std::vector& expected_scores) { + std::shared_ptr vector_search = std::make_shared( + /*field_name=*/"f0", /*limit=*/5, query_, pre_filter, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scored_result, + reader->VisitVectorSearch(vector_search)); + CheckResult(scored_result, expected_ids, expected_scores); + }; + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::vector& expected_ids, + const std::vector& expected_scores) { + search_and_check_with_filter(/*pre_filter=*/nullptr, predicate, expected_ids, + expected_scores); + }; + auto search_and_check_error = [&](const std::shared_ptr& predicate, + const std::string& expected_message) { + ASSERT_NOK_WITH_MSG( + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/5, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options)), + expected_message); + }; + + search_and_check(/*predicate=*/nullptr, {4l, 2l, 3l, 0l}, {0.01f, 2.01f, 2.21f, 4.21f}); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", + FieldType::STRING, Literal(FieldType::STRING, " ", 1)), + {3l}, {2.21f}); + search_and_check( + PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"labels", FieldType::STRING, + {Literal(FieldType::STRING, "vip", 3)}), + {4l}, {0.01f}); + search_and_check(PredicateBuilder::LessOrEqual(/*field_index=*/3, /*field_name=*/"price", + FieldType::FLOAT, Literal(10.0f)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::LessThan(/*field_index=*/3, /*field_name=*/"price", + FieldType::FLOAT, Literal(10.0f)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"price", + FieldType::FLOAT, Literal(10.0f)), + {4l, 3l}, {0.01f, 2.21f}); + search_and_check(PredicateBuilder::In(/*field_index=*/4, /*field_name=*/"scores", + FieldType::FLOAT, {Literal(0.25f), Literal(0.75f)}), + {4l, 0l}, {0.01f, 4.21f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/5, /*field_name=*/"category", + FieldType::INT, Literal(7)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::In(/*field_index=*/6, /*field_name=*/"category_ids", + FieldType::INT, {Literal(9)}), + {4l}, {0.01f}); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "green", 5)), + {}, {}); + search_and_check_error( + PredicateBuilder::NotIn(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + {Literal(FieldType::STRING, "red", 3)}), + "lumina tag predicate does not support leaf function NotIn"); + search_and_check_error( + PredicateBuilder::NotEqual(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "lumina tag predicate does not support leaf function NotEqual"); + search_and_check_error( + PredicateBuilder::Equal(/*field_index=*/7, /*field_name=*/"unknown", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "unknown tag key 'unknown' in label filter"); + search_and_check_error(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", + FieldType::INT, Literal(1)), + "tag value type mismatch for key 'color'"); + search_and_check_error( + PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"price", FieldType::STRING, + Literal(FieldType::STRING, "x", 1)), + "tag value type mismatch for key 'price'"); + { + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + search_and_check_with_filter([](int64_t id) { return id == 0 || id == 4; }, predicate, {0l}, + {4.21f}); + search_and_check_with_filter([](int64_t id) { return id == 4; }, predicate, {}, {}); + } + { + std::shared_ptr red_predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + std::shared_ptr blue_predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "blue", 4)); + std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/3, /*field_name=*/"price", FieldType::FLOAT, Literal(30.0f)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr blue_high_price_predicate, + PredicateBuilder::And({blue_predicate, high_price_predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr compound_predicate, + PredicateBuilder::Or({red_predicate, blue_high_price_predicate})); + search_and_check(compound_predicate, {0l, 4l}, {4.21f, 0.01f}); + search_and_check_with_filter([](int64_t id) { return id == 4; }, compound_predicate, {4l}, + {0.01f}); + } +} + +TEST_F(LuminaGlobalIndexTest, TestTagSchemaValidation) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string index_root = test_root_dir->Str(); + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8())}); + + { + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"range","value_type":"string"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, tag_data_type, tag_options, array_, Range(0, 3)), + "Option extension.build.tag.tag_schema tag[0] range type does not support value_type " + "'string'"); + } + { + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"int32"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, tag_data_type, tag_options, array_, Range(0, 3)), + "lumina tag field color type string is not compatible with tag_schema value_type"); + } + { + std::shared_ptr int64_tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("category", arrow::int64())}); + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"category","type":"enum","value_type":"int32"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, int64_tag_data_type, tag_options, array_, Range(0, 3)), + "lumina tag field category type int64 is not compatible with tag_schema value_type"); + } + { + std::shared_ptr double_tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("price", arrow::float64())}); + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"price","type":"range","value_type":"double"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, double_tag_data_type, tag_options, array_, Range(0, 3)), + "Option extension.build.tag.tag_schema tag[0] range type does not support value_type " + "'double'"); + } + { + std::shared_ptr int64_tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("price", arrow::int64())}); + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"price","type":"range","value_type":"int64"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, int64_tag_data_type, tag_options, array_, Range(0, 3)), + "Option extension.build.tag.tag_schema tag[0] range type does not support value_type " + "'int64'"); + } +} + +TEST_F(LuminaGlobalIndexTest, TestGetExtraFieldNames) { + { + LuminaGlobalIndex global_index(options_); + ASSERT_OK_AND_ASSIGN(std::optional> field_names, + global_index.GetExtraFieldNames()); + ASSERT_FALSE(field_names); + } + { + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"float"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"; + LuminaGlobalIndex global_index(tag_options); + ASSERT_OK_AND_ASSIGN(std::optional> field_names, + global_index.GetExtraFieldNames()); + ASSERT_TRUE(field_names); + ASSERT_EQ(field_names.value(), + std::vector({"color", "price", "category_ids"})); + } +} + TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(test_root_dir); @@ -306,8 +743,20 @@ TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { .ValueOrDie(); ASSERT_NOK_WITH_MSG( WriteGlobalIndex(index_root, data_type_, options_, array, Range(0, 2)), - "invalid input array in LuminaIndexWriter, length of field array [2] multiplied " - "dimension [4] must match length of field value array [7]"); + "invalid input array in LuminaIndexWriter, vector at row [1] has length [3], " + "expected dimension [4]"); + } + { + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, + R"([ + [[0.0, 0.0, 0.0]], + [[0.0, 1.0, 0.0, 1.0, 0.0]] + ])") + .ValueOrDie(); + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, data_type_, options_, array, Range(0, 2)), + "invalid input array in LuminaIndexWriter, vector at row [0] has length [3], " + "expected dimension [4]"); } { @@ -374,10 +823,10 @@ TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { "f1", /*limit=*/2, query_, /*filter=*/nullptr, PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)), + FieldType::INT, Literal(5)), /*distance_type=*/std::nullopt, /*options=*/std::map())), - "lumina index not support predicate in VisitVectorSearch"); + "lumina index was not built with tag"); } { ASSERT_OK_AND_ASSIGN(auto reader, diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.cpp b/src/paimon/global_index/tantivy/tantivy_global_index.cpp index 77980448..f281c035 100644 --- a/src/paimon/global_index/tantivy/tantivy_global_index.cpp +++ b/src/paimon/global_index/tantivy/tantivy_global_index.cpp @@ -37,6 +37,10 @@ namespace paimon::tantivy { TantivyGlobalIndex::TantivyGlobalIndex(const std::map& options) : options_(OptionsUtils::FetchOptionsWithPrefix(kOptionKeyPrefix, options)) {} +Result>> TantivyGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> TantivyGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.h b/src/paimon/global_index/tantivy/tantivy_global_index.h index 747a2156..4b7b944f 100644 --- a/src/paimon/global_index/tantivy/tantivy_global_index.h +++ b/src/paimon/global_index/tantivy/tantivy_global_index.h @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -37,6 +38,8 @@ class TantivyGlobalIndex : public GlobalIndexer { public: explicit TantivyGlobalIndex(const std::map& options); + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 3cb84874..5ea50d9d 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -337,8 +337,76 @@ TEST_P(GlobalIndexTest, TestWriteLuminaIndexWithMismatchedDimension) { ASSERT_NOK_WITH_MSG( WriteIndex(table_path, /*partition_filters=*/{}, "f1", "lumina", /*options=*/lumina_options, Range(0, 1)), - "invalid input array in LuminaIndexWriter, length of field array [1] multiplied " - "dimension [3] must match length of field value array [4]"); + "invalid input array in LuminaIndexWriter, vector at row [0] has length [4], " + "expected dimension [3]"); +} + +TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithOrcDictionaryStringTags) { + if (file_format_ != "orc") { + GTEST_SKIP() << "ORC-only dictionary encoding case"; + } + + arrow::FieldVector fields = {arrow::field("embedding", arrow::list(arrow::float32())), + arrow::field("color", arrow::utf8()), + arrow::field("labels", arrow::list(arrow::utf8()))}; + std::shared_ptr schema = arrow::schema(fields); + std::map lumina_options = { + {"lumina.index.dimension", "4"}, + {"lumina.index.type", "bruteforce"}, + {"lumina.distance.metric", "l2"}, + {"lumina.encoding.type", "rawf32"}, + {"lumina.search.parallel_number", "10"}, + {"lumina.extension.build.tag.tag_schema", + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"labels","type":"enum","value_type":"string"}])"}}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, {"orc.dictionary-key-size-threshold", "1"}, + {"orc.read.enable-lazy-decoding", "true"}}; + CreateTable(/*partition_keys=*/{}, schema, options); + + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + std::shared_ptr src_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +[[0.0, 0.0, 0.0, 0.0], "red", ["hot", "common"]], +[[0.0, 1.0, 0.0, 1.0], "blue", ["cold", "common"]], +[[1.0, 0.0, 1.0, 0.0], "red", ["cold"]], +[[1.0, 1.0, 1.0, 1.0], "blue", ["hot"]] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::vector> commit_msgs, + WriteArray(table_path, schema->field_names(), src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "embedding", "lumina", + lumina_options, Range(0, 3))); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, lumina_options, fs_, + /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(std::vector> lumina_readers, + global_index_scan->CreateReaders("embedding", std::nullopt)); + ASSERT_EQ(lumina_readers.size(), 1u); + + std::vector query = {1.0f, 1.0f, 1.0f, 1.1f}; + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::string& expected) { + std::shared_ptr vector_search = std::make_shared( + "embedding", /*limit=*/4, query, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/lumina_options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scored_result, + lumina_readers[0]->VisitVectorSearch(vector_search)); + ASSERT_EQ(scored_result->ToString(), expected); + }; + search_and_check( + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "row ids: {0,2}, scores: {4.21,2.21}"); + search_and_check( + PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"labels", FieldType::STRING, + {Literal(FieldType::STRING, "hot", 3)}), + "row ids: {0,3}, scores: {4.21,0.01}"); } TEST_P(GlobalIndexTest, TestWriteIndex) { @@ -1150,6 +1218,158 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithScore) { ASSERT_FALSE(typed_result->bitmap_.Contains(7)); } } + +TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { + arrow::FieldVector fields = {arrow::field("name", arrow::utf8()), + arrow::field("embedding", arrow::list(arrow::float32())), + arrow::field("color", arrow::utf8()), + arrow::field("labels", arrow::list(arrow::utf8())), + arrow::field("price", arrow::float32()), + arrow::field("scores", arrow::list(arrow::float32())), + arrow::field("category", arrow::int32()), + arrow::field("category_ids", arrow::list(arrow::int32()))}; + auto schema = arrow::schema(fields); + std::map lumina_options = { + {"lumina.index.dimension", "4"}, + {"lumina.index.type", "bruteforce"}, + {"lumina.distance.metric", "l2"}, + {"lumina.encoding.type", "rawf32"}, + {"lumina.search.parallel_number", "10"}, + {"lumina.extension.build.tag.tag_schema", + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"labels","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"float"},)" + R"({"key_name":"scores","type":"range","value_type":"float"},)" + R"({"key_name":"category","type":"enum","value_type":"int32"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"}}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(/*partition_keys=*/{}, schema, options); + + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + std::vector write_cols = schema->field_names(); + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +["row0", [0.0, 0.0, 0.0, 0.0], "red", ["hot"], 5.0, [0.25], 7, [1, 2]], +["row1", null, "red", ["vip"], 8.0, [0.8], 7, [9]], +["row2", [0.0, 1.0, 0.0, 1.0], null, null, null, null, null, null], +["row3", [1.0, 0.0, 1.0, 0.0], " ", [], 20.0, [], 0, []], +["row4", [1.0, 1.0, 1.0, 1.0], "blue", ["vip", null], 30.0, [null, 0.75], 3, [null, 9]] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols, src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + ScanData(table_path, /*partition_filters=*/{})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr index_commit_msg, + GlobalIndexWriteTask::WriteIndex( + table_path, "embedding", "lumina", + std::make_shared(split, std::vector({Range(0, 4)})), + /*options=*/lumina_options, pool_, fs_)); + + std::shared_ptr index_commit_msg_impl = + std::dynamic_pointer_cast(index_commit_msg); + ASSERT_TRUE(index_commit_msg_impl); + const auto& new_index_files = index_commit_msg_impl->GetNewFilesIncrement().NewIndexFiles(); + ASSERT_EQ(new_index_files.size(), 1u); + ASSERT_EQ(new_index_files[0]->IndexType(), "lumina"); + ASSERT_EQ(new_index_files[0]->RowCount(), 5); + const std::optional& global_index_meta = + new_index_files[0]->GetGlobalIndexMeta(); + ASSERT_TRUE(global_index_meta); + std::string expected_index_meta_json = + R"({"distance.metric":"l2","encoding.type":"rawf32","extension.build.tag.tag_schema":"[{\"key_name\":\"color\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"labels\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"price\",\"type\":\"range\",\"value_type\":\"float\"},{\"key_name\":\"scores\",\"type\":\"range\",\"value_type\":\"float\"},{\"key_name\":\"category\",\"type\":\"enum\",\"value_type\":\"int32\"},{\"key_name\":\"category_ids\",\"type\":\"enum\",\"value_type\":\"int32\"}]","index.dimension":"4","index.type":"bruteforce","search.parallel_number":"10"})"; + GlobalIndexMeta expected_global_index_meta( + /*row_range_start=*/0, /*row_range_end=*/4, /*index_field_id=*/1, + /*extra_field_ids=*/std::optional>({2, 3, 4, 5, 6, 7}), + std::make_shared(expected_index_meta_json, pool_.get())); + ASSERT_EQ(global_index_meta.value(), expected_global_index_meta); + + ASSERT_OK(Commit(table_path, {index_commit_msg})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, lumina_options, fs_, + /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(auto lumina_readers, + global_index_scan->CreateReaders("embedding", std::nullopt)); + ASSERT_EQ(lumina_readers.size(), 1u); + + std::vector query = {1.0f, 1.0f, 1.0f, 1.1f}; + auto search_and_check_with_filter = [&](VectorSearch::PreFilter pre_filter, + const std::shared_ptr& predicate, + const std::string& expected) { + std::shared_ptr vector_search = std::make_shared( + "embedding", /*limit=*/5, query, pre_filter, predicate, + /*distance_type=*/std::nullopt, /*options=*/lumina_options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scored_result, + lumina_readers[0]->VisitVectorSearch(vector_search)); + ASSERT_EQ(scored_result->ToString(), expected); + }; + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::string& expected) { + search_and_check_with_filter(/*pre_filter=*/nullptr, predicate, expected); + }; + + search_and_check(/*predicate=*/nullptr, "row ids: {0,2,3,4}, scores: {4.21,2.01,2.21,0.01}"); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "row ids: {0}, scores: {4.21}"); + search_and_check(PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", + FieldType::STRING, Literal(FieldType::STRING, " ", 1)), + "row ids: {3}, scores: {2.21}"); + search_and_check( + PredicateBuilder::In(/*field_index=*/3, /*field_name=*/"labels", FieldType::STRING, + {Literal(FieldType::STRING, "vip", 3)}), + "row ids: {4}, scores: {0.01}"); + search_and_check(PredicateBuilder::LessOrEqual(/*field_index=*/4, /*field_name=*/"price", + FieldType::FLOAT, Literal(10.0f)), + "row ids: {0}, scores: {4.21}"); + search_and_check(PredicateBuilder::GreaterOrEqual(/*field_index=*/5, /*field_name=*/"scores", + FieldType::FLOAT, Literal(0.5f)), + "row ids: {4}, scores: {0.01}"); + search_and_check(PredicateBuilder::Equal(/*field_index=*/6, /*field_name=*/"category", + FieldType::INT, Literal(7)), + "row ids: {0}, scores: {4.21}"); + search_and_check(PredicateBuilder::In(/*field_index=*/7, /*field_name=*/"category_ids", + FieldType::INT, {Literal(9)}), + "row ids: {4}, scores: {0.01}"); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "green", 5)), + "row ids: {}, scores: {}"); + { + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + search_and_check_with_filter([](int64_t id) { return id == 0 || id == 4; }, predicate, + "row ids: {0}, scores: {4.21}"); + search_and_check_with_filter([](int64_t id) { return id == 4; }, predicate, + "row ids: {}, scores: {}"); + } + { + std::shared_ptr red_predicate = PredicateBuilder::Equal( + /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + std::shared_ptr blue_predicate = PredicateBuilder::Equal( + /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "blue", 4)); + std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/4, /*field_name=*/"price", FieldType::FLOAT, Literal(30.0f)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr blue_high_price_predicate, + PredicateBuilder::And({blue_predicate, high_price_predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr compound_predicate, + PredicateBuilder::Or({red_predicate, blue_high_price_predicate})); + search_and_check(compound_predicate, "row ids: {0,4}, scores: {4.21,0.01}"); + search_and_check_with_filter([](int64_t id) { return id == 4; }, compound_predicate, + "row ids: {4}, scores: {0.01}"); + } +} #endif TEST_P(GlobalIndexTest, TestDataEvolutionBatchScan) { diff --git a/third_party/versions.txt b/third_party/versions.txt index 1266761d..17fa9b3a 100644 --- a/third_party/versions.txt +++ b/third_party/versions.txt @@ -88,8 +88,8 @@ PAIMON_RAPIDJSON_BUILD_VERSION=232389d4f1012dddec4ef84861face2d2ba85709 PAIMON_RAPIDJSON_BUILD_SHA256_CHECKSUM=b9290a9a6d444c8e049bd589ab804e0ccf2b05dc5984a19ed5ae75d090064806 PAIMON_RAPIDJSON_PKG_NAME=rapidjson-${PAIMON_RAPIDJSON_BUILD_VERSION}.tar.gz -PAIMON_LUMINA_BUILD_VERSION=0.3.0-rc1 -PAIMON_LUMINA_BUILD_SHA256_CHECKSUM=6bdb9eeeeb0c6192e480ea0523712df6f07ea58521ca5ca1abc023673dfa1fa5 +PAIMON_LUMINA_BUILD_VERSION=0.3.1 +PAIMON_LUMINA_BUILD_SHA256_CHECKSUM=2aed1ae238c866c12ee01fad17d52651a9029cba1b972b23cd640959310e2149 PAIMON_LUMINA_PKG_NAME=lumina_release-${PAIMON_LUMINA_BUILD_VERSION}.tar.gz PAIMON_JINDOSDK_C_BUILD_VERSION=6.10.2 From 90c3bb7070a4e473fe213e57bd3458cd7cc7ed1b Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:44:32 +0800 Subject: [PATCH 113/138] fix(reader): ensure batch readers return zero-offset Arrow arrays --- include/paimon/reader/batch_reader.h | 4 ++ src/paimon/common/utils/arrow/arrow_utils.cpp | 39 +++++++++++++++ src/paimon/common/utils/arrow/arrow_utils.h | 3 ++ .../common/utils/arrow/arrow_utils_test.cpp | 49 +++++++++++++++++++ .../format/parquet/file_reader_wrapper.cpp | 5 ++ .../page_filtered_row_group_reader.cpp | 32 +++++++++--- .../page_filtered_row_group_reader_test.cpp | 24 +++++++-- .../testing/utils/read_result_collector.h | 19 +++++++ 8 files changed, 164 insertions(+), 11 deletions(-) diff --git a/include/paimon/reader/batch_reader.h b/include/paimon/reader/batch_reader.h index 07835c43..c0033c20 100644 --- a/include/paimon/reader/batch_reader.h +++ b/include/paimon/reader/batch_reader.h @@ -43,6 +43,8 @@ class PAIMON_EXPORT BatchReader { /// If EOF is reached, returns an OK status with a nullptr array. Returns an error status only /// for critical failures (e.g., IO errors). Once an error is returned, this method must not be /// retried, as it will repeatedly return the same error code. + /// \note IMPORTANT: A non-EOF ArrowArray and all its nested child arrays must have offset 0 to + /// avoid potential issues during conversion through the Arrow C Data Interface. /// /// @return A result containing a `::ReadBatch`, which consists of a unique pointer to /// `ArrowArray` and a unique pointer to `ArrowSchema`. Returned array contains a `_VALUE_KIND` @@ -55,6 +57,8 @@ class PAIMON_EXPORT BatchReader { /// If EOF is reached, returns an OK status with a nullptr array. Returns an error status only /// for critical failures (e.g., IO errors). Once an error is returned, this method must not be /// retried, as it will repeatedly return the same error code. + /// \note IMPORTANT: A non-EOF ArrowArray and all its nested child arrays must have offset 0 to + /// avoid potential issues during conversion through the Arrow C Data Interface. /// /// @return A result containing a `::ReadBatch` and a valid bitmap. `::ReadBatch` consists of a /// unique pointer to `ArrowArray` and a unique pointer to `ArrowSchema`. Returned array diff --git a/src/paimon/common/utils/arrow/arrow_utils.cpp b/src/paimon/common/utils/arrow/arrow_utils.cpp index 94849f59..de6aedbb 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils.cpp @@ -21,6 +21,7 @@ #include "arrow/array/array_base.h" #include "arrow/array/array_nested.h" +#include "arrow/array/concatenate.h" #include "arrow/util/compression.h" #include "fmt/format.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -28,6 +29,22 @@ namespace paimon { +namespace { + +bool HasNonZeroOffset(const std::shared_ptr& data) { + if (data->offset != 0) { + return true; + } + for (const auto& child : data->child_data) { + if (HasNonZeroOffset(child)) { + return true; + } + } + return false; +} + +} // namespace + const char* ArrowUtils::kArrowSchemaMetadataKey = "ARROW:schema"; Result> ArrowUtils::DataTypeToSchema( @@ -170,6 +187,28 @@ Result> ArrowUtils::RemoveFieldFromStructArr return array; } +Result> ArrowUtils::NormalizeRecordBatchOffsets( + const std::shared_ptr& record_batch, arrow::MemoryPool* pool) { + arrow::ArrayVector normalized_columns; + for (int32_t i = 0; i < record_batch->num_columns(); ++i) { + const std::shared_ptr& column = record_batch->column(i); + if (!HasNonZeroOffset(column->data())) { + continue; + } + if (normalized_columns.empty()) { + normalized_columns = record_batch->columns(); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr normalized_column, + arrow::Concatenate({column}, pool)); + normalized_columns[i] = std::move(normalized_column); + } + if (normalized_columns.empty()) { + return record_batch; + } + return arrow::RecordBatch::Make(record_batch->schema(), record_batch->num_rows(), + std::move(normalized_columns)); +} + Result ArrowUtils::GetCompressionType(const std::string& compression) { std::string normalized = StringUtils::ToLowerCase(compression); if (normalized.empty() || normalized == "none") { diff --git a/src/paimon/common/utils/arrow/arrow_utils.h b/src/paimon/common/utils/arrow/arrow_utils.h index 439609f9..1d4b7855 100644 --- a/src/paimon/common/utils/arrow/arrow_utils.h +++ b/src/paimon/common/utils/arrow/arrow_utils.h @@ -51,6 +51,9 @@ class PAIMON_EXPORT ArrowUtils { static Result> RemoveFieldFromStructArray( const std::shared_ptr& struct_array, const std::string& field_name); + static Result> NormalizeRecordBatchOffsets( + const std::shared_ptr& record_batch, arrow::MemoryPool* pool); + static bool EqualsIgnoreNullable(const std::shared_ptr& type, const std::shared_ptr& other_type); diff --git a/src/paimon/common/utils/arrow/arrow_utils_test.cpp b/src/paimon/common/utils/arrow/arrow_utils_test.cpp index fc1b1c24..a2f8d5bc 100644 --- a/src/paimon/common/utils/arrow/arrow_utils_test.cpp +++ b/src/paimon/common/utils/arrow/arrow_utils_test.cpp @@ -382,6 +382,54 @@ TEST(ArrowUtilsTest, TestRemoveFieldFromStructArraySuccess) { ASSERT_TRUE(result->Equals(expected_struct_array)); } +TEST(ArrowUtilsTest, TestNormalizeRecordBatchOffsets) { + auto value_field = arrow::field("value", arrow::int32()); + auto nested_field = arrow::field("nested", arrow::struct_({value_field})); + auto text_field = arrow::field("text", arrow::utf8()); + auto clean_field = arrow::field("clean", arrow::boolean()); + auto schema = arrow::schema({nested_field, text_field, clean_field}); + + std::shared_ptr values = + arrow::ipc::internal::json::ArrayFromJSON(arrow::int32(), "[0, 1, 2, 3, 4]").ValueOrDie(); + std::shared_ptr sliced_values = values->Slice(1, 3); + std::shared_ptr nested_column = + arrow::StructArray::Make({sliced_values}, {value_field->name()}).ValueOrDie(); + std::shared_ptr text = + arrow::ipc::internal::json::ArrayFromJSON(arrow::utf8(), R"(["a", "b", "c", "d", "e"])") + .ValueOrDie(); + std::shared_ptr sliced_text = text->Slice(1, 3); + std::shared_ptr clean_column = + arrow::ipc::internal::json::ArrayFromJSON(arrow::boolean(), "[true, false, true]") + .ValueOrDie(); + std::shared_ptr record_batch = arrow::RecordBatch::Make( + schema, /*num_rows=*/3, {nested_column, sliced_text, clean_column}); + + ASSERT_EQ(nested_column->offset(), 0); + ASSERT_EQ(nested_column->field(0)->offset(), 1); + ASSERT_EQ(sliced_text->offset(), 1); + ASSERT_EQ(clean_column->offset(), 0); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr normalized_batch, + ArrowUtils::NormalizeRecordBatchOffsets(record_batch, arrow::default_memory_pool())); + ASSERT_NE(normalized_batch.get(), record_batch.get()); + ASSERT_TRUE(normalized_batch->Equals(*record_batch)); + std::shared_ptr normalized_nested = + std::static_pointer_cast(normalized_batch->column(0)); + ASSERT_EQ(normalized_nested->offset(), 0); + ASSERT_EQ(normalized_nested->field(0)->offset(), 0); + ASSERT_EQ(normalized_batch->column(1)->offset(), 0); + ASSERT_EQ(normalized_batch->column(2)->offset(), 0); + ASSERT_NE(normalized_batch->column_data(0).get(), record_batch->column_data(0).get()); + ASSERT_NE(normalized_batch->column_data(1).get(), record_batch->column_data(1).get()); + ASSERT_EQ(normalized_batch->column_data(2).get(), record_batch->column_data(2).get()); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr unchanged_batch, + ArrowUtils::NormalizeRecordBatchOffsets(normalized_batch, arrow::default_memory_pool())); + ASSERT_EQ(unchanged_batch.get(), normalized_batch.get()); +} + TEST(ArrowUtilsTest, TestEqualsIgnoreNullable) { { // test simple @@ -484,6 +532,7 @@ TEST(ArrowUtilsTest, TestGetCompressionType) { ASSERT_EQ(type, arrow::Compression::GZIP); } { + // test invalid codec ASSERT_NOK(ArrowUtils::GetCompressionType("invalid_codec")); } } diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp index 82515696..0f553c96 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.cpp +++ b/src/paimon/format/parquet/file_reader_wrapper.cpp @@ -26,6 +26,7 @@ #include "arrow/record_batch.h" #include "arrow/util/range.h" #include "fmt/format.h" +#include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/format/parquet/column_index_filter.h" #include "paimon/format/parquet/page_filtered_row_group_reader.h" #include "paimon/format/parquet/parquet_format_defs.h" @@ -275,6 +276,10 @@ Result> FileReaderWrapper::NextFullyMatched( if (!record_batch) { return std::shared_ptr(); } + // Large binary columns (exceed 2GB) may split at different row boundaries. TableBatchReader + // aligns their chunks by slicing columns, which may leave non-zero child offsets. + PAIMON_ASSIGN_OR_RAISE(record_batch, + ArrowUtils::NormalizeRecordBatchOffsets(record_batch, pool_.get())); int32_t rg_id = target_row_groups_[current_row_group_idx_].GetRowGroupIndex(); uint64_t rg_end = all_row_group_ranges_[rg_id].second; diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp index e7ca9d3b..9197fef0 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp @@ -29,6 +29,7 @@ #include "arrow/table.h" #include "arrow/util/future.h" #include "fmt/format.h" +#include "paimon/common/utils/arrow/arrow_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "parquet/arrow/reader_internal.h" #include "parquet/metadata.h" @@ -39,12 +40,13 @@ namespace paimon::parquet { namespace { /// Wraps an arrow::Table + TableBatchReader as a RecordBatchReader so the caller can -/// stream zero-copy-sliced batches without deep-copying multi-chunk columns. The Table -/// is held to keep its ChunkedArrays alive for the inner TableBatchReader. +/// stream batches while ensuring every returned array offset is zero. The Table is held +/// to keep its ChunkedArrays alive for the inner TableBatchReader. class TableRecordBatchReader : public arrow::RecordBatchReader { public: - TableRecordBatchReader(std::shared_ptr table, int64_t chunksize) - : table_(std::move(table)), inner_(*table_) { + TableRecordBatchReader(std::shared_ptr table, int64_t chunksize, + std::shared_ptr pool) + : table_(std::move(table)), inner_(*table_), pool_(std::move(pool)) { inner_.set_chunksize(chunksize); } @@ -53,12 +55,26 @@ class TableRecordBatchReader : public arrow::RecordBatchReader { } arrow::Status ReadNext(std::shared_ptr* out) override { - return inner_.ReadNext(out); + ARROW_RETURN_NOT_OK(inner_.ReadNext(out)); + if (!*out) { + return arrow::Status::OK(); + } + + // Page filtering may produce columns with different chunk boundaries. TableBatchReader + // aligns them by slicing columns, which may leave non-zero child offsets. + Result> normalized_result = + ArrowUtils::NormalizeRecordBatchOffsets(*out, pool_.get()); + if (!normalized_result.ok()) { + return ToArrowStatus(normalized_result.status()); + } + *out = std::move(normalized_result).value(); + return arrow::Status::OK(); } private: std::shared_ptr table_; arrow::TableBatchReader inner_; + std::shared_ptr pool_; }; } // namespace @@ -243,7 +259,8 @@ Result> PageFilteredRowGroupReader::Re if (row_ranges.IsEmpty()) { PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr empty_table, arrow::Table::MakeEmpty(arrow_schema, pool.get())); - return std::make_unique(std::move(empty_table), max_chunksize); + return std::make_unique(std::move(empty_table), max_chunksize, + pool); } int64_t expected_rows = row_ranges.RowCount(); @@ -286,7 +303,8 @@ Result> PageFilteredRowGroupReader::Re } auto table = arrow::Table::Make(arrow_schema, std::move(columns), expected_rows); - return std::make_unique(std::move(table), max_chunksize); + return std::make_unique(std::move(table), max_chunksize, + std::move(pool)); } std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRanges( diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp index e0672fd5..e5a4c0e3 100644 --- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp +++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp @@ -664,14 +664,13 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) { ASSERT_LT(ranges[0].offset, ranges[1].offset); } -/// Test: variable-length columns are streamed across multiple zero-copy-sliced +/// Test: variable-length columns are streamed across multiple offset-normalized /// RecordBatches when batch_size is smaller than the matched row count, instead of /// being concatenated into a single RecordBatch via CombineChunks. /// /// This verifies the alignment with Arrow's standard TableBatchReader path: -/// multi-chunk binary/string columns split along chunk + batch_size boundaries, -/// with no deep copy. Asserts both correctness (total rows + full content order) and -/// the multi-batch shape (more than one chunk in the collected ChunkedArray). +/// multi-chunk binary/string columns split along chunk + batch_size boundaries. It +/// asserts correctness and the multi-batch shape. TEST_F(PageFilteredRowGroupReaderTest, StringColumnMultiBatchStreaming) { std::string file_name = dir_->Str() + "/string_multi_batch.parquet"; @@ -724,6 +723,23 @@ TEST_F(PageFilteredRowGroupReaderTest, StringColumnMultiBatchStreaming) { ASSERT_EQ(40, seen); } +TEST_F(PageFilteredRowGroupReaderTest, NormalizesSlicedBatchOffsets) { + std::string file_name = dir_->Str() + "/normalized_sliced_offsets.parquet"; + std::shared_ptr data = MakeSequentialIntData(60); + WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/60); + + std::shared_ptr read_schema = + arrow::schema({arrow::field("val", arrow::int32())}); + std::shared_ptr predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(20)); + + std::shared_ptr result; + ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/7); + ASSERT_TRUE(result); + ASSERT_EQ(40, result->length()); + ASSERT_GT(result->num_chunks(), 1); +} + /// Test: end-to-end page-filtered read produces correct results when using page-level PreBuffer. /// /// This exercises the full path: ComputePageRanges → PreBufferRanges → CachedInputStream → diff --git a/src/paimon/testing/utils/read_result_collector.h b/src/paimon/testing/utils/read_result_collector.h index bd8347d8..8321a838 100644 --- a/src/paimon/testing/utils/read_result_collector.h +++ b/src/paimon/testing/utils/read_result_collector.h @@ -182,6 +182,7 @@ class ReadResultCollector { if (BatchReader::IsEofBatch(batch_with_bitmap)) { return std::shared_ptr(); } + PAIMON_RETURN_NOT_OK(CheckBatchOffset(batch_with_bitmap.first)); assert(!batch_with_bitmap.second.IsEmpty()); PAIMON_ASSIGN_OR_RAISE( batch, ReaderUtils::ApplyBitmapToReadBatch(std::move(batch_with_bitmap), @@ -194,6 +195,7 @@ class ReadResultCollector { if (BatchReader::IsEofBatch(batch)) { return std::shared_ptr(); } + PAIMON_RETURN_NOT_OK(CheckBatchOffset(batch)); } auto& [c_array, c_schema] = batch; assert(c_array->length > 0); @@ -201,5 +203,22 @@ class ReadResultCollector { arrow::ImportArray(c_array.get(), c_schema.get())); return result_array; } + + static Status CheckBatchOffset(const BatchReader::ReadBatch& batch) { + assert(!BatchReader::IsEofBatch(batch)); + return CheckArrayOffset(batch.first.get()); + } + + static Status CheckArrayOffset(const ArrowArray* array) { + assert(array); + if (array->offset != 0) { + return Status::Invalid("BatchReader returned an array with non-zero offset " + + std::to_string(array->offset)); + } + for (int64_t i = 0; i < array->n_children; i++) { + PAIMON_RETURN_NOT_OK(CheckArrayOffset(array->children[i])); + } + return Status::OK(); + } }; } // namespace paimon::test From 9ed061e7f32c04cbd3689c6af780c2fbea08d973 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 23 Jul 2026 21:50:06 +0800 Subject: [PATCH 114/138] feat: add global system tables framework under `sys` database --- include/paimon/catalog/catalog.h | 5 + src/paimon/CMakeLists.txt | 1 + src/paimon/core/catalog/catalog.cpp | 2 +- .../core/catalog/file_system_catalog.cpp | 40 +- src/paimon/core/catalog/file_system_catalog.h | 5 +- .../core/catalog/file_system_catalog_test.cpp | 70 ++- .../table/system/global_system_tables.cpp | 543 ++++++++++++++++++ .../core/table/system/global_system_tables.h | 123 ++++ src/paimon/core/table/system/system_table.cpp | 29 +- src/paimon/core/table/system/system_table.h | 2 + .../core/table/system/system_table_test.cpp | 8 + test/inte/read_inte_test.cpp | 436 ++++++++++++++ 12 files changed, 1229 insertions(+), 35 deletions(-) create mode 100644 src/paimon/core/table/system/global_system_tables.cpp create mode 100644 src/paimon/core/table/system/global_system_tables.h diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 5f1f04b9..b5d6bddd 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -185,6 +185,11 @@ class PAIMON_EXPORT Catalog { /// @return A shared pointer to the file system instance. virtual std::shared_ptr GetFileSystem() const = 0; + /// Returns the catalog-level options that were passed during catalog creation. + /// + /// @return A const reference to the map of catalog options (key-value pairs). + virtual const std::map& GetOptions() const = 0; + /// Loads the latest schema of a specified table. /// /// @note System tables will not be supported. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 9f78a69e..2be943b4 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -376,6 +376,7 @@ set(PAIMON_CORE_SRCS core/table/source/data_evolution_batch_scan.cpp core/table/system/audit_log_system_table.cpp core/table/system/binlog_system_table.cpp + core/table/system/global_system_tables.cpp core/table/system/in_memory_system_table.cpp core/table/system/metadata_system_tables.cpp core/table/system/read_optimized_system_table.cpp diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 4c1b06bb..ab26ce0c 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -34,7 +34,7 @@ Result> Catalog::Create(const std::string& root_path, const std::map& options, const std::shared_ptr& file_system) { PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); - return std::make_unique(core_options.GetFileSystem(), root_path); + return std::make_unique(core_options.GetFileSystem(), root_path, options); } } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index ea5b1087..292af2e8 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -33,6 +33,7 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" @@ -49,8 +50,12 @@ struct ArrowSchema; namespace paimon { FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, - const std::string& warehouse) - : fs_(fs), warehouse_(warehouse), logger_(Logger::GetLogger("FileSystemCatalog")) {} + const std::string& warehouse, + const std::map& catalog_options) + : fs_(fs), + warehouse_(warehouse), + catalog_options_(catalog_options), + logger_(Logger::GetLogger("FileSystemCatalog")) {} Status FileSystemCatalog::CreateDatabase(const std::string& db_name, const std::map& options, @@ -90,13 +95,16 @@ Status FileSystemCatalog::CreateDatabaseImpl(const std::string& db_name, Result FileSystemCatalog::DatabaseExists(const std::string& db_name) const { if (IsSystemDatabase(db_name)) { - return Status::NotImplemented( - "do not support checking DatabaseExists for system database."); + return true; } return fs_->Exists(NewDatabasePath(warehouse_, db_name)); } Result FileSystemCatalog::TableExists(const Identifier& identifier) const { + // Handle sys database global tables + if (IsSystemDatabase(identifier.GetDatabaseName())) { + return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), catalog_options_); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, @@ -186,6 +194,10 @@ std::shared_ptr FileSystemCatalog::GetFileSystem() const { return fs_; } +const std::map& FileSystemCatalog::GetOptions() const { + return catalog_options_; +} + bool FileSystemCatalog::IsSystemDatabase(const std::string& db_name) { return db_name == SYSTEM_DATABASE_NAME; } @@ -230,7 +242,7 @@ Result> FileSystemCatalog::ListDatabases() const { Result> FileSystemCatalog::ListTables(const std::string& db_name) const { if (IsSystemDatabase(db_name)) { - return Status::NotImplemented("do not support listing tables for system database."); + return GlobalSystemTableLoader::GetSupportedTableNames(catalog_options_); } std::string database_path = NewDatabasePath(warehouse_, db_name); std::vector> file_status_list; @@ -263,6 +275,24 @@ Result FileSystemCatalog::TableExistsInFileSystem(const std::string& table Result> FileSystemCatalog::LoadTableSchema( const Identifier& identifier) const { + // Handle sys database global tables + if (IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( + identifier.GetTableName(), catalog_options_)); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = this; + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = catalog_options_; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 0d974cac..3a464ae8 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -40,7 +40,8 @@ class Logger; class FileSystemCatalog : public Catalog { public: - FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse); + FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, + const std::map& catalog_options); Status CreateDatabase(const std::string& db_name, const std::map& options, @@ -63,6 +64,7 @@ class FileSystemCatalog : public Catalog { Result> LoadTableSchema(const Identifier& identifier) const override; std::string GetRootPath() const override; std::shared_ptr GetFileSystem() const override; + const std::map& GetOptions() const override; Result> GetTable(const Identifier& identifier) const override; Result> ListSnapshots(const Identifier& identifier, const std::string& branch) const override; @@ -94,6 +96,7 @@ class FileSystemCatalog : public Catalog { std::shared_ptr fs_; std::string warehouse_; + std::map catalog_options_; std::shared_ptr logger_; }; diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index b2f1da16..2a07b262 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -18,6 +18,8 @@ #include "paimon/core/catalog/file_system_catalog.h" +#include + #include "arrow/api.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" @@ -28,6 +30,7 @@ #include "paimon/common/utils/path_util.h" #include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" @@ -44,7 +47,7 @@ TEST(FileSystemCatalogTest, TestDatabaseExists) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK_AND_ASSIGN(auto exist, catalog.DatabaseExists("db1")); ASSERT_FALSE(exist); @@ -70,7 +73,7 @@ TEST(FileSystemCatalogTest, TestInvalidCreateDatabase) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_NOK_WITH_MSG( catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true), @@ -87,7 +90,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_NOK_WITH_MSG(catalog.CreateDatabase(Catalog::SYSTEM_DATABASE_NAME, options, /*ignore_if_exists=*/true), "Cannot create database for system database"); @@ -100,7 +103,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { arrow::field("f0", arrow::boolean()), @@ -125,7 +128,7 @@ TEST(FileSystemCatalogTest, TestCreateTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { @@ -161,7 +164,7 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())}); @@ -219,7 +222,7 @@ TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); auto typed_schema = @@ -290,7 +293,7 @@ TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); auto typed_schema = @@ -435,7 +438,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = {arrow::field("f0", arrow::boolean()), @@ -491,7 +494,7 @@ TEST(FileSystemCatalogTest, TestInvalidCreateTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { arrow::field("f0", arrow::boolean()), arrow::field("f1", arrow::int8()), @@ -516,7 +519,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWhileDbNotExist) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); arrow::FieldVector fields = { arrow::field("f0", arrow::boolean()), arrow::field("f1", arrow::int8()), @@ -539,7 +542,7 @@ TEST(FileSystemCatalogTest, TestCreateTableWhileTableExist) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); { ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { @@ -600,16 +603,29 @@ TEST(FileSystemCatalogTest, TestCreateTableWhileTableExist) { } } -TEST(FileSystemCatalogTest, TestInvalidList) { +TEST(FileSystemCatalogTest, TestSystemList) { std::map options; options[Options::FILE_SYSTEM] = "local"; options[Options::FILE_FORMAT] = "orc"; ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); - ASSERT_NOK_WITH_MSG(catalog.ListTables("sys"), - "do not support listing tables for system database."); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK_AND_ASSIGN(auto sys_tables, catalog.ListTables("sys")); + ASSERT_FALSE(sys_tables.empty()); + // catalog_options is disabled by default, matching Java Paimon. + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "catalog_options") == + sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "all_table_options") != + sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "tables") != sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "partitions") != sys_tables.end()); + + options[CatalogOptionsSystemTable::kEnabledOption] = "true"; + FileSystemCatalog enabled_catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK_AND_ASSIGN(auto enabled_sys_tables, enabled_catalog.ListTables("sys")); + ASSERT_TRUE(std::find(enabled_sys_tables.begin(), enabled_sys_tables.end(), + "catalog_options") != enabled_sys_tables.end()); } TEST(FileSystemCatalogTest, TestValidateTableSchema) { @@ -619,7 +635,7 @@ TEST(FileSystemCatalogTest, TestValidateTableSchema) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); arrow::FieldVector fields = { arrow::field("f0", arrow::utf8()), @@ -688,7 +704,7 @@ TEST(FileSystemCatalogTest, TestDropDatabase) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); // Test 1: Drop non-existent database with ignore_if_not_exists=true ASSERT_OK(catalog.DropDatabase("non_existent_db", /*ignore_if_not_exists=*/true, @@ -744,7 +760,7 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Test 1: Drop non-existent table with ignore_if_not_exists=true @@ -786,7 +802,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Test 1: Rename non-existent table with ignore_if_not_exists=true @@ -853,7 +869,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithExternalPath) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Create external path directory @@ -915,7 +931,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithMultipleExternalPaths) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Create multiple external path directories @@ -972,7 +988,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathOnMainBranch ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_OK(catalog.CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); // Create external path directory for global index @@ -1076,7 +1092,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithGlobalIndexExternalPathOnBranch) { ASSERT_TRUE(external_exists); // Drop the table via catalog - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); Identifier identifier("test_db", "append_table_with_rt_branch"); ASSERT_OK_AND_ASSIGN(bool table_exists, catalog.TableExists(identifier)); ASSERT_TRUE(table_exists); @@ -1104,7 +1120,7 @@ TEST(FileSystemCatalogTest, TestListSnapshots) { std::string db_path = dir->Str() + "/test_db.db"; ASSERT_TRUE(TestUtil::CopyDirectory(test_data_path, db_path)); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); Identifier id("test_db", "append_table_with_multiple_file_format"); ASSERT_OK_AND_ASSIGN(std::vector snapshots, catalog.ListSnapshots(id, "")); @@ -1138,7 +1154,7 @@ TEST(FileSystemCatalogTest, TestListSnapshotsTableNotExist) { ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_NOK_WITH_MSG( catalog.ListSnapshots(Identifier("non_existent_db", "non_existent_table"), ""), @@ -1196,7 +1212,7 @@ TEST(FileSystemCatalogTest, TestDropTableWithBranchExternalPaths) { ASSERT_TRUE(external_exists); // Drop the table via catalog - FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); Identifier identifier("test_db", "append_table_with_rt_branch"); ASSERT_OK_AND_ASSIGN(bool table_exists, catalog.TableExists(identifier)); ASSERT_TRUE(table_exists); diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp new file mode 100644 index 00000000..bdfab0f0 --- /dev/null +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -0,0 +1,543 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/system/global_system_tables.h" + +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/data/binary_string.h" +#include "paimon/common/data/generic_row.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_entry.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_file_meta.h" +#include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/defs.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/status.h" + +namespace paimon { +namespace { + +// ============================================================================= +// Registry +// ============================================================================= + +using GlobalSystemTableFactory = + std::function>(const GlobalSystemTableContext&)>; + +struct GlobalSystemTableRegistryEntry { + std::string name; + bool requires_catalog; + GlobalSystemTableFactory factory; +}; + +const std::vector& GlobalSystemTableRegistry() { + static const std::vector registry = { + {CatalogOptionsSystemTable::kName, false, + [](const GlobalSystemTableContext& ctx) -> Result> { + return std::make_shared(ctx); + }}, + {AllTableOptionsSystemTable::kName, true, + [](const GlobalSystemTableContext& ctx) -> Result> { + return std::make_shared(ctx); + }}, + {TablesSystemTable::kName, true, + [](const GlobalSystemTableContext& ctx) -> Result> { + return std::make_shared(ctx); + }}, + {PartitionsSystemTable::kName, true, + [](const GlobalSystemTableContext& ctx) -> Result> { + return std::make_shared(ctx); + }}, + }; + return registry; +} + +// ============================================================================= +// Helpers for sys.tables and sys.partitions +// ============================================================================= + +VariantType StringValue(const std::string& value) { + return BinaryString::FromString(value, GetDefaultPool().get()); +} + +VariantType OptionalStringValue(const std::map& options, + const std::string& key) { + auto it = options.find(key); + return it == options.end() ? VariantType(NullType()) : VariantType(StringValue(it->second)); +} + +Result OptionalLongValue(const std::map& options, + const std::string& key) { + if (options.find(key) == options.end()) { + return VariantType(NullType()); + } + PAIMON_ASSIGN_OR_RAISE(int64_t value, OptionsUtils::GetValueFromMap(options, key)); + return VariantType(value); +} + +Result IsEnabled(const GlobalSystemTableRegistryEntry& entry, + const std::map& catalog_options) { + if (entry.name != CatalogOptionsSystemTable::kName) { + return true; + } + return OptionsUtils::GetValueFromMap(catalog_options, + CatalogOptionsSystemTable::kEnabledOption, false); +} + +struct CatalogTableInfo { + std::string database_name; + std::string table_name; + std::shared_ptr schema; +}; + +// Match Java CatalogUtils::listAllTables: tolerate databases or tables removed concurrently, but +// propagate all other catalog and schema errors instead of returning incomplete system-table rows. +Result> LoadAllDataTables(const Catalog& catalog) { + std::vector result; + PAIMON_ASSIGN_OR_RAISE(std::vector databases, catalog.ListDatabases()); + for (const std::string& database : databases) { + Result> tables_result = catalog.ListTables(database); + if (!tables_result.ok()) { + if (tables_result.status().IsNotExist()) { + continue; + } + return tables_result.status(); + } + for (const std::string& table : tables_result.value()) { + Identifier identifier(database, table); + Result> schema_result = catalog.LoadTableSchema(identifier); + if (!schema_result.ok()) { + if (schema_result.status().IsNotExist()) { + continue; + } + return schema_result.status(); + } + std::shared_ptr data_schema = + std::dynamic_pointer_cast(schema_result.value()); + if (!data_schema) { + return Status::Invalid("catalog returned a non-data schema for ", + identifier.ToString()); + } + result.push_back({database, table, std::move(data_schema)}); + } + } + return result; +} + +// Aggregated file-level statistics for a table or partition. +struct FileStats { + int64_t record_count = 0; + int64_t file_size_in_bytes = 0; + int64_t file_count = 0; + int64_t last_file_creation_time_millis = 0; +}; + +struct AggregatedFileStats { + bool has_snapshot = false; + std::map by_partition; +}; + +// Read the latest snapshot's data files and aggregate statistics. +Result AggregateFileStats(const std::shared_ptr& fs, + const std::string& table_path, + const DataSchema& table_schema) { + AggregatedFileStats result; + + SnapshotManager snapshot_manager(fs, table_path, BranchManager::DEFAULT_MAIN_BRANCH); + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, snapshot_manager.LatestSnapshot()); + if (!snapshot) { + return result; + } + result.has_snapshot = true; + + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(table_schema.Options())); + + auto pool = GetDefaultPool(); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_arrow_schema, + table_schema.GetArrowSchema()); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, + arrow::ImportSchema(c_arrow_schema.get())); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, + core_options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + table_path, arrow_schema, table_schema.PartitionKeys(), + core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), + core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), + pool)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_list, + ManifestList::Create(fs, core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, + core_options.GetCache(), pool)); + + std::vector manifests; + PAIMON_RETURN_NOT_OK(manifest_list->ReadDataManifests(*snapshot, &manifests)); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema.PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_file, + ManifestFile::Create(fs, core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, + core_options.GetManifestTargetFileSize(), pool, + core_options, partition_schema)); + + std::vector entries; + for (const auto& manifest : manifests) { + PAIMON_RETURN_NOT_OK( + manifest_file->Read(manifest.FileName(), /*filter=*/nullptr, &entries)); + } + + std::vector merged_entries; + PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(entries, &merged_entries)); + + for (const auto& entry : merged_entries) { + if (!(entry.Kind() == FileKind::Add())) { + continue; + } + const auto& file = entry.File(); + + // Convert partition BinaryRow to string representation + std::string partition_key; + if (entry.Partition().GetFieldCount() > 0) { + PAIMON_ASSIGN_OR_RAISE(auto partition_values, + path_factory->GeneratePartitionVector(entry.Partition())); + for (const auto& [key, value] : partition_values) { + if (!partition_key.empty()) { + partition_key += "/"; + } + partition_key += key + "=" + value; + } + } + + auto& stats = result.by_partition[partition_key]; + stats.record_count += file->row_count; + stats.file_size_in_bytes += file->file_size; + stats.file_count++; + int64_t creation_millis = file->creation_time.GetMillisecond(); + if (creation_millis > stats.last_file_creation_time_millis) { + stats.last_file_creation_time_millis = creation_millis; + } + } + + return result; +} + +} // namespace + +// ============================================================================= +// GlobalSystemTableLoader +// ============================================================================= + +Result GlobalSystemTableLoader::IsSupported( + const std::string& table_name, const std::map& catalog_options) { + std::string normalized = StringUtils::ToLowerCase(table_name); + for (const auto& entry : GlobalSystemTableRegistry()) { + if (entry.name == normalized) { + return IsEnabled(entry, catalog_options); + } + } + return false; +} + +Result> GlobalSystemTableLoader::Load( + const std::string& table_name, const GlobalSystemTableContext& context) { + std::string normalized = StringUtils::ToLowerCase(table_name); + for (const auto& entry : GlobalSystemTableRegistry()) { + if (entry.name == normalized) { + PAIMON_ASSIGN_OR_RAISE(bool enabled, IsEnabled(entry, context.catalog_options)); + if (!enabled) { + return Status::NotExist("global system table is disabled: ", table_name); + } + if (entry.requires_catalog && context.catalog == nullptr) { + return Status::NotImplemented("global system table requires catalog context: ", + table_name); + } + return entry.factory(context); + } + } + return Status::NotImplemented("unsupported global system table: ", table_name); +} + +Result> GlobalSystemTableLoader::GetSupportedTableNames( + const std::map& catalog_options) { + std::vector names; + names.reserve(GlobalSystemTableRegistry().size()); + for (const auto& entry : GlobalSystemTableRegistry()) { + PAIMON_ASSIGN_OR_RAISE(bool enabled, IsEnabled(entry, catalog_options)); + if (enabled) { + names.push_back(entry.name); + } + } + return names; +} + +// ============================================================================= +// sys.catalog_options +// ============================================================================= + +CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) + : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {} + +std::string CatalogOptionsSystemTable::Name() const { + return kName; +} + +Result> CatalogOptionsSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("key", arrow::utf8(), /*nullable=*/false), + arrow::field("value", arrow::utf8(), /*nullable=*/false), + }); +} + +Result> CatalogOptionsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::vector rows; + rows.reserve(context_.catalog_options.size()); + for (const auto& [key, value] : context_.catalog_options) { + GenericRow row(schema->num_fields()); + row.SetField(0, StringValue(key)); + row.SetField(1, StringValue(value)); + rows.push_back(std::move(row)); + } + return rows; +} + +// ============================================================================= +// sys.all_table_options +// ============================================================================= + +AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) + : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {} + +std::string AllTableOptionsSystemTable::Name() const { + return kName; +} + +Result> AllTableOptionsSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("database_name", arrow::utf8(), /*nullable=*/false), + arrow::field("table_name", arrow::utf8(), /*nullable=*/false), + arrow::field("key", arrow::utf8(), /*nullable=*/false), + arrow::field("value", arrow::utf8(), /*nullable=*/false), + }); +} + +Result> AllTableOptionsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::vector rows; + + PAIMON_ASSIGN_OR_RAISE(std::vector tables, + LoadAllDataTables(*context_.catalog)); + for (const CatalogTableInfo& table : tables) { + for (const auto& [key, value] : table.schema->Options()) { + GenericRow row(schema->num_fields()); + row.SetField(0, StringValue(table.database_name)); + row.SetField(1, StringValue(table.table_name)); + row.SetField(2, StringValue(key)); + row.SetField(3, StringValue(value)); + rows.push_back(std::move(row)); + } + } + return rows; +} + +// ============================================================================= +// sys.tables +// ============================================================================= + +TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) + : InMemorySystemTable("sys/tables"), context_(std::move(context)) {} + +std::string TablesSystemTable::Name() const { + return kName; +} + +Result> TablesSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("database_name", arrow::utf8(), /*nullable=*/false), + arrow::field("table_name", arrow::utf8(), /*nullable=*/false), + arrow::field("table_type", arrow::utf8(), /*nullable=*/false), + arrow::field("partitioned", arrow::boolean(), /*nullable=*/false), + arrow::field("primary_key", arrow::boolean(), /*nullable=*/false), + arrow::field("owner", arrow::utf8(), /*nullable=*/true), + arrow::field("created_at", arrow::int64(), /*nullable=*/true), + arrow::field("created_by", arrow::utf8(), /*nullable=*/true), + arrow::field("updated_at", arrow::int64(), /*nullable=*/true), + arrow::field("updated_by", arrow::utf8(), /*nullable=*/true), + arrow::field("record_count", arrow::int64(), /*nullable=*/true), + arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/true), + arrow::field("file_count", arrow::int64(), /*nullable=*/true), + arrow::field("last_file_creation_time", arrow::int64(), /*nullable=*/true), + }); +} + +Result> TablesSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::vector rows; + + PAIMON_ASSIGN_OR_RAISE(std::vector tables, + LoadAllDataTables(*context_.catalog)); + for (const CatalogTableInfo& table : tables) { + const std::shared_ptr& data_schema = table.schema; + + const auto& opts = data_schema->Options(); + auto table_type = opts.find("type"); + const std::string table_type_str = table_type == opts.end() ? "table" : table_type->second; + + bool partitioned = !data_schema->PartitionKeys().empty(); + + GenericRow row(schema->num_fields()); + row.SetField(0, StringValue(table.database_name)); + row.SetField(1, StringValue(table.table_name)); + row.SetField(2, StringValue(table_type_str)); + row.SetField(3, partitioned); + row.SetField(4, !data_schema->PrimaryKeys().empty()); + row.SetField(5, OptionalStringValue(opts, "owner")); + PAIMON_ASSIGN_OR_RAISE(VariantType created_at, OptionalLongValue(opts, "createdAt")); + row.SetField(6, std::move(created_at)); + row.SetField(7, OptionalStringValue(opts, "createdBy")); + PAIMON_ASSIGN_OR_RAISE(VariantType updated_at, OptionalLongValue(opts, "updatedAt")); + row.SetField(8, std::move(updated_at)); + row.SetField(9, OptionalStringValue(opts, "updatedBy")); + + // Match Java CatalogUtils::toTableAndSnapshots when version management is unsupported. + // The C++ Catalog API currently has no version-management capability, so leave snapshot + // statistics null instead of deriving different live-file semantics from manifests. + row.SetField(10, NullType()); + row.SetField(11, NullType()); + row.SetField(12, NullType()); + row.SetField(13, NullType()); + + rows.push_back(std::move(row)); + } + return rows; +} + +// ============================================================================= +// sys.partitions +// ============================================================================= + +PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) + : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {} + +std::string PartitionsSystemTable::Name() const { + return kName; +} + +Result> PartitionsSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("database_name", arrow::utf8(), /*nullable=*/false), + arrow::field("table_name", arrow::utf8(), /*nullable=*/false), + arrow::field("partition_name", arrow::utf8(), /*nullable=*/true), + arrow::field("record_count", arrow::int64(), /*nullable=*/true), + arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/true), + arrow::field("file_count", arrow::int64(), /*nullable=*/true), + arrow::field("last_file_creation_time", arrow::int64(), /*nullable=*/true), + arrow::field("done", arrow::boolean(), /*nullable=*/false), + }); +} + +Result> PartitionsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::vector rows; + + PAIMON_ASSIGN_OR_RAISE(std::vector tables, + LoadAllDataTables(*context_.catalog)); + for (const CatalogTableInfo& table : tables) { + const std::shared_ptr& data_schema = table.schema; + Identifier id(table.database_name, table.table_name); + + // Only emit rows for partitioned tables + if (data_schema->PartitionKeys().empty()) { + continue; + } + + // Match Java's toAllPartitions by ignoring only concurrent table deletion. All other + // metadata and I/O errors must fail the query. + Result table_path_result = context_.catalog->GetTableLocation(id); + if (!table_path_result.ok()) { + if (table_path_result.status().IsNotExist()) { + continue; + } + return table_path_result.status(); + } + + Result file_stats_result = + AggregateFileStats(context_.fs, table_path_result.value(), *data_schema); + if (!file_stats_result.ok()) { + if (file_stats_result.status().IsNotExist()) { + continue; + } + return file_stats_result.status(); + } + + auto& stats_map = file_stats_result.value().by_partition; + for (const auto& [partition_key, stats] : stats_map) { + if (stats.file_count == 0) { + continue; + } + GenericRow row(schema->num_fields()); + row.SetField(0, StringValue(table.database_name)); + row.SetField(1, StringValue(table.table_name)); + row.SetField(2, partition_key.empty() ? VariantType(NullType()) + : VariantType(StringValue(partition_key))); + row.SetField(3, VariantType(stats.record_count)); + row.SetField(4, VariantType(stats.file_size_in_bytes)); + row.SetField(5, VariantType(stats.file_count)); + row.SetField(6, stats.last_file_creation_time_millis > 0 + ? VariantType(stats.last_file_creation_time_millis) + : VariantType(NullType())); + // File-system catalog partitions are not explicitly marked as done. + row.SetField(7, false); + rows.push_back(std::move(row)); + } + } + return rows; +} + +} // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h new file mode 100644 index 00000000..3d693626 --- /dev/null +++ b/src/paimon/core/table/system/global_system_tables.h @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/table/system/in_memory_system_table.h" + +namespace paimon { +class Catalog; +class FileSystem; + +/// Context passed to global system table constructors, providing catalog-level +/// access for enumerating databases, tables, and reading metadata. +struct GlobalSystemTableContext { + const Catalog* catalog = nullptr; // non-owning pointer + std::shared_ptr fs; + std::string warehouse; + std::map catalog_options; +}; + +/// System table for `sys.catalog_options`, exposing catalog-level configuration +/// as key/value rows. +class CatalogOptionsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "catalog_options"; + static constexpr const char* kEnabledOption = "catalog-options-table.enabled"; + + explicit CatalogOptionsSystemTable(GlobalSystemTableContext context); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + GlobalSystemTableContext context_; +}; + +/// System table for `sys.all_table_options`, exposing all table options across +/// all databases as (database_name, table_name, key, value) rows. +class AllTableOptionsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "all_table_options"; + + explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + GlobalSystemTableContext context_; +}; + +/// System table for `sys.tables`, exposing metadata for all tables across all +/// databases including record counts and file statistics. +class TablesSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "tables"; + + explicit TablesSystemTable(GlobalSystemTableContext context); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + GlobalSystemTableContext context_; +}; + +/// System table for `sys.partitions`, exposing partition-level file statistics +/// for all tables across all databases. +class PartitionsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "partitions"; + + explicit PartitionsSystemTable(GlobalSystemTableContext context); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + GlobalSystemTableContext context_; +}; + +/// Loader for global system tables under the `sys` database. +/// +/// Maintains its own registry with a factory signature that receives a +/// GlobalSystemTableContext instead of a per-table TableSchema. +class GlobalSystemTableLoader { + public: + static Result IsSupported(const std::string& table_name, + const std::map& catalog_options); + + static Result> Load(const std::string& table_name, + const GlobalSystemTableContext& context); + + static Result> GetSupportedTableNames( + const std::map& catalog_options); +}; + +} // namespace paimon diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index d2ca7ce8..2d67355a 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -26,6 +26,7 @@ #include #include +#include "paimon/catalog/catalog.h" #include "paimon/catalog/identifier.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" @@ -33,6 +34,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/core/utils/branch_manager.h" @@ -191,6 +193,17 @@ Result> SystemTableLoader::Load( Result> SystemTableLoader::TryParsePath(const std::string& path) { std::string table_name = PathUtil::GetName(path); + std::string parent = PathUtil::GetParentDirPath(path); + std::string parent_name = PathUtil::GetName(parent); + + // Detect global system table paths: /sys/ + if (parent_name == Catalog::SYSTEM_DATABASE_NAME) { + SystemTablePath system_table_path; + system_table_path.is_global = true; + system_table_path.system_table_name = table_name; + return std::optional(std::move(system_table_path)); + } + Identifier identifier(table_name); PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (!is_system_table) { @@ -200,7 +213,6 @@ Result> SystemTableLoader::TryParsePath(const std PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, identifier.GetSystemTableName()); - std::string parent = PathUtil::GetParentDirPath(path); SystemTablePath system_table_path; system_table_path.table_path = PathUtil::JoinPath(parent, data_table_name); system_table_path.branch = std::move(branch); @@ -216,6 +228,21 @@ Result> SystemTableLoader::LoadFromPath( return Status::Invalid("path is not a system table path: ", path); } const auto& parsed = system_table_path.value(); + + // Handle global system tables (under sys/ directory) + if (parsed.is_global) { + GlobalSystemTableContext context; + context.fs = fs; + // The warehouse is the grandparent of the sys/
path + context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); + context.catalog_options = dynamic_options; + // Note: context.catalog is intentionally left as nullptr here. + // Global tables loaded from path do not have a Catalog reference and + // cannot enumerate databases/tables. Only tables that don't require + // catalog enumeration (e.g. catalog_options) will work in this path. + return GlobalSystemTableLoader::Load(parsed.system_table_name, context); + } + SchemaManager schema_manager(fs, parsed.table_path, parsed.branch.value_or(BranchManager::DEFAULT_MAIN_BRANCH)); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index ed35b6c2..39e1da31 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -45,6 +45,8 @@ struct SystemTablePath { std::optional branch; /// System table name, for example `options` or `snapshots`. std::string system_table_name; + /// Whether this is a global system table under the `sys` database. + bool is_global = false; }; /// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp index caffe92b..3e9233ba 100644 --- a/src/paimon/core/table/system/system_table_test.cpp +++ b/src/paimon/core/table/system/system_table_test.cpp @@ -33,6 +33,7 @@ #include "paimon/core/table/system/read_optimized_system_table.h" #include "paimon/defs.h" #include "paimon/fs/file_system.h" +#include "paimon/fs/file_system_factory.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" @@ -99,4 +100,11 @@ TEST(SystemTableTest, TestReadOptimizedSystemTablePathParsing) { ASSERT_EQ(parsed->system_table_name, ReadOptimizedSystemTable::kName); } +TEST(SystemTableTest, TestGlobalSystemTableWithoutCatalogReturnsNotImplemented) { + ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFactory::Get("local", "/tmp", {})); + std::shared_ptr shared_fs(std::move(fs)); + ASSERT_NOK_WITH_MSG(SystemTableLoader::LoadFromPath(shared_fs, "/tmp/warehouse/sys/tables", {}), + "global system table requires catalog context: tables"); +} + } // namespace paimon::test diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index d6757150..9b9f8bde 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -16,6 +16,7 @@ * limitations under the License. */ +#include #include #include #include @@ -48,11 +49,14 @@ #include "paimon/common/utils/scope_guard.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_source.h" +#include "paimon/core/snapshot.h" #include "paimon/core/stats/simple_stats.h" #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/deletion_file.h" #include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/tag/tag.h" +#include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/defs.h" @@ -3721,4 +3725,436 @@ TEST_P(ReadInteTest, TestSpecificFs) { ASSERT_GT(io_count.load(std::memory_order_relaxed), 0); } +// ============================================================================= +// Global System Table Tests +// ============================================================================= + +namespace { + +Result ReadGlobalSystemTable( + const std::string& table_name, Catalog* catalog, const std::shared_ptr& fs, + const std::string& warehouse, const std::map& options) { + GlobalSystemTableContext ctx; + ctx.catalog = catalog; + ctx.fs = fs; + ctx.warehouse = warehouse; + ctx.catalog_options = options; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(table_name, ctx)); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + + std::string sys_path = PathUtil::JoinPath(PathUtil::JoinPath(warehouse, "sys"), table_name); + + ScanContextBuilder scan_context_builder(sys_path); + scan_context_builder.SetOptions(options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + system_table->NewScan(scan_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + + ReadContextBuilder read_context_builder(sys_path); + read_context_builder.SetOptions(options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + system_table->NewRead(read_context)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + ReadResultCollector::CollectResult(batch_reader.get())); + return SystemTableReadResult(std::move(batch_reader), result); +} + +} // namespace + +TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { + std::map options = { + {Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {CatalogOptionsSystemTable::kEnabledOption, "true"}, + {"custom.catalog.option", "test-value"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadGlobalSystemTable("catalog_options", catalog.get(), + catalog->GetFileSystem(), warehouse, options)); + auto struct_array = SingleStructChunk(result); + ASSERT_TRUE(struct_array); + auto key_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto value_array = std::dynamic_pointer_cast(struct_array->field(1)); + ASSERT_TRUE(key_array); + ASSERT_TRUE(value_array); + + // Build a map from the result + std::map result_map; + for (int64_t i = 0; i < struct_array->length(); ++i) { + result_map[key_array->GetString(i)] = value_array->GetString(i); + } + ASSERT_EQ(result_map["file-system"], "local"); + ASSERT_EQ(result_map["file.format"], "orc"); +} + +TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {"table.option.custom", "my-value"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + // Create a database and table + ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + // Verify basic enumeration works + ASSERT_OK_AND_ASSIGN(auto dbs, catalog->ListDatabases()); + ASSERT_TRUE(std::find(dbs.begin(), dbs.end(), "test_db") != dbs.end()); + ASSERT_OK_AND_ASSIGN(auto tbls, catalog->ListTables("test_db")); + ASSERT_TRUE(std::find(tbls.begin(), tbls.end(), "test_tbl") != tbls.end()); + + // Verify schema loads and has Options + ASSERT_OK_AND_ASSIGN(auto loaded_schema, + catalog->LoadTableSchema(Identifier("test_db", "test_tbl"))); + auto ds = std::dynamic_pointer_cast(loaded_schema); + ASSERT_TRUE(ds != nullptr) << "LoadTableSchema did not return DataSchema"; + ASSERT_FALSE(ds->Options().empty()) << "Table schema has no options"; + + // Directly test BuildRows + { + GlobalSystemTableContext ctx; + ctx.catalog = catalog.get(); + ctx.fs = catalog->GetFileSystem(); + ctx.warehouse = warehouse; + ctx.catalog_options = options; + AllTableOptionsSystemTable table(ctx); + ASSERT_OK_AND_ASSIGN(auto rows, table.BuildRows()); + ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row"; + } + + ASSERT_OK_AND_ASSIGN(auto result, + ReadGlobalSystemTable("all_table_options", catalog.get(), + catalog->GetFileSystem(), warehouse, options)); + auto struct_array = SingleStructChunk(result); + ASSERT_TRUE(struct_array); + ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows"; + + auto db_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto tbl_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto key_array = std::dynamic_pointer_cast(struct_array->field(2)); + auto val_array = std::dynamic_pointer_cast(struct_array->field(3)); + ASSERT_TRUE(db_array); + ASSERT_TRUE(tbl_array); + ASSERT_TRUE(key_array); + ASSERT_TRUE(val_array); + + // Verify that our table's options appear in the result + bool found_db = false; + bool found_format = false; + for (int64_t i = 0; i < struct_array->length(); ++i) { + auto db_name = std::string(db_array->GetString(i)); + auto tbl_name = std::string(tbl_array->GetString(i)); + if (db_name == "test_db" && tbl_name == "test_tbl") { + found_db = true; + auto key_str = std::string(key_array->GetString(i)); + if (key_str == "file.format") { + EXPECT_EQ(std::string(val_array->GetString(i)), "orc"); + found_format = true; + } + } + } + ASSERT_TRUE(found_db) << "test_db.test_tbl not found in sys.all_table_options"; + ASSERT_TRUE(found_format) << "file.format option not found in sys.all_table_options"; +} + +TEST(SystemTableReadInteTest, TestReadGlobalTables) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {"owner", "alice"}, + {"createdAt", "1000"}, + {"createdBy", "creator"}, + {"updatedAt", "2000"}, + {"updatedBy", "updater"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + auto fs = catalog->GetFileSystem(); + + // Create a database and a PK table + ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); + auto typed_schema = arrow::schema({ + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(std::string table_path, + catalog->GetTableLocation(Identifier("test_db", "test_tbl"))); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(table_path, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(typed_schema->fields()), R"([["k", 1]])", + /*partition_map=*/{}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + std::map no_pk_options = options; + no_pk_options[Options::BUCKET_KEY] = "pk"; + ::ArrowSchema no_pk_schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &no_pk_schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_no_pk_tbl"), &no_pk_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, no_pk_options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&no_pk_schema); + + ASSERT_OK_AND_ASSIGN(auto result, + ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options)); + auto struct_array = SingleStructChunk(result); + ASSERT_TRUE(struct_array); + ASSERT_GE(struct_array->length(), 1); + ASSERT_EQ(StructFieldNames(struct_array), + (std::vector{ + "database_name", "table_name", "table_type", "partitioned", "primary_key", + "owner", "created_at", "created_by", "updated_at", "updated_by", "record_count", + "file_size_in_bytes", "file_count", "last_file_creation_time"})); + + auto db_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto tbl_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto type_array = std::dynamic_pointer_cast(struct_array->field(2)); + auto part_array = std::dynamic_pointer_cast(struct_array->field(3)); + auto pk_array = std::dynamic_pointer_cast(struct_array->field(4)); + auto owner_array = std::dynamic_pointer_cast(struct_array->field(5)); + auto created_at_array = std::dynamic_pointer_cast(struct_array->field(6)); + auto created_by_array = std::dynamic_pointer_cast(struct_array->field(7)); + auto updated_at_array = std::dynamic_pointer_cast(struct_array->field(8)); + auto updated_by_array = std::dynamic_pointer_cast(struct_array->field(9)); + auto record_count_array = std::dynamic_pointer_cast(struct_array->field(10)); + auto file_size_array = std::dynamic_pointer_cast(struct_array->field(11)); + auto file_count_array = std::dynamic_pointer_cast(struct_array->field(12)); + auto last_creation_time_array = + std::dynamic_pointer_cast(struct_array->field(13)); + ASSERT_TRUE(db_array); + ASSERT_TRUE(tbl_array); + ASSERT_TRUE(type_array); + ASSERT_TRUE(part_array); + ASSERT_TRUE(pk_array); + ASSERT_TRUE(owner_array); + ASSERT_TRUE(created_at_array); + ASSERT_TRUE(created_by_array); + ASSERT_TRUE(updated_at_array); + ASSERT_TRUE(updated_by_array); + ASSERT_TRUE(record_count_array); + ASSERT_TRUE(file_size_array); + ASSERT_TRUE(file_count_array); + ASSERT_TRUE(last_creation_time_array); + + // Find our table by table name + bool found = false; + bool found_no_pk = false; + for (int64_t i = 0; i < struct_array->length(); ++i) { + if (std::string(tbl_array->GetString(i)) == "test_tbl") { + EXPECT_EQ(std::string(db_array->GetString(i)), "test_db"); + EXPECT_EQ(std::string(type_array->GetString(i)), "table"); + EXPECT_FALSE(part_array->Value(i)); + EXPECT_TRUE(pk_array->Value(i)); + EXPECT_EQ(owner_array->GetString(i), "alice"); + EXPECT_EQ(created_at_array->Value(i), 1000); + EXPECT_EQ(created_by_array->GetString(i), "creator"); + EXPECT_EQ(updated_at_array->Value(i), 2000); + EXPECT_EQ(updated_by_array->GetString(i), "updater"); + EXPECT_TRUE(record_count_array->IsNull(i)); + EXPECT_TRUE(file_size_array->IsNull(i)); + EXPECT_TRUE(file_count_array->IsNull(i)); + EXPECT_TRUE(last_creation_time_array->IsNull(i)); + found = true; + } else if (std::string(tbl_array->GetString(i)) == "test_no_pk_tbl") { + EXPECT_FALSE(pk_array->Value(i)); + found_no_pk = true; + } + } + ASSERT_TRUE(found) << "table not found in sys.tables"; + ASSERT_TRUE(found_no_pk) << "no-PK table not found in sys.tables"; +} + +TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = dir->Str(); + + arrow::FieldVector fields = { + arrow::field("dt", arrow::utf8()), + arrow::field("region", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(warehouse, schema, + /*partition_keys=*/{"dt", "region"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["2026-07-13", "cn", 1]])", + /*partition_map=*/{{"dt", "2026-07-13"}, {"region", "cn"}}, + /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + auto fs = catalog->GetFileSystem(); + ASSERT_OK_AND_ASSIGN(auto part_result, ReadGlobalSystemTable("partitions", catalog.get(), fs, + warehouse, options)); + auto struct_array = SingleStructChunk(part_result); + ASSERT_TRUE(struct_array); + ASSERT_EQ(StructFieldNames(struct_array), + (std::vector{"database_name", "table_name", "partition_name", + "record_count", "file_size_in_bytes", "file_count", + "last_file_creation_time", "done"})); + ASSERT_EQ(struct_array->length(), 1); + auto db_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto table_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto partition_array = std::dynamic_pointer_cast(struct_array->field(2)); + auto creation_time_array = std::dynamic_pointer_cast(struct_array->field(6)); + auto done_array = std::dynamic_pointer_cast(struct_array->field(7)); + ASSERT_TRUE(db_array); + ASSERT_TRUE(table_array); + ASSERT_TRUE(partition_array); + ASSERT_TRUE(creation_time_array); + ASSERT_TRUE(done_array); + EXPECT_EQ(db_array->GetString(0), "foo"); + EXPECT_EQ(table_array->GetString(0), "bar"); + EXPECT_EQ(partition_array->GetString(0), "dt=2026-07-13/region=cn"); + EXPECT_FALSE(creation_time_array->IsNull(0)); + EXPECT_FALSE(done_array->Value(0)); +} + +TEST(SystemTableReadInteTest, TestGlobalSystemTablesPropagateCorruptSchema) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = dir->Str(); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); + + auto typed_schema = arrow::schema({arrow::field("v", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + Identifier identifier("test_db", "test_tbl"); + ASSERT_OK(catalog->CreateTable(identifier, &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(std::string table_path, catalog->GetTableLocation(identifier)); + std::shared_ptr fs = catalog->GetFileSystem(); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "schema/schema-0"), "{invalid-json", + /*overwrite=*/true)); + + for (const std::string system_table : {"all_table_options", "tables", "partitions"}) { + ASSERT_NOK(ReadGlobalSystemTable(system_table, catalog.get(), fs, warehouse, options)); + } +} + +TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptSnapshot) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = dir->Str(); + + arrow::FieldVector fields = { + arrow::field("dt", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(warehouse, schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["2026-07-19", 1]])", + /*partition_map=*/{{"dt", "2026-07-19"}}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + std::shared_ptr fs = catalog->GetFileSystem(); + std::string table_path = PathUtil::JoinPath(warehouse, "foo.db/bar"); + ASSERT_OK(fs->WriteFile(PathUtil::JoinPath(table_path, "snapshot/snapshot-1"), "{invalid-json", + /*overwrite=*/true)); + + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); +} + +TEST(SystemTableReadInteTest, TestPartitionsSystemTablePropagatesCorruptManifest) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}, + {Options::BUCKET_KEY, "v"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = dir->Str(); + + arrow::FieldVector fields = { + arrow::field("dt", arrow::utf8()), + arrow::field("v", arrow::int32()), + }; + auto schema = arrow::schema(fields); + ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(warehouse, schema, + /*partition_keys=*/{"dt"}, + /*primary_keys=*/{}, options, + /*is_streaming_mode=*/true)); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr batch, + TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["2026-07-19", 1]])", + /*partition_map=*/{{"dt", "2026-07-19"}}, /*bucket=*/0, {})); + ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0, + /*expected_commit_messages=*/std::nullopt)); + + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + std::shared_ptr fs = catalog->GetFileSystem(); + std::string table_path = PathUtil::JoinPath(warehouse, "foo.db/bar"); + SnapshotManager snapshot_manager(fs, table_path); + ASSERT_OK_AND_ASSIGN(std::optional snapshot, snapshot_manager.LatestSnapshot()); + ASSERT_TRUE(snapshot); + ASSERT_OK(fs->WriteFile( + PathUtil::JoinPath(table_path, "manifest/" + snapshot->BaseManifestList()), "corrupt", + /*overwrite=*/true)); + + ASSERT_NOK(ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); +} + } // namespace paimon::test From 60011b4992c3fa369f6f255793c0d576f9eefa39 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:17:02 +0800 Subject: [PATCH 115/138] refactor(format): remove Lance support --- .devcontainer/centos7/run.sh | 1 - build_support/lsan-suppressions.txt | 7 ------ build_support/tsan-suppressions.txt | 2 -- include/paimon/format/reader_builder.h | 2 +- scripts/tantivy_smoke.sh | 1 - src/paimon/common/fs/file_system_test.cpp | 3 ++- src/paimon/format/avro/avro_reader_builder.h | 3 --- .../testing/mock/mock_format_reader_builder.h | 1 - test/inte/blob_table_inte_test.cpp | 24 ------------------- test/inte/write_and_read_inte_test.cpp | 4 +--- test/inte/write_inte_test.cpp | 2 +- 11 files changed, 5 insertions(+), 45 deletions(-) diff --git a/.devcontainer/centos7/run.sh b/.devcontainer/centos7/run.sh index 17818b56..a33ad247 100755 --- a/.devcontainer/centos7/run.sh +++ b/.devcontainer/centos7/run.sh @@ -126,7 +126,6 @@ case "${cmd}" in -DPAIMON_BUILD_TESTS=ON \ -DPAIMON_ENABLE_FSLIB=OFF \ -DPAIMON_ENABLE_LUMINA=OFF \ - -DPAIMON_ENABLE_LANCE=OFF \ -DPAIMON_ENABLE_JINDO=OFF \ -DPAIMON_ENABLE_LUCENE=ON \ -DPAIMON_ENABLE_ORC=ON \ diff --git a/build_support/lsan-suppressions.txt b/build_support/lsan-suppressions.txt index e9f3bb0c..927afb39 100644 --- a/build_support/lsan-suppressions.txt +++ b/build_support/lsan-suppressions.txt @@ -17,10 +17,3 @@ # False positive from atexit() registration in libc leak:*__new_exitfn* - -# Lance's Rust/Tokio runtime can leave worker/TLS allocations alive at process -# exit. Suppress these third-party runtime shutdown leftovers without hiding -# all leaks from liblance_lib_rc.so. -leak:tokio::runtime::blocking::pool::spawn_blocking -leak:tokio::runtime::scheduler::multi_thread::worker::create -leak:std::thread::Builder::spawn_unchecked diff --git a/build_support/tsan-suppressions.txt b/build_support/tsan-suppressions.txt index 4c7aef1b..a5b1d655 100644 --- a/build_support/tsan-suppressions.txt +++ b/build_support/tsan-suppressions.txt @@ -16,7 +16,5 @@ # under the License. # Prebuilt shared libraries are not TSAN-instrumented. Suppress reports from the whole library. -race:liblance_lib_rc.so -thread:liblance_lib_rc.so race:liblumina.so race:libjindosdk_c.so.6 diff --git a/include/paimon/format/reader_builder.h b/include/paimon/format/reader_builder.h index 7ef1bf8f..b0837a26 100644 --- a/include/paimon/format/reader_builder.h +++ b/include/paimon/format/reader_builder.h @@ -27,7 +27,7 @@ namespace paimon { class Cache; -/// Create a file batch reader based on the file path. Allows you to specify memory pool. +/// Create a file batch reader based on an input stream. Allows you to specify memory pool. class PAIMON_EXPORT ReaderBuilder { public: virtual ~ReaderBuilder() = default; diff --git a/scripts/tantivy_smoke.sh b/scripts/tantivy_smoke.sh index e4598418..0e3adad7 100755 --- a/scripts/tantivy_smoke.sh +++ b/scripts/tantivy_smoke.sh @@ -61,7 +61,6 @@ if [ "${DO_CONFIGURE}" = "1" ]; then -DPAIMON_USE_TSAN="${USE_TSAN}" \ -DPAIMON_ENABLE_FSLIB=OFF \ -DPAIMON_ENABLE_LUMINA=OFF \ - -DPAIMON_ENABLE_LANCE=OFF \ -DPAIMON_ENABLE_JINDO=OFF \ -DPAIMON_ENABLE_LUCENE=ON \ -DPAIMON_ENABLE_ORC=ON \ diff --git a/src/paimon/common/fs/file_system_test.cpp b/src/paimon/common/fs/file_system_test.cpp index c40a7678..49482381 100644 --- a/src/paimon/common/fs/file_system_test.cpp +++ b/src/paimon/common/fs/file_system_test.cpp @@ -860,7 +860,8 @@ TEST_P(FileSystemTest, TestExistingFileDeletion) { TEST_P(FileSystemTest, TestNotExistingFileDeletion) { auto check = [&](bool recursive) { std::string path = PathUtil::JoinPath(test_root_, RandomName()); - ASSERT_TRUE(fs_->Delete(path, recursive).IsIOError()); + Status status = fs_->Delete(path, recursive); + ASSERT_TRUE(status.IsIOError() || status.IsNotExist()) << status.ToString(); }; check(true); check(false); diff --git a/src/paimon/format/avro/avro_reader_builder.h b/src/paimon/format/avro/avro_reader_builder.h index eb87917b..f19d8683 100644 --- a/src/paimon/format/avro/avro_reader_builder.h +++ b/src/paimon/format/avro/avro_reader_builder.h @@ -18,10 +18,7 @@ #pragma once -#include #include -#include -#include #include "avro/DataFile.hh" #include "paimon/format/avro/avro_file_batch_reader.h" diff --git a/src/paimon/testing/mock/mock_format_reader_builder.h b/src/paimon/testing/mock/mock_format_reader_builder.h index 7f8906cd..16819611 100644 --- a/src/paimon/testing/mock/mock_format_reader_builder.h +++ b/src/paimon/testing/mock/mock_format_reader_builder.h @@ -19,7 +19,6 @@ #pragma once #include -#include #include "arrow/type.h" #include "paimon/format/reader_builder.h" diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 9c7406cd..41887b5a 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -1948,9 +1948,6 @@ TEST_P(BlobTableInteTest, TestReadTableWithMultiBlobFields) { } TEST_P(BlobTableInteTest, TestBlobDescriptorField) { - if (GetParam() == "lance") { - return; - } // Two blob fields configured via BLOB_DESCRIPTOR_FIELD and stored inline as descriptors. arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), @@ -2004,9 +2001,6 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorField) { } TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) { - if (GetParam() == "lance") { - return; - } // 4 blob fields: b0,b1 are inline descriptors; b2,b3 are regular blob fields written to // .blob files. arrow::FieldVector fields = { @@ -2068,9 +2062,6 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldPartialInline) { } TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) { - if (GetParam() == "lance") { - return; - } // Multiple write+commit rounds with a shuffled read schema: b3, b2, b1, b0, f0. arrow::FieldVector fields = { arrow::field("f0", arrow::int32()), BlobUtils::ToArrowField("b0", true), @@ -2441,9 +2432,6 @@ TEST_P(BlobTableInteTest, TestOrcMapStorageLayoutEvolutionWithBlobDataEvolution) } TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { - if (GetParam() == "lance") { - return; - } // Test DataEvolution (split-column write) combined with blob descriptor fields. // Schema: f0(int32), b0/b1(blob descriptor inline), b2/b3(blob). // Commit 1: file A writes (f0, b2, b3) @@ -2565,9 +2553,6 @@ TEST_P(BlobTableInteTest, TestDataEvolutionWithBlobDescriptorField) { } TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) { - if (GetParam() == "lance") { - return; - } // Similar to TestBlobDescriptorField but writes raw bytes directly without converting to // descriptor first. Descriptor fields reject values without the descriptor magic header. arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), @@ -2922,9 +2907,6 @@ TEST_P(BlobTableInteTest, TestForwardBlobViewReference) { TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) { auto file_format = GetParam(); - if (GetParam() == "lance") { - return; - } // Upstream table has two blob descriptor fields. The downstream view references cells from // both b0 (field_id=1) and b1 (field_id=2). const std::string upstream_db_name = "upstream_two_blob"; @@ -3211,9 +3193,6 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithMultipleUpstreamTables) { TEST_P(BlobTableInteTest, TestBlobViewFailsWhenBothPathsAbsent) { auto file_format = GetParam(); - if (GetParam() == "lance") { - return; - } auto upstream_dir = UniqueTestDirectory::Create("local"); const std::string upstream_db_name = "nonexistent_db"; const std::string upstream_table_name = "nonexistent_table"; @@ -3264,9 +3243,6 @@ TEST_P(BlobTableInteTest, TestBlobViewFailsWhenBothPathsAbsent) { TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) { auto file_format = GetParam(); - if (GetParam() == "lance") { - return; - } const std::string upstream_db_name = "fallback_db"; const std::string upstream_table_name = "fallback_table"; arrow::FieldVector upstream_fields = {arrow::field("f0", arrow::int32()), diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp index a58f25ce..d27d08e3 100644 --- a/test/inte/write_and_read_inte_test.cpp +++ b/test/inte/write_and_read_inte_test.cpp @@ -365,10 +365,8 @@ TEST_P(WriteAndReadInteTest, TestPKSimple) { } TEST_P(WriteAndReadInteTest, TestNestedType) { - // Represent a map as list(struct(key, value)) for cross-format compatibility. arrow::FieldVector fields = { - arrow::field("f1", arrow::list(arrow::struct_({arrow::field("key", arrow::int8()), - arrow::field("value", arrow::int16())}))), + arrow::field("f1", arrow::map(arrow::int8(), arrow::int16())), arrow::field("f2", arrow::list(arrow::float32())), arrow::field("f3", arrow::struct_({arrow::field("f0", arrow::boolean()), arrow::field("f1", arrow::int64())})), diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp index a5e43bd1..114dc9e4 100644 --- a/test/inte/write_inte_test.cpp +++ b/test/inte/write_inte_test.cpp @@ -1942,7 +1942,7 @@ TEST_P(WriteInteTest, TestPkTableWriteWithIOException) { // Loop bound must exceed the workflow's total IO operations so the loop can // naturally terminate at the iteration where injection position falls past - // the last IO. Measured IO counts: orc=310, parquet=506, avro=195, lance=69. + // the last IO. Measured IO counts: orc=310, parquet=506, avro=195. // 1000 leaves headroom for future format/workflow changes. for (size_t i = 0; i < 1000; i++) { auto dir = UniqueTestDirectory::Create(); From bd4466d7097c87152246eb34ab61d4eaf24032c9 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:00:14 +0800 Subject: [PATCH 116/138] refactor(tantivy): move FFI crate under crates --- .devcontainer/centos7/Dockerfile | 5 +- .devcontainer/centos7/run.sh | 29 +------ .devcontainer/devcontainer.json.template | 6 +- .../x86_64/devcontainer.json.template | 6 +- .gitignore | 2 +- CMakeLists.txt | 10 +-- ci/scripts/setup_rust.sh | 5 +- cmake_modules/CorrosionFetch.cmake | 5 +- .../tantivy_ffi/Cargo.lock | 0 .../tantivy_ffi/Cargo.toml | 16 ++++ {third_party => crates}/tantivy_ffi/build.rs | 16 ++++ .../tantivy_ffi/cbindgen.toml | 20 ++++- crates/tantivy_ffi/rust-toolchain.toml | 25 ++++++ .../tantivy_ffi/src/buffer.rs | 16 ++++ .../tantivy_ffi/src/callback_directory.rs | 16 ++++ .../tantivy_ffi/src/error.rs | 16 ++++ .../tantivy_ffi/src/handle.rs | 16 ++++ .../tantivy_ffi/src/lib.rs | 16 ++++ .../tantivy_ffi/src/log_bridge.rs | 16 ++++ .../tantivy_ffi/src/reader.rs | 19 ++++- .../tantivy_ffi/src/tokenizer.rs | 16 ++++ .../tantivy_ffi/src/writer.rs | 21 ++++- docs/code-style.md | 2 +- docs/requirements.txt | 4 - scripts/tantivy_smoke.sh | 82 ------------------- .../global_index/tantivy/CMakeLists.txt | 24 +++--- .../test_data/java_tantivy_fixtures/README.md | 12 +-- .../english_simple.golden.json | 2 +- third_party/tantivy_ffi/rust-toolchain.toml | 11 --- 29 files changed, 263 insertions(+), 171 deletions(-) rename {third_party => crates}/tantivy_ffi/Cargo.lock (100%) rename {third_party => crates}/tantivy_ffi/Cargo.toml (50%) rename {third_party => crates}/tantivy_ffi/build.rs (65%) rename {third_party => crates}/tantivy_ffi/cbindgen.toml (69%) create mode 100644 crates/tantivy_ffi/rust-toolchain.toml rename {third_party => crates}/tantivy_ffi/src/buffer.rs (80%) rename {third_party => crates}/tantivy_ffi/src/callback_directory.rs (95%) rename {third_party => crates}/tantivy_ffi/src/error.rs (84%) rename {third_party => crates}/tantivy_ffi/src/handle.rs (79%) rename {third_party => crates}/tantivy_ffi/src/lib.rs (76%) rename {third_party => crates}/tantivy_ffi/src/log_bridge.rs (81%) rename {third_party => crates}/tantivy_ffi/src/reader.rs (98%) rename {third_party => crates}/tantivy_ffi/src/tokenizer.rs (95%) rename {third_party => crates}/tantivy_ffi/src/writer.rs (97%) delete mode 100755 scripts/tantivy_smoke.sh delete mode 100644 third_party/tantivy_ffi/rust-toolchain.toml diff --git a/.devcontainer/centos7/Dockerfile b/.devcontainer/centos7/Dockerfile index cd14667e..01940d02 100644 --- a/.devcontainer/centos7/Dockerfile +++ b/.devcontainer/centos7/Dockerfile @@ -36,7 +36,8 @@ # source /opt/paimon-env.sh # PATH for rust, cmake # cd /workspaces/paimon-cpp # git lfs install --local && git lfs pull # critical: boost & friends are LFS -# ./scripts/tantivy_smoke.sh +# +# Run ./.devcontainer/centos7/run.sh smoke from the host for the full check. # ---------- Base ---------- # CentOS 7 reached EOL 2024-06-30; its default mirrorlist.centos.org is down. @@ -238,4 +239,4 @@ CMD ["bash", "-lc", "\ echo '--- rust ---'; rustc --version; \ echo '--- cargo ---'; cargo --version; \ echo '--- cbindgen ---'; cbindgen --version; \ - echo 'Ready. Mount paimon-cpp at /workspaces/paimon-cpp and run ./scripts/tantivy_smoke.sh'"] + echo 'Ready. From the host repository, run ./.devcontainer/centos7/run.sh smoke'"] diff --git a/.devcontainer/centos7/run.sh b/.devcontainer/centos7/run.sh index a33ad247..d9ae5d21 100755 --- a/.devcontainer/centos7/run.sh +++ b/.devcontainer/centos7/run.sh @@ -22,7 +22,7 @@ # ./.devcontainer/centos7/run.sh build # build image only # ./.devcontainer/centos7/run.sh up # start container (detached) # ./.devcontainer/centos7/run.sh shell # exec into it -# ./.devcontainer/centos7/run.sh smoke # run scripts/tantivy_smoke.sh inside +# ./.devcontainer/centos7/run.sh smoke # run the smoke suite inside # ./.devcontainer/centos7/run.sh down # stop + remove set -euo pipefail @@ -61,35 +61,19 @@ case "${cmd}" in ;; up) docker rm -f "${CONTAINER}" 2>/dev/null || true - # Mount host SSH keys read-only (mirrors paimon-dev) so git clones of - # internal repos (e.g. aliorc_ep on gitlab.alibaba-inc.com) that go - # over SSH can authenticate with the host's key. Skip the mount if - # ~/.ssh doesn't exist so the script still works for external users. - ssh_mount=() - if [ -d "${HOME}/.ssh" ]; then - ssh_mount=(-v "${HOME}/.ssh:/home/paimon/.ssh:ro") - fi docker run -d \ --name "${CONTAINER}" \ --privileged \ -v "${repo}:/workspaces/paimon-cpp" \ -v "paimon-centos7-cargo-registry:/opt/rust/cargo/registry" \ -v "paimon-centos7-build:/workspaces/paimon-cpp/build-centos7" \ - "${ssh_mount[@]}" \ "${IMAGE}" sleep infinity # Named volumes mount as root-owned; `paimon` user (uid 1000) needs # write access to build-centos7 and the cargo registry cache. - # Also set up the gitlab.alibaba-inc.com url rewrite so aliorc_ep - # (and any other ExternalProject pointing at internal gitlab via - # http://) picks up the mounted SSH key. docker exec --user root "${CONTAINER}" bash -c ' chown -R paimon:paimon /workspaces/paimon-cpp/build-centos7 \ /opt/rust/cargo/registry ' - docker exec "${CONTAINER}" bash -c ' - git config --global url."git@gitlab.alibaba-inc.com:".insteadOf \ - "http://gitlab.alibaba-inc.com/" - ' echo "Container started. \`${0} shell\` to enter." ;; shell) @@ -101,7 +85,7 @@ case "${cmd}" in echo "Container ${CONTAINER} not running; starting it." "$0" up fi - # Two env vars pass through for Rosetta 2 (Apple Silicon) compat: + # Set two environment variables for Rosetta 2 (Apple Silicon) compatibility: # MALLOC_CHECK_=0 disables glibc 2.17 extra malloc integrity checks # that fire false positives under Rosetta's x86_64 emulation. # ARROW_USER_SIMD_LEVEL=SSE4_2 keeps arrow runtime-dispatched kernels @@ -109,10 +93,7 @@ case "${cmd}" in # Both are no-ops on real x86_64 CentOS 7 hardware. # Use a distinct build dir inside the container so it does not clash # with the Ubuntu dev container's build/ dir on the same volume. - # Propagate PAIMON_ENABLE_ALIORC so `PAIMON_ENABLE_ALIORC=OFF` env - # on the host reaches the cmake inside the container. docker exec \ - -e "PAIMON_ENABLE_ALIORC=${PAIMON_ENABLE_ALIORC:-ON}" \ -e "MALLOC_CHECK_=0" \ -e "ARROW_USER_SIMD_LEVEL=SSE4_2" \ "${CONTAINER}" bash -lc ' @@ -128,13 +109,9 @@ case "${cmd}" in -DPAIMON_ENABLE_LUMINA=OFF \ -DPAIMON_ENABLE_JINDO=OFF \ -DPAIMON_ENABLE_LUCENE=ON \ + -DPAIMON_ENABLE_TANTIVY=ON \ -DPAIMON_ENABLE_ORC=ON \ - -DPAIMON_ENABLE_ALIORC="${PAIMON_ENABLE_ALIORC:-ON}" \ -DPAIMON_ENABLE_AVRO=ON - # ALIORC clones from internal gitlab. `up` mounts $HOME/.ssh and - # configures the url.insteadOf rewrite, so by default ALIORC works - # for alibaba-inc users. External users without gitlab access can - # opt out with `PAIMON_ENABLE_ALIORC=OFF ./run.sh smoke`. cmake --build build-centos7 -j "$(nproc)" ctest --test-dir build-centos7 \ -R "paimon-lucene-index-test|paimon-global-index-test|paimon-tantivy-.*-test" \ diff --git a/.devcontainer/devcontainer.json.template b/.devcontainer/devcontainer.json.template index bc170f50..f5a6c742 100644 --- a/.devcontainer/devcontainer.json.template +++ b/.devcontainer/devcontainer.json.template @@ -48,11 +48,11 @@ "source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly", "source=paimon-cargo-registry,target=/home/paimon/.cargo/registry,type=volume", "source=paimon-cargo-git,target=/home/paimon/.cargo/git,type=volume", - "source=paimon-rust-target,target=${containerWorkspaceFolder}/third_party/tantivy_ffi/target,type=volume", + "source=paimon-rust-target,target=${containerWorkspaceFolder}/crates/tantivy_ffi/target,type=volume", "source=paimon-build,target=${containerWorkspaceFolder}/build,type=volume", "source=paimon-ccache,target=/home/paimon/.ccache,type=volume" ], - "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/third_party/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true", + "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true", "customizations": { "vscode": { "extensions": [ @@ -66,7 +66,7 @@ "settings": { "editor.formatOnSave": true, "rust-analyzer.linkedProjects": [ - "third_party/tantivy_ffi/Cargo.toml" + "crates/tantivy_ffi/Cargo.toml" ] } } diff --git a/.devcontainer/x86_64/devcontainer.json.template b/.devcontainer/x86_64/devcontainer.json.template index baa40099..00763314 100644 --- a/.devcontainer/x86_64/devcontainer.json.template +++ b/.devcontainer/x86_64/devcontainer.json.template @@ -19,7 +19,7 @@ // x86_64 variant of the Paimon CPP Dev Container. // On Apple Silicon hosts this runs under QEMU emulation (5-10x slower). -// Use it ONLY for cross-architecture verification (Stage 11), not daily dev. +// Use it only for cross-architecture verification, not daily development. // // Reuses the same Dockerfile as the default container; only the platform differs. // @@ -54,11 +54,11 @@ "source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly", "source=paimon-cargo-registry-amd64,target=/home/paimon/.cargo/registry,type=volume", "source=paimon-cargo-git-amd64,target=/home/paimon/.cargo/git,type=volume", - "source=paimon-rust-target-amd64,target=${containerWorkspaceFolder}/third_party/tantivy_ffi/target,type=volume", + "source=paimon-rust-target-amd64,target=${containerWorkspaceFolder}/crates/tantivy_ffi/target,type=volume", "source=paimon-build-amd64,target=${containerWorkspaceFolder}/build,type=volume", "source=paimon-ccache-amd64,target=/home/paimon/.ccache,type=volume" ], - "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/third_party/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true", + "postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true", "customizations": { "vscode": { "extensions": [ diff --git a/.gitignore b/.gitignore index 3ff833af..4d213996 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,4 @@ FlameGraph third_party/*.tar.gz # Rust / Cargo build artifacts -third_party/tantivy_ffi/target/ +crates/tantivy_ffi/target/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 05047055..511aa6f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,9 +19,9 @@ cmake_minimum_required(VERSION 3.22) # 3.22 is the minimum required by Corrosion-rs (used for the Rust<->C++ FFI -# integration, see third_party/tantivy_ffi). Ubuntu 24.04 ships CMake 3.28 and -# CentOS 8+/RHEL 9+ ship 3.20+. To build on older distros, see -# docs/dev/tantivy_fts_migration_plan.md. +# integration, see crates/tantivy_ffi). Ubuntu 24.04 ships CMake 3.28 and +# CentOS 8+/RHEL 9+ ship 3.20+, so older distributions need a newer CMake +# installation. message(STATUS "Building using CMake version: ${CMAKE_VERSION}") # https://cmake.org/cmake/help/latest/policy/CMP0135.html @@ -302,15 +302,13 @@ add_subdirectory(third_party/roaring_bitmap EXCLUDE_FROM_ALL) add_subdirectory(third_party/xxhash EXCLUDE_FROM_ALL) # ---- tantivy-fulltext Rust FFI via Corrosion-rs -------------------------------- -# See docs/dev/tantivy_fts_migration_plan.md Stage 1. -# # Corrosion wraps the Cargo crate as a CMake target named `paimon_tantivy_ffi`. # `corrosion_experimental_cbindgen` runs cbindgen from CMake and writes the # header to a stable path; it also adds that path to the target's INTERFACE # include dirs so C++ consumers pick it up via target_link_libraries. if(PAIMON_ENABLE_TANTIVY) include(CorrosionFetch) - corrosion_import_crate(MANIFEST_PATH third_party/tantivy_ffi/Cargo.toml CRATES + corrosion_import_crate(MANIFEST_PATH crates/tantivy_ffi/Cargo.toml CRATES paimon_tantivy_ffi) corrosion_experimental_cbindgen(TARGET paimon_tantivy_ffi HEADER_NAME paimon_tantivy_ffi.h) diff --git a/ci/scripts/setup_rust.sh b/ci/scripts/setup_rust.sh index 64ae9aea..bdb8f622 100755 --- a/ci/scripts/setup_rust.sh +++ b/ci/scripts/setup_rust.sh @@ -16,12 +16,11 @@ # limitations under the License. # # Install the Rust toolchain + cbindgen required to build the -# tantivy-fts FFI crate (third_party/tantivy_ffi) from CI. +# tantivy-fts FFI crate (crates/tantivy_ffi) from CI. # # The dev container (see .devcontainer/) already has these preinstalled; # this script is for the GitHub Actions runners. Called by -# .github/workflows/gcc_test.yaml and test_with_sanitizer.yaml before -# ci/scripts/build_paimon.sh. +# .github/workflows/build_and_test.yaml before ci/scripts/build_paimon.sh. # # Idempotent: a second invocation is a no-op when the tools already exist. diff --git a/cmake_modules/CorrosionFetch.cmake b/cmake_modules/CorrosionFetch.cmake index 40ab1d2b..b56b2795 100644 --- a/cmake_modules/CorrosionFetch.cmake +++ b/cmake_modules/CorrosionFetch.cmake @@ -14,9 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Pull Corrosion-rs via FetchContent so we can import Cargo crates as CMake -# targets. Used to bring in third_party/tantivy_ffi for the tantivy-fulltext -# global index (see docs/dev/tantivy_fts_migration_plan.md). +# Pull Corrosion-rs via FetchContent so we can import crates/tantivy_ffi as +# CMake targets for the tantivy-fulltext global index. # # Pinned to v0.5.2 (stable release). Requires CMake >= 3.22. diff --git a/third_party/tantivy_ffi/Cargo.lock b/crates/tantivy_ffi/Cargo.lock similarity index 100% rename from third_party/tantivy_ffi/Cargo.lock rename to crates/tantivy_ffi/Cargo.lock diff --git a/third_party/tantivy_ffi/Cargo.toml b/crates/tantivy_ffi/Cargo.toml similarity index 50% rename from third_party/tantivy_ffi/Cargo.toml rename to crates/tantivy_ffi/Cargo.toml index 68fe24dd..ea843bd9 100644 --- a/third_party/tantivy_ffi/Cargo.toml +++ b/crates/tantivy_ffi/Cargo.toml @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [package] name = "paimon_tantivy_ffi" version = "0.1.0" diff --git a/third_party/tantivy_ffi/build.rs b/crates/tantivy_ffi/build.rs similarity index 65% rename from third_party/tantivy_ffi/build.rs rename to crates/tantivy_ffi/build.rs index 107d3f42..72d76b6f 100644 --- a/third_party/tantivy_ffi/build.rs +++ b/crates/tantivy_ffi/build.rs @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! build.rs: runs cbindgen to generate the C header paimon_tantivy_ffi.h. //! //! Output path: $OUT_DIR/paimon_tantivy_ffi.h diff --git a/third_party/tantivy_ffi/cbindgen.toml b/crates/tantivy_ffi/cbindgen.toml similarity index 69% rename from third_party/tantivy_ffi/cbindgen.toml rename to crates/tantivy_ffi/cbindgen.toml index 646051b0..c2ab5f13 100644 --- a/third_party/tantivy_ffi/cbindgen.toml +++ b/crates/tantivy_ffi/cbindgen.toml @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # cbindgen config: Rust FFI -> C header generator. # Invoked by build.rs, outputs to $OUT_DIR/paimon_tantivy_ffi.h. # CMake picks up $OUT_DIR via Corrosion and adds it to the C++ include path. @@ -25,9 +41,9 @@ header = """ * under the License. */ /* - * AUTO-GENERATED by cbindgen from Rust sources under third_party/tantivy_ffi - DO NOT EDIT. + * AUTO-GENERATED by cbindgen from Rust sources under crates/tantivy_ffi - DO NOT EDIT. * - * C ABI for paimon_tantivy_ffi. See docs/dev/tantivy_ffi_design.md for contract. + * C ABI for paimon_tantivy_ffi. */ #pragma once """ diff --git a/crates/tantivy_ffi/rust-toolchain.toml b/crates/tantivy_ffi/rust-toolchain.toml new file mode 100644 index 00000000..4c1d88ae --- /dev/null +++ b/crates/tantivy_ffi/rust-toolchain.toml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pin the Rust toolchain used to build paimon_tantivy_ffi so Corrosion can +# resolve a non-empty toolchain name during a fresh CMake configure. +# +# Only the `channel` is pinned — no extra components, because rustup in +# CI/containers may lack network access to fetch clippy/rustfmt, and build +# doesn't need them. +[toolchain] +channel = "stable" +profile = "minimal" diff --git a/third_party/tantivy_ffi/src/buffer.rs b/crates/tantivy_ffi/src/buffer.rs similarity index 80% rename from third_party/tantivy_ffi/src/buffer.rs rename to crates/tantivy_ffi/src/buffer.rs index 13e9f43f..2d96e89d 100644 --- a/third_party/tantivy_ffi/src/buffer.rs +++ b/crates/tantivy_ffi/src/buffer.rs @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! `paimon_tantivy_buffer_t`: Rust-allocated byte buffer returned to C++. //! //! Contract: diff --git a/third_party/tantivy_ffi/src/callback_directory.rs b/crates/tantivy_ffi/src/callback_directory.rs similarity index 95% rename from third_party/tantivy_ffi/src/callback_directory.rs rename to crates/tantivy_ffi/src/callback_directory.rs index fabeb3cb..12768f32 100644 --- a/third_party/tantivy_ffi/src/callback_directory.rs +++ b/crates/tantivy_ffi/src/callback_directory.rs @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! PaimonCallbackDirectory: streaming tantivy `Directory` backed by C FFI //! callbacks. Replaces the V1 `PaimonDirectory` (RamDirectory wrapper) with a //! callback-driven design that mirrors Java paimon-tantivy-jni's `JniDirectory`. diff --git a/third_party/tantivy_ffi/src/error.rs b/crates/tantivy_ffi/src/error.rs similarity index 84% rename from third_party/tantivy_ffi/src/error.rs rename to crates/tantivy_ffi/src/error.rs index 6be463c7..36c36f48 100644 --- a/third_party/tantivy_ffi/src/error.rs +++ b/crates/tantivy_ffi/src/error.rs @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! Error model for paimon_tantivy_ffi. //! //! Contract: diff --git a/third_party/tantivy_ffi/src/handle.rs b/crates/tantivy_ffi/src/handle.rs similarity index 79% rename from third_party/tantivy_ffi/src/handle.rs rename to crates/tantivy_ffi/src/handle.rs index 6ec776a2..7bd98e73 100644 --- a/third_party/tantivy_ffi/src/handle.rs +++ b/crates/tantivy_ffi/src/handle.rs @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! Opaque handle helpers. //! //! Contract: diff --git a/third_party/tantivy_ffi/src/lib.rs b/crates/tantivy_ffi/src/lib.rs similarity index 76% rename from third_party/tantivy_ffi/src/lib.rs rename to crates/tantivy_ffi/src/lib.rs index fc544c90..df4b80a5 100644 --- a/third_party/tantivy_ffi/src/lib.rs +++ b/crates/tantivy_ffi/src/lib.rs @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! paimon_tantivy_ffi: C ABI layer for tantivy + jieba-rs, //! consumed by paimon-cpp's `tantivy-fulltext` global index. //! diff --git a/third_party/tantivy_ffi/src/log_bridge.rs b/crates/tantivy_ffi/src/log_bridge.rs similarity index 81% rename from third_party/tantivy_ffi/src/log_bridge.rs rename to crates/tantivy_ffi/src/log_bridge.rs index 7f4ab00f..6b6b286f 100644 --- a/third_party/tantivy_ffi/src/log_bridge.rs +++ b/crates/tantivy_ffi/src/log_bridge.rs @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! Log bridge: tantivy internally emits log records via the `log` crate //! (via `tantivy::debug` / `info` etc.). This module registers a global //! `log::Log` implementation that forwards records to a C callback. diff --git a/third_party/tantivy_ffi/src/reader.rs b/crates/tantivy_ffi/src/reader.rs similarity index 98% rename from third_party/tantivy_ffi/src/reader.rs rename to crates/tantivy_ffi/src/reader.rs index 4bd8e0fa..460e4447 100644 --- a/third_party/tantivy_ffi/src/reader.rs +++ b/crates/tantivy_ffi/src/reader.rs @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! PaimonTantivyReader: query side of tantivy-fulltext. //! //! Constructs a tantivy Index from a packed-blob produced by writer.rs (via @@ -257,8 +273,7 @@ impl PaimonTantivyReader { Ok(ids) } - /// 4-path dispatch on `(with_score, limit)` — see `docs/dev/tantivy_bm25_score_contract.md` - /// §4. + /// Four-path dispatch on `(with_score, limit)`. /// /// | with_score | limit | path | collector | sort | truncate | output score | /// |------------|--------|------|------------------------|----------------|----------|--------------| diff --git a/third_party/tantivy_ffi/src/tokenizer.rs b/crates/tantivy_ffi/src/tokenizer.rs similarity index 95% rename from third_party/tantivy_ffi/src/tokenizer.rs rename to crates/tantivy_ffi/src/tokenizer.rs index e3e69f24..b8f8b5bb 100644 --- a/third_party/tantivy_ffi/src/tokenizer.rs +++ b/crates/tantivy_ffi/src/tokenizer.rs @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! PaimonJiebaTokenizer: tantivy Tokenizer impl wrapping jieba-rs. //! //! Contract: diff --git a/third_party/tantivy_ffi/src/writer.rs b/crates/tantivy_ffi/src/writer.rs similarity index 97% rename from third_party/tantivy_ffi/src/writer.rs rename to crates/tantivy_ffi/src/writer.rs index 3ea5ab4b..1a068e7b 100644 --- a/third_party/tantivy_ffi/src/writer.rs +++ b/crates/tantivy_ffi/src/writer.rs @@ -1,6 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + //! PaimonTantivyWriter: Writer for tantivy-fulltext global index. //! -//! Contract (see docs/dev/tantivy_java_compat_plan.md §2.5 + §5.1 J2): +//! Contract: //! - `writer_new(field_name, mode, with_position, dict_dir, out)` — create on a //! private tmp dir backed by MmapDirectory + PaimonJiebaTokenizer. //! `field_name` is **ignored** by the Rust schema (kept for FFI ABI @@ -12,8 +28,7 @@ //! single segment + pack all on-disk index files into a Rust-allocated buffer //! - `writer_free(writer)` — destroy (RAII removes tmp dir) //! -//! Packing format (big-endian, **cross-readable with paimon-java archive**; -//! see `paimon-tantivy-index/README.md` §Archive File Format): +//! Packing format (big-endian, **cross-readable with paimon-java archives**): //! `[i32 BE file_count | //! (i32 BE name_len | name_bytes | i64 BE file_len | file_bytes)*]` diff --git a/docs/code-style.md b/docs/code-style.md index 7cf1d650..23ddb5c6 100644 --- a/docs/code-style.md +++ b/docs/code-style.md @@ -441,4 +441,4 @@ Before submitting a PR, please verify: - [ ] Tests are added or updated for the changed functionality. - [ ] `ASSERT_*` is preferred over `EXPECT_*` in tests. - [ ] No raw `new` / `delete` outside factory methods. -- [ ] PR description follows the [template](.github/PULL_REQUEST_TEMPLATE.md). +- [ ] PR description follows the [template](../.github/PULL_REQUEST_TEMPLATE.md). diff --git a/docs/requirements.txt b/docs/requirements.txt index 818836b6..cf9f23bc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,3 @@ -# -# Note: keep this file in sync with conda_env_sphinx.txt ! -# - breathe myst-parser[linkify] pydata-sphinx-theme~=0.16 diff --git a/scripts/tantivy_smoke.sh b/scripts/tantivy_smoke.sh deleted file mode 100755 index 0e3adad7..00000000 --- a/scripts/tantivy_smoke.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -# Smoke-test script for the tantivy-fts migration. -# -# Purpose: one-shot regression of the lucene-fts + tantivy-fts tests inside the -# Dev Container. -# Rationale: the command line gets long and error-prone, so wrap it in a script -# maintained per stage. -# -# Usage: -# ./scripts/tantivy_smoke.sh # default: release, no sanitizer -# ./scripts/tantivy_smoke.sh --asan # ASAN build -# ./scripts/tantivy_smoke.sh --tsan # TSAN build -# ./scripts/tantivy_smoke.sh --configure # cmake configure only -# ./scripts/tantivy_smoke.sh --build # cmake build only (skip configure) -# ./scripts/tantivy_smoke.sh --tests-only # ctest only (assumes already built) -# -# Maintenance notes: -# - From Stage 1 on, update TEST_REGEX below whenever a new ctest target is added -# - Stage 11 adds the full --with-asan / --with-tsan path - -set -e - -CMAKE_BUILD_TYPE="Release" -USE_ASAN="OFF" -USE_TSAN="OFF" -BUILD_DIR_SUFFIX="" -DO_CONFIGURE=1 -DO_BUILD=1 -DO_TEST=1 - -# ctest regex: during per-stage acceptance, run only this subset rather than the -# full ctest (~531s, too slow). Contents = the lucene-fts baseline + the -# tantivy-fts targets added in the current and previous stages. Append a target -# here as each stage completes. Only Stage 11 should run the full ctest. -TEST_REGEX='paimon-lucene-index-test|paimon-global-index-test|paimon-tantivy-smoke-test|paimon-tantivy-ffi-test|paimon-tantivy-tokenizer-test|paimon-tantivy-writer-test|paimon-tantivy-reader-test|paimon-tantivy-filter-limit-test|paimon-tantivy-index-test|paimon-tantivy-lucene-coexist-test|paimon-tantivy-equivalence-test|paimon-tantivy-streaming-test|paimon-tantivy-java-compat-test' - -while [ $# -gt 0 ]; do - case "$1" in - --asan) USE_ASAN="ON"; CMAKE_BUILD_TYPE="Debug"; BUILD_DIR_SUFFIX="-asan" ;; - --tsan) USE_TSAN="ON"; CMAKE_BUILD_TYPE="Debug"; BUILD_DIR_SUFFIX="-tsan" ;; - --configure) DO_BUILD=0; DO_TEST=0 ;; - --build) DO_CONFIGURE=0; DO_TEST=0 ;; - --tests-only) DO_CONFIGURE=0; DO_BUILD=0 ;; - -h|--help) sed -n '2,20p' "$0"; exit 0 ;; - *) echo "Unknown option: $1"; exit 2 ;; - esac - shift -done - -REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -BUILD_DIR="${REPO_ROOT}/build${BUILD_DIR_SUFFIX}" - -cd "${REPO_ROOT}" - -if [ "${DO_CONFIGURE}" = "1" ]; then - echo "==> cmake configure (${BUILD_DIR})" - cmake -S . -B "${BUILD_DIR}" \ - -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ - -DPAIMON_BUILD_TESTS=ON \ - -DPAIMON_USE_ASAN="${USE_ASAN}" \ - -DPAIMON_USE_TSAN="${USE_TSAN}" \ - -DPAIMON_ENABLE_FSLIB=OFF \ - -DPAIMON_ENABLE_LUMINA=OFF \ - -DPAIMON_ENABLE_JINDO=OFF \ - -DPAIMON_ENABLE_LUCENE=ON \ - -DPAIMON_ENABLE_ORC=ON \ - -DPAIMON_ENABLE_ALIORC=ON \ - -DPAIMON_ENABLE_AVRO=ON \ - -G Ninja -fi - -if [ "${DO_BUILD}" = "1" ]; then - echo "==> cmake build" - cmake --build "${BUILD_DIR}" -j -fi - -if [ "${DO_TEST}" = "1" ]; then - echo "==> ctest (${TEST_REGEX})" - ctest --test-dir "${BUILD_DIR}" -R "${TEST_REGEX}" --output-on-failure -fi - -echo "==> tantivy_smoke.sh DONE" diff --git a/src/paimon/global_index/tantivy/CMakeLists.txt b/src/paimon/global_index/tantivy/CMakeLists.txt index 3873e250..8e1da23f 100644 --- a/src/paimon/global_index/tantivy/CMakeLists.txt +++ b/src/paimon/global_index/tantivy/CMakeLists.txt @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# tantivy-fulltext global index (Rust FFI). See docs/dev/tantivy_fts_migration_plan.md. -# Stage 4 grows the support lib with the C++ writer wrapper + writer test. +# Tantivy-fulltext global index support library and tests (Rust FFI). if(NOT PAIMON_ENABLE_TANTIVY) return() @@ -110,9 +109,9 @@ if(PAIMON_BUILD_TESTS) PRIVATE ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}) endif() - # Stage 4 — Writer test. Builds an Arrow batch, runs the writer through + # Writer test. Builds an Arrow batch, runs the writer through # GlobalIndexFileManager + LocalFileSystem, then validates the packed - # on-disk format. Reader round-trip lives in Stage 6. + # on-disk format. add_paimon_test(tantivy_writer_test SOURCES tantivy_writer_test.cpp @@ -131,7 +130,7 @@ if(PAIMON_BUILD_TESTS) target_compile_definitions(paimon-tantivy-writer-test PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") - # Stage 6 — Reader + 5 query types end-to-end. + # Reader + five query types end-to-end. add_paimon_test(tantivy_filter_limit_test SOURCES tantivy_filter_limit_test.cpp @@ -150,9 +149,10 @@ if(PAIMON_BUILD_TESTS) target_compile_definitions(paimon-tantivy-filter-limit-test PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") - # Java → C++ cross-read test. Fixture produced by paimon-java's - # `TantivyIndexFixtureGen` (see docs/dev/tantivy_java_cross_read_plan.md) - # and checked in under test/test_data/java_tantivy_fixtures/. + # Java → C++ cross-read test. The fixture is produced by paimon-java's + # `TantivyIndexFixtureGen` and checked in under + # test/test_data/java_tantivy_fixtures/. See that directory's README for + # regeneration instructions. add_paimon_test(tantivy_java_compat_test SOURCES tantivy_java_compat_test.cpp @@ -174,7 +174,7 @@ if(PAIMON_BUILD_TESTS) PAIMON_TANTIVY_CPP_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/test/test_data/cpp_tantivy_fixtures" ) - # K4 — V3 streaming reader + W1 streaming writer integration coverage: + # Streaming reader and writer integration coverage: # ParseArchiveHeader fuzz, concurrent query on shared reader, concurrent # reader create+drop lifecycle, streaming benchmark log. add_paimon_test(tantivy_streaming_test @@ -195,7 +195,7 @@ if(PAIMON_BUILD_TESTS) target_compile_definitions(paimon-tantivy-streaming-test PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") - # Stage 8 — TantivyGlobalIndex + factory + end-to-end integration test. + # TantivyGlobalIndex + factory + end-to-end integration test. # `--whole-archive` is required so the static REGISTER_PAIMON_FACTORY # symbols are not stripped out of the test binary. add_paimon_test(tantivy_index_test @@ -216,7 +216,7 @@ if(PAIMON_BUILD_TESTS) target_compile_definitions(paimon-tantivy-index-test PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") - # Stage 9 — Cross-implementation coexistence. Links against BOTH the + # Cross-implementation coexistence. Links against both the # lucene and tantivy support static libs to verify they resolve their # `REGISTER_PAIMON_FACTORY` registrations side by side and don't # collide on shared symbols. Only built when lucene-fts is enabled. @@ -242,7 +242,7 @@ if(PAIMON_BUILD_TESTS) target_compile_definitions(paimon-tantivy-lucene-coexist-test PRIVATE JIEBA_TEST_DICT_DIR="${JIEBA_DICT_DIR}") - # Stage 10 — Equivalence + benchmark. Same link line as the coexist + # Equivalence + benchmark. Same link line as the coexistence # test (needs both impls); benchmark output goes to stderr. add_paimon_test(tantivy_equivalence_test SOURCES diff --git a/test/test_data/java_tantivy_fixtures/README.md b/test/test_data/java_tantivy_fixtures/README.md index fa7fd4e1..09361154 100644 --- a/test/test_data/java_tantivy_fixtures/README.md +++ b/test/test_data/java_tantivy_fixtures/README.md @@ -1,6 +1,6 @@ # Java -> C++ tantivy cross-read fixtures -> Generated on **2026-04-23** for the J6 `paimon-tantivy-java-compat-test`. +> Generated on **2026-04-23** for `paimon-tantivy-java-compat-test`. ## Contents @@ -15,7 +15,7 @@ |---|---| | tantivy crate | **0.22.1** | | paimon-tantivy-jni | latest git sha at generation time (commit lives in the paimon repo) | -| schema | B1: `row_id` u64 stored+indexed+fast + `text` TEXT | +| schema | `row_id` u64 stored+indexed+fast + `text` TEXT | | archive byte format | Java-compatible, big-endian, no version header | Upgrading any component (especially the **tantivy version**) can make the segment @@ -44,8 +44,8 @@ xxd english_simple.archive | head -1 # force-merge, so multiple segments) ``` -## Related docs +## Related code -- `docs/dev/tantivy_java_cross_read_plan.md` — overall J6 plan -- `docs/dev/test_execute.md` — J6 execution log -- `docs/dev/tantivy_java_compat_plan.md` — overall paimon-cpp <-> paimon-java alignment plan +- [`tantivy_java_compat_test.cpp`](../../../src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp) +- [`tantivy CMakeLists.txt`](../../../src/paimon/global_index/tantivy/CMakeLists.txt) +- [`tantivy_ffi writer.rs`](../../../crates/tantivy_ffi/src/writer.rs) diff --git a/test/test_data/java_tantivy_fixtures/english_simple.golden.json b/test/test_data/java_tantivy_fixtures/english_simple.golden.json index 9776b720..a3c12060 100644 --- a/test/test_data/java_tantivy_fixtures/english_simple.golden.json +++ b/test/test_data/java_tantivy_fixtures/english_simple.golden.json @@ -1,5 +1,5 @@ { - "description": "10 English docs; row_ids 0..9; generated by TantivyIndexFixtureGen via TantivyFullTextGlobalIndexWriter production path; consumed by paimon-cpp V3 reader cross-read test (J6).", + "description": "10 English docs; row_ids 0..9; generated by TantivyIndexFixtureGen via TantivyFullTextGlobalIndexWriter production path; consumed by the paimon-cpp cross-read test.", "docs": [ {"row_id": 0, "text": "apple banana cherry"}, {"row_id": 1, "text": "apple durian"}, diff --git a/third_party/tantivy_ffi/rust-toolchain.toml b/third_party/tantivy_ffi/rust-toolchain.toml deleted file mode 100644 index 8a8c3664..00000000 --- a/third_party/tantivy_ffi/rust-toolchain.toml +++ /dev/null @@ -1,11 +0,0 @@ -# Pin the Rust toolchain used to build paimon_tantivy_ffi. Without this, -# Corrosion's FindRust.cmake invokes `rustup which rustc --toolchain ''` -# which fails on fresh CMake configure (no rust-toolchain → empty toolchain -# name → rustup rejects it). See docs/dev/execute.md Stage 11 for context. -# -# Only the `channel` is pinned — no extra components, because rustup in -# CI/containers may lack network access to fetch clippy/rustfmt, and build -# doesn't need them. -[toolchain] -channel = "stable" -profile = "minimal" From 1eb50bbdd5168345ece12420746402ed4cc1aea6 Mon Sep 17 00:00:00 2001 From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:31:23 +0800 Subject: [PATCH 117/138] chore: clean up attribution and test data whitespace --- LICENSE | 49 +------------------------------------------------ 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/LICENSE b/LICENSE index b454b24c..fc64292e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -436,54 +437,6 @@ License: BSD-3-Clause, see licenses/LICENSE-pytorch.txt -------------------------------------------------------------------------------- -This product includes code derived from PyTorch TH simd.h. - -* SIMD detection code in third_party/roaring_bitmap/roaring.cpp - -Copyright (c) 2016- Facebook, Inc (Adam Paszke) -Copyright (c) 2014- Facebook, Inc (Soumith Chintala) -Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) -Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) -Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) -Copyright (c) 2011-2013 NYU (Clement Farabet) -Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, -Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute -(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, -Samy Bengio, Johnny Mariethoz) - -All rights reserved. - -License: BSD-3-Clause - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories -America and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - --------------------------------------------------------------------------------- - This product includes code from cppjieba. * cppjieba patch in cmake_modules/jieba.diff From 1577b0ca441e23a55e6779bb34f20dc1a7eabca9 Mon Sep 17 00:00:00 2001 From: lxy <38709059+lxy-9602@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:47:35 +0800 Subject: [PATCH 118/138] fix(blob): support blob-only writes and reject nested blob fields --- src/paimon/common/data/blob_utils.cpp | 9 +- src/paimon/common/data/blob_utils.h | 3 +- src/paimon/common/data/blob_utils_test.cpp | 8 +- src/paimon/core/append/append_only_writer.cpp | 10 ++- .../core/append/append_only_writer_test.cpp | 47 ++++++++++ .../core/io/rolling_blob_file_writer.cpp | 77 ++++++++-------- src/paimon/core/io/rolling_blob_file_writer.h | 6 +- .../core/io/rolling_blob_file_writer_test.cpp | 14 +-- .../core/schema/arrow_schema_validator.cpp | 48 +++++++--- .../core/schema/arrow_schema_validator.h | 3 +- .../schema/arrow_schema_validator_test.cpp | 49 +++++++++- test/inte/blob_table_inte_test.cpp | 90 +++++++++++++++---- 12 files changed, 275 insertions(+), 89 deletions(-) diff --git a/src/paimon/common/data/blob_utils.cpp b/src/paimon/common/data/blob_utils.cpp index 8d6236b4..41df10b9 100644 --- a/src/paimon/common/data/blob_utils.cpp +++ b/src/paimon/common/data/blob_utils.cpp @@ -85,13 +85,12 @@ Result BlobUtils::SeparateBlobArray( return Status::Invalid( "SeparateBlobArray expects at least one non-inline blob field, but got none."); } - if (main_fields.empty()) { - return Status::Invalid("SeparateBlobArray expects at least one main field, but got none."); - } SeparatedStructArrays result; - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(result.main_array, - arrow::StructArray::Make(main_arrays, main_fields)); + if (!main_fields.empty()) { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(result.main_array, + arrow::StructArray::Make(main_arrays, main_fields)); + } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(result.blob_array, arrow::StructArray::Make(blob_arrays, blob_fields)); return result; diff --git a/src/paimon/common/data/blob_utils.h b/src/paimon/common/data/blob_utils.h index 1c244089..c13e24f9 100644 --- a/src/paimon/common/data/blob_utils.h +++ b/src/paimon/common/data/blob_utils.h @@ -54,7 +54,8 @@ class PAIMON_EXPORT BlobUtils { }; struct SeparatedStructArrays { - /// Non-blob fields (includes inline blob fields when inline_fields is provided) + /// Non-blob fields (includes inline blob fields when inline_fields is provided). + /// nullptr when all fields are stored in blob files. std::shared_ptr main_array; /// Blob fields that go to separate .blob files std::shared_ptr blob_array; diff --git a/src/paimon/common/data/blob_utils_test.cpp b/src/paimon/common/data/blob_utils_test.cpp index 3074c54d..d699d555 100644 --- a/src/paimon/common/data/blob_utils_test.cpp +++ b/src/paimon/common/data/blob_utils_test.cpp @@ -185,11 +185,13 @@ TEST_F(BlobUtilsTest, SeparateBlobArray) { BlobUtils::SeparateBlobArray(struct_array, /*inline_fields=*/{"f2_blob"}), "SeparateBlobArray expects at least one non-inline blob field, but got none."); - // All fields are blob with no inline -> no main field -> should return error + // All fields are blob with no inline -> no main array is needed auto all_blob_struct = arrow::StructArray::Make({blob_array_data}, {blob_field}).ValueOrDie(); auto all_blob_sa = std::dynamic_pointer_cast(all_blob_struct); - ASSERT_NOK_WITH_MSG(BlobUtils::SeparateBlobArray(all_blob_sa, /*inline_fields=*/{}), - "SeparateBlobArray expects at least one main field, but got none."); + ASSERT_OK_AND_ASSIGN(auto all_blob_separated, + BlobUtils::SeparateBlobArray(all_blob_sa, /*inline_fields=*/{})); + ASSERT_EQ(nullptr, all_blob_separated.main_array); + ASSERT_TRUE(all_blob_separated.blob_array->Equals(*all_blob_sa)); } TEST_F(BlobUtilsTest, SeparateBlobArrayWithPartialInline) { diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 833f540f..249b431d 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -240,10 +240,14 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri options_.GetBlobTargetFileSize(), single_blob_file_writer_factory); }; + WriterFactory main_writer_factory; + if (schemas.main_schema->num_fields() > 0) { + main_writer_factory = + GetDataFileWriterFactory(schemas.main_schema, schemas.main_schema->field_names()); + } return std::make_unique( - options_.GetTargetFileSize(/*has_primary_key=*/false), - GetDataFileWriterFactory(schemas.main_schema, schemas.main_schema->field_names()), - blob_schema, blob_writer_creator, arrow::struct_(write_schema_->fields()), inline_fields); + options_.GetTargetFileSize(/*has_primary_key=*/false), main_writer_factory, blob_schema, + blob_writer_creator, arrow::struct_(write_schema_->fields()), inline_fields); } Status AppendOnlyWriter::Sync() { diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp index 0363286e..8488efff 100644 --- a/src/paimon/core/append/append_only_writer_test.cpp +++ b/src/paimon/core/append/append_only_writer_test.cpp @@ -743,6 +743,53 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) { ASSERT_OK(writer->Close()); } +TEST_F(AppendOnlyWriterTest, TestWriteWithOnlyBlobField) { + auto options = + CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + auto path_factory = CreatePathFactory(dir->Str(), "orc", options); + + auto blob_field = BlobUtils::ToArrowField("blob", false); + auto schema = arrow::schema({blob_field}); + ASSERT_OK_AND_ASSIGN(auto writer, + CreateAppendOnlyWriter(options, /*schema_id=*/0, schema, + /*write_cols=*/std::vector{"blob"}, + /*max_sequence_number=*/-1, path_factory, + compact_manager_, memory_pool_)); + + arrow::LargeBinaryBuilder blob_builder; + ASSERT_TRUE(blob_builder.Append("a", 1).ok()); + ASSERT_TRUE(blob_builder.Append("bb", 2).ok()); + auto blob_array = blob_builder.Finish().ValueOrDie(); + + ASSERT_OK(writer->Write(CreateStructBatch(schema, {blob_array}))); + ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true)); + ASSERT_OK(writer->Close()); + + const auto& new_files = inc.GetNewFilesIncrement().NewFiles(); + ASSERT_EQ(new_files.size(), 1); + ASSERT_TRUE(BlobUtils::IsBlobFile(new_files[0]->file_name)); + ASSERT_EQ(new_files[0]->row_count, 2); + ASSERT_TRUE(new_files[0]->write_cols.has_value()); + ASSERT_EQ(new_files[0]->write_cols.value(), std::vector({"blob"})); + std::string blob_file_path = path_factory->ToPath(new_files[0]->file_name); + ASSERT_TRUE(options.GetFileSystem()->Exists(blob_file_path).value()); + + auto blob_reader = OpenFormatReader(blob_file_path, "blob"); + ::ArrowSchema c_blob_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_blob_schema).ok()); + ASSERT_OK(blob_reader->SetReadSchema(&c_blob_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto actual_array, ReadResultCollector::CollectResult(blob_reader.get())); + auto expected_struct_array = + arrow::StructArray::Make({blob_array}, {blob_field->name()}).ValueOrDie(); + auto expected_array = std::make_shared(expected_struct_array); + ASSERT_TRUE(expected_array->Equals(actual_array)) << "Expected:\n" + << expected_array->ToString() << "\nActual:\n" + << actual_array->ToString(); +} + TEST_F(AppendOnlyWriterTest, TestWriteWithMultipleBlobFields) { auto options = CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}}); diff --git a/src/paimon/core/io/rolling_blob_file_writer.cpp b/src/paimon/core/io/rolling_blob_file_writer.cpp index 06eb1199..4bc7c5c0 100644 --- a/src/paimon/core/io/rolling_blob_file_writer.cpp +++ b/src/paimon/core/io/rolling_blob_file_writer.cpp @@ -18,6 +18,7 @@ #include "paimon/core/io/rolling_blob_file_writer.h" +#include #include #include #include @@ -29,7 +30,6 @@ #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" #include "fmt/format.h" -#include "fmt/ranges.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" @@ -57,7 +57,7 @@ RollingBlobFileWriter::RollingBlobFileWriter( Status RollingBlobFileWriter::Write(::ArrowArray* record) { ScopeGuard guard([this]() -> void { this->Abort(); }); // Open the current writer if write the first record or roll over happen before. - if (PAIMON_UNLIKELY(current_writer_ == nullptr)) { + if (writer_factory_ != nullptr && PAIMON_UNLIKELY(current_writer_ == nullptr)) { PAIMON_RETURN_NOT_OK(OpenCurrentWriter()); } if (PAIMON_UNLIKELY(blob_writer_ == nullptr)) { @@ -71,12 +71,14 @@ Status RollingBlobFileWriter::Write(::ArrowArray* record) { PAIMON_ASSIGN_OR_RAISE(BlobUtils::SeparatedStructArrays separated_arrays, BlobUtils::SeparateBlobArray(struct_array, inline_fields_)); // Write main (non-blob) data - ::ArrowArray c_main_array; - PAIMON_RETURN_NOT_OK_FROM_ARROW( - arrow::ExportArray(*separated_arrays.main_array, &c_main_array)); - ScopeGuard array_lifecycle_guard( - [&c_main_array]() -> void { ArrowArrayRelease(&c_main_array); }); - PAIMON_RETURN_NOT_OK(current_writer_->Write(&c_main_array)); + if (current_writer_ != nullptr) { + ::ArrowArray c_main_array; + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*separated_arrays.main_array, &c_main_array)); + ScopeGuard array_lifecycle_guard( + [&c_main_array]() -> void { ArrowArrayRelease(&c_main_array); }); + PAIMON_RETURN_NOT_OK(current_writer_->Write(&c_main_array)); + } // Write blob data via MultipleBlobFileWriter (each blob field independently) ::ArrowArray c_blob_array; @@ -86,28 +88,31 @@ Status RollingBlobFileWriter::Write(::ArrowArray* record) { PAIMON_RETURN_NOT_OK(blob_writer_->Write(&c_blob_array)); record_count_ += record_count; - PAIMON_ASSIGN_OR_RAISE(bool need_rolling_file, NeedRollingFile()); - if (need_rolling_file) { - PAIMON_RETURN_NOT_OK(CloseCurrentWriter()); + if (current_writer_ != nullptr) { + PAIMON_ASSIGN_OR_RAISE(bool need_rolling_file, NeedRollingFile()); + if (need_rolling_file) { + PAIMON_RETURN_NOT_OK(CloseCurrentWriter()); + } } guard.Release(); return Status::OK(); } Status RollingBlobFileWriter::CloseCurrentWriter() { - if (current_writer_ == nullptr) { - return Status::OK(); - } if (blob_writer_ == nullptr) { return Status::OK(); } - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr main_data_file_meta, CloseMainWriter()); + std::shared_ptr main_data_file_meta; + if (current_writer_ != nullptr) { + PAIMON_ASSIGN_OR_RAISE(main_data_file_meta, CloseMainWriter()); + } PAIMON_ASSIGN_OR_RAISE(std::vector> blob_metas, CloseBlobWriter()); - PAIMON_RETURN_NOT_OK( - ValidateFileConsistency(main_data_file_meta, blob_metas, blob_schema_->num_fields())); - results_.push_back(main_data_file_meta); + if (main_data_file_meta != nullptr) { + PAIMON_RETURN_NOT_OK(ValidateFileConsistency(main_data_file_meta, blob_metas)); + results_.push_back(main_data_file_meta); + } results_.insert(results_.end(), blob_metas.begin(), blob_metas.end()); current_writer_.reset(); @@ -139,29 +144,25 @@ Result>> RollingBlobFileWriter::CloseB Status RollingBlobFileWriter::ValidateFileConsistency( const std::shared_ptr& main_data_file_meta, - const std::vector>& blob_tagged_metas, int32_t blob_field_count) { - if (blob_tagged_metas.empty()) { - return Status::OK(); - } - // With multiple blob fields, each blob field produces its own set of files. - // total_blob_row_count should be exactly main_row_count * blob_field_count. - int64_t main_row_count = main_data_file_meta->row_count; - int64_t expected_blob_row_count = main_row_count * blob_field_count; - int64_t total_blob_row_count = 0; + const std::vector>& blob_tagged_metas) { + std::map blob_field_row_counts; for (const auto& blob_tagged_meta : blob_tagged_metas) { - total_blob_row_count += blob_tagged_meta->row_count; + if (!blob_tagged_meta->write_cols || blob_tagged_meta->write_cols->empty()) { + return Status::Invalid( + fmt::format("This is a bug: Blob file {} must contain a write column.", + blob_tagged_meta->file_name)); + } + blob_field_row_counts[blob_tagged_meta->write_cols->at(0)] += blob_tagged_meta->row_count; } - if (total_blob_row_count != expected_blob_row_count) { - std::vector blob_file_names; - for (const auto& blob_tagged_meta : blob_tagged_metas) { - blob_file_names.push_back(blob_tagged_meta->file_name); + + int64_t main_row_count = main_data_file_meta->row_count; + for (const auto& [field_name, row_count] : blob_field_row_counts) { + if (row_count != main_row_count) { + return Status::Invalid(fmt::format( + "This is a bug: The row count of main file and blob file does not match. Main " + "file: {} (row count: {}), blob field name: {} (row count: {})", + main_data_file_meta->file_name, main_row_count, field_name, row_count)); } - return Status::Invalid(fmt::format( - "This is a bug: The row count of main file and blob files does not match. " - "Main file: {} (row count: {}), blob field count: {}, " - "expected blob row count: {}, blob files: {} (actual total row count: {})", - main_data_file_meta->file_name, main_row_count, blob_field_count, - expected_blob_row_count, fmt::join(blob_file_names, ", "), total_blob_row_count)); } return Status::OK(); } diff --git a/src/paimon/core/io/rolling_blob_file_writer.h b/src/paimon/core/io/rolling_blob_file_writer.h index 70094581..d091c17d 100644 --- a/src/paimon/core/io/rolling_blob_file_writer.h +++ b/src/paimon/core/io/rolling_blob_file_writer.h @@ -40,7 +40,8 @@ namespace paimon { /// between them. /// /// Multiple blob fields are supported. Each blob field is written to its own set of blob files -/// independently via MultipleBlobFileWriter. +/// independently via MultipleBlobFileWriter. For blob-only writes, the main writer factory may be +/// nullptr and only blob files are produced. /// /// 
 /// For example,
@@ -78,8 +79,7 @@ class RollingBlobFileWriter
  private:
     static Status ValidateFileConsistency(
         const std::shared_ptr& main_data_file_meta,
-        const std::vector>& blob_tagged_metas,
-        int32_t blob_field_count);
+        const std::vector>& blob_tagged_metas);
 
     Status CloseCurrentWriter();
 
diff --git a/src/paimon/core/io/rolling_blob_file_writer_test.cpp b/src/paimon/core/io/rolling_blob_file_writer_test.cpp
index 9e2a4be5..654c9126 100644
--- a/src/paimon/core/io/rolling_blob_file_writer_test.cpp
+++ b/src/paimon/core/io/rolling_blob_file_writer_test.cpp
@@ -84,11 +84,15 @@ TEST_F(RollingBlobFileWriterTest, ValidateFileConsistency) {
         /*delete_row_count=*/0, /*embedded_index=*/nullptr, FileSource::Append(),
         /*value_stats_cols=*/std::nullopt, /*external_path=*/std::nullopt, /*first_row_id=*/3,
         /*write_cols=*/std::vector({"blob"}));
-    ASSERT_OK(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2, file_meta3},
-                                                             /*blob_field_count=*/1));
-    ASSERT_NOK_WITH_MSG(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2},
-                                                                       /*blob_field_count=*/2),
-                        "This is a bug: The row count of main file and blob files does not match.");
+    ASSERT_OK(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2, file_meta3}));
+    ASSERT_NOK_WITH_MSG(RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2}),
+                        "This is a bug: The row count of main file and blob file does not match.");
+
+    file_meta2->write_cols = std::vector({"blob1"});
+    file_meta3->write_cols = std::vector({"blob2"});
+    ASSERT_NOK_WITH_MSG(
+        RollingBlobFileWriter::ValidateFileConsistency(file_meta1, {file_meta2, file_meta3}),
+        "This is a bug: The row count of main file and blob file does not match.");
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp
index 60869d56..e01afe01 100644
--- a/src/paimon/core/schema/arrow_schema_validator.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator.cpp
@@ -58,10 +58,19 @@ Status ArrowSchemaValidator::ValidateSchema(const arrow::Schema& schema) {
 
 Status ArrowSchemaValidator::ValidateSchemaWithFieldId(const arrow::Schema& schema) {
     PAIMON_RETURN_NOT_OK(ValidateSchema(schema));
-    auto struct_type = arrow::struct_(schema.fields());
     std::set field_id_set;
-    PAIMON_RETURN_NOT_OK(
-        ValidateDataTypeWithFieldId(struct_type, /*key_value_metadata=*/nullptr, &field_id_set));
+    for (const auto& field : schema.fields()) {
+        PAIMON_ASSIGN_OR_RAISE(DataField data_field,
+                               DataField::ConvertArrowFieldToDataField(field));
+        auto iter = field_id_set.find(data_field.Id());
+        if (iter != field_id_set.end()) {
+            return Status::Invalid(
+                fmt::format("field id must be unique, duplicate field id {}", data_field.Id()));
+        }
+        field_id_set.insert(data_field.Id());
+        PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(field->type(), field->metadata(),
+                                                         /*allow_blob=*/true, &field_id_set));
+    }
     return Status::OK();
 }
 
@@ -96,7 +105,7 @@ Status ArrowSchemaValidator::ValidateNoWhitespaceOnlyFields(const arrow::FieldVe
 
 Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
     const std::shared_ptr& type,
-    const std::shared_ptr& key_value_metadata,
+    const std::shared_ptr& key_value_metadata, bool allow_blob,
     std::set* field_id_set) {
     const auto kind = type->id();
     switch (kind) {
@@ -117,7 +126,7 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
             const auto& value_field =
                 arrow::internal::checked_cast(type.get())->value_field();
             PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(
-                value_field->type(), value_field->metadata(), field_id_set));
+                value_field->type(), value_field->metadata(), /*allow_blob=*/false, field_id_set));
             break;
         }
         case arrow::Type::type::STRUCT: {
@@ -138,7 +147,7 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
                 }
                 field_id_set->insert(data_field.Id());
                 PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(
-                    sub_field->type(), sub_field->metadata(), field_id_set));
+                    sub_field->type(), sub_field->metadata(), /*allow_blob=*/false, field_id_set));
             }
             break;
         }
@@ -147,14 +156,17 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
                 arrow::internal::checked_cast(type.get())->key_field();
             const auto& item_field =
                 arrow::internal::checked_cast(type.get())->item_field();
-            PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(key_field->type(),
-                                                             key_field->metadata(), field_id_set));
-            PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(item_field->type(),
-                                                             item_field->metadata(), field_id_set));
+            PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(
+                key_field->type(), key_field->metadata(), /*allow_blob=*/false, field_id_set));
+            PAIMON_RETURN_NOT_OK(ValidateDataTypeWithFieldId(
+                item_field->type(), item_field->metadata(), /*allow_blob=*/false, field_id_set));
             break;
         }
         case arrow::Type::type::LARGE_BINARY: {
             if (BlobUtils::IsBlobMetadata(key_value_metadata)) {
+                if (!allow_blob) {
+                    return Status::Invalid("Blob field must be a top-level field.");
+                }
                 break;
             }
             [[fallthrough]];
@@ -167,6 +179,11 @@ Status ArrowSchemaValidator::ValidateDataTypeWithFieldId(
 }
 
 Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& field) {
+    return ValidateField(field, /*allow_blob=*/true);
+}
+
+Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& field,
+                                           bool allow_blob) {
     const auto kind = field->type()->id();
     switch (kind) {
         case arrow::Type::type::BOOL:
@@ -188,7 +205,7 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr&
             const auto& value_field =
                 arrow::internal::checked_cast(*field->type())
                     .value_field();
-            PAIMON_RETURN_NOT_OK(ValidateField(value_field));
+            PAIMON_RETURN_NOT_OK(ValidateField(value_field, /*allow_blob=*/false));
             break;
         }
         case arrow::Type::type::STRUCT: {
@@ -205,7 +222,7 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr&
             arrow::FieldVector arrow_fields =
                 arrow::internal::checked_cast(*field->type()).fields();
             for (const auto& sub_field : arrow_fields) {
-                PAIMON_RETURN_NOT_OK(ValidateField(sub_field));
+                PAIMON_RETURN_NOT_OK(ValidateField(sub_field, /*allow_blob=*/false));
             }
             break;
         }
@@ -214,12 +231,15 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr&
                 arrow::internal::checked_cast(*field->type()).key_field();
             const auto& item_field =
                 arrow::internal::checked_cast(*field->type()).item_field();
-            PAIMON_RETURN_NOT_OK(ValidateField(key_field));
-            PAIMON_RETURN_NOT_OK(ValidateField(item_field));
+            PAIMON_RETURN_NOT_OK(ValidateField(key_field, /*allow_blob=*/false));
+            PAIMON_RETURN_NOT_OK(ValidateField(item_field, /*allow_blob=*/false));
             break;
         }
         case arrow::Type::type::LARGE_BINARY: {
             if (BlobUtils::IsBlobField(field)) {
+                if (!allow_blob) {
+                    return Status::Invalid("Blob field must be a top-level field.");
+                }
                 break;
             }
             [[fallthrough]];
diff --git a/src/paimon/core/schema/arrow_schema_validator.h b/src/paimon/core/schema/arrow_schema_validator.h
index 61184458..f7abf6d8 100644
--- a/src/paimon/core/schema/arrow_schema_validator.h
+++ b/src/paimon/core/schema/arrow_schema_validator.h
@@ -58,7 +58,8 @@ class PAIMON_EXPORT ArrowSchemaValidator {
  private:
     static Status ValidateDataTypeWithFieldId(
         const std::shared_ptr& type,
-        const std::shared_ptr& key_value_metadata,
+        const std::shared_ptr& key_value_metadata, bool allow_blob,
         std::set* field_id_set);
+    static Status ValidateField(const std::shared_ptr& field, bool allow_blob);
 };
 }  // namespace paimon
diff --git a/src/paimon/core/schema/arrow_schema_validator_test.cpp b/src/paimon/core/schema/arrow_schema_validator_test.cpp
index 4fed03da..0363dff6 100644
--- a/src/paimon/core/schema/arrow_schema_validator_test.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator_test.cpp
@@ -24,6 +24,7 @@
 
 #include "arrow/type.h"
 #include "gtest/gtest.h"
+#include "paimon/common/data/blob_utils.h"
 #include "paimon/common/data/variant/variant_access_utils.h"
 #include "paimon/common/data/variant/variant_defs.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
@@ -154,6 +155,51 @@ TEST(ArrowSchemaValidatorTest, TestInvalidDataType) {
     }
 }
 
+TEST(ArrowSchemaValidatorTest, TestBlobFieldMustBeTopLevel) {
+    {
+        auto arrow_schema =
+            arrow::schema(arrow::FieldVector({BlobUtils::ToArrowField("blob", true)}));
+        ASSERT_OK(ArrowSchemaValidator::ValidateSchema(*arrow_schema));
+    }
+    {
+        std::vector fields = {DataField(0, BlobUtils::ToArrowField("blob", true))};
+        auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields);
+        ASSERT_OK(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema));
+    }
+    {
+        auto nested_blob_field =
+            arrow::field("nested", arrow::struct_({BlobUtils::ToArrowField("blob", true)}));
+        auto arrow_schema = arrow::schema(arrow::FieldVector({nested_blob_field}));
+        ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
+                            "Blob field must be a top-level field.");
+    }
+    {
+        auto array_blob_field =
+            arrow::field("array_blob", arrow::list(BlobUtils::ToArrowField("item", true)));
+        auto arrow_schema = arrow::schema(arrow::FieldVector({array_blob_field}));
+        ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
+                            "Blob field must be a top-level field.");
+    }
+    {
+        auto map_blob_field = arrow::field(
+            "map_blob",
+            arrow::map(arrow::utf8(), arrow::struct_({BlobUtils::ToArrowField("blob", true)})));
+        auto arrow_schema = arrow::schema(arrow::FieldVector({map_blob_field}));
+        ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchema(*arrow_schema),
+                            "Blob field must be a top-level field.");
+    }
+    {
+        std::vector nested_fields = {
+            DataField(1, BlobUtils::ToArrowField("blob", true))};
+        std::vector fields = {DataField(
+            0,
+            arrow::field("nested", DataField::ConvertDataFieldsToArrowStructType(nested_fields)))};
+        auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(fields);
+        ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateSchemaWithFieldId(*arrow_schema),
+                            "Blob field must be a top-level field.");
+    }
+}
+
 TEST(ArrowSchemaValidatorTest, ValidateDataTypeWithFieldId) {
     {
         std::vector fields = {DataField(3, arrow::field("f3", arrow::float64())),
@@ -271,7 +317,8 @@ TEST(ArrowSchemaValidatorTest, ValidateDataTypeWithFieldId) {
         auto struct_type = DataField::ConvertDataFieldsToArrowStructType(fields);
         std::set field_id_set;
         ASSERT_NOK_WITH_MSG(ArrowSchemaValidator::ValidateDataTypeWithFieldId(
-                                struct_type, /*key_value_metadata=*/nullptr, &field_id_set),
+                                struct_type, /*key_value_metadata=*/nullptr,
+                                /*allow_blob=*/true, &field_id_set),
                             "Unknown or unsupported arrow type: large_string");
     }
 }
diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp
index 41887b5a..b4c9c281 100644
--- a/test/inte/blob_table_inte_test.cpp
+++ b/test/inte/blob_table_inte_test.cpp
@@ -996,30 +996,90 @@ TEST_P(BlobTableInteTest, TestMultipleAppends) {
     }
 }
 
-TEST_P(BlobTableInteTest, TestOnlySomeColumns) {
-    CreateTable();
+TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
     std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
-    auto schema = arrow::schema(fields_);
+    auto schema = arrow::schema(fields);
 
-    // write field: f0
-    std::vector write_cols0 = {"f0"};
+    // Initial full-row write assigns row ids 0 and 1.
     auto src_array0 = std::dynamic_pointer_cast(
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[0]}), R"([
-        [1]
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [1, "a", "old_blob_0"],
+        [2, "b", "old_blob_1"]
     ])")
             .ValueOrDie());
-    ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, {}, write_cols0, {src_array0}));
-    ASSERT_OK(Commit(table_path, commit_msgs));
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0,
+                         WriteArray(table_path, {}, schema->field_names(), {src_array0}));
+    ASSERT_OK(Commit(table_path, commit_msgs0));
 
-    // write field: f1
-    std::vector write_cols1 = {"f1"};
+    // Update only b0 and align it with the existing row ids.
+    std::vector blob_write_cols = {"b0"};
     auto src_array1 = std::dynamic_pointer_cast(
-        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([
-        ["a"]
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[2]}), R"([
+        ["new_blob_0"],
+        ["new_blob_1"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1,
+                         WriteArray(table_path, {}, blob_write_cols, {src_array1}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    auto expected_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [1, "a", "new_blob_0"],
+        [2, "b", "new_blob_1"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
+
+    auto expected_with_row_id = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}),
+            R"([
+        [1, "a", "new_blob_0", 0],
+        [2, "b", "new_blob_1", 1]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id));
+}
+
+TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+
+    auto blob_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields[2]}), R"([
+        ["blob_0"]
     ])")
             .ValueOrDie());
-    ASSERT_NOK_WITH_MSG(WriteArray(table_path, {}, write_cols1, {src_array1}),
-                        "SeparateBlobArray expects at least one main field, but got none.");
+    std::vector blob_write_cols = {"b0"};
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs,
+                         WriteArray(table_path, {}, blob_write_cols, {blob_array}));
+
+    ASSERT_EQ(commit_msgs.size(), 1);
+    auto commit_msg = std::dynamic_pointer_cast(commit_msgs[0]);
+    ASSERT_TRUE(commit_msg);
+    ASSERT_EQ(commit_msg->data_increment_.new_files_.size(), 1);
+    const auto& blob_file = commit_msg->data_increment_.new_files_[0];
+    ASSERT_TRUE(BlobUtils::IsBlobFile(blob_file->file_name));
+    ASSERT_FALSE(blob_file->first_row_id.has_value());
+
+    ASSERT_NOK_WITH_MSG(Commit(table_path, commit_msgs), "blobStart 0 should be less than start 0");
 }
 
 TEST_P(BlobTableInteTest, TestMultipleAppendsDifferentFirstRowIds) {

From 1df1baac1eb5f749f12f63fd4464652b340953b5 Mon Sep 17 00:00:00 2001
From: XiaoHongbo <1346652787@qq.com>
Date: Mon, 27 Jul 2026 21:05:28 +0800
Subject: [PATCH 119/138] fix: nested list schema evolution

---
 .../reader/data_evolution_file_reader.cpp     |   4 +
 src/paimon/core/io/field_mapping_reader.cpp   |  25 +-
 .../core/io/field_mapping_reader_test.cpp     |  85 +++++
 src/paimon/core/utils/field_mapping.cpp       |   9 +-
 .../core/utils/nested_projection_utils.cpp    | 312 ++++++++++++++++--
 .../core/utils/nested_projection_utils.h      |   7 +
 .../utils/nested_projection_utils_test.cpp    | 173 ++++++++++
 test/inte/write_and_read_inte_test.cpp        |  96 +++++-
 8 files changed, 673 insertions(+), 38 deletions(-)

diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp
index 32eb5f96..0bba204b 100644
--- a/src/paimon/common/reader/data_evolution_file_reader.cpp
+++ b/src/paimon/common/reader/data_evolution_file_reader.cpp
@@ -19,6 +19,8 @@
 
 #include "paimon/common/reader/data_evolution_file_reader.h"
 
+#include "arrow/array/array_nested.h"
+#include "arrow/array/util.h"
 #include "arrow/c/abi.h"
 #include "arrow/c/bridge.h"
 #include "fmt/format.h"
@@ -28,6 +30,7 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 
 namespace paimon {
+
 Result> DataEvolutionFileReader::Create(
     std::vector>&& readers,
     const std::shared_ptr& read_schema, int32_t read_batch_size,
@@ -85,6 +88,7 @@ Result DataEvolutionFileReader::NextBatchWithB
         }
         const auto& sub_array = array_for_each_reader[reader_offsets_[i]];
         assert(sub_array->num_fields() > field_offsets_[i]);
+        // Each file is already aligned to its read schema by its FieldMappingReader.
         target_sub_array_vec.push_back(sub_array->field(field_offsets_[i]));
     }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
diff --git a/src/paimon/core/io/field_mapping_reader.cpp b/src/paimon/core/io/field_mapping_reader.cpp
index 738a99a0..3bd04b49 100644
--- a/src/paimon/core/io/field_mapping_reader.cpp
+++ b/src/paimon/core/io/field_mapping_reader.cpp
@@ -151,6 +151,11 @@ Result> FieldMappingReader::Create(
         if (mapping_reader->non_partition_info_.cast_executors[i] != nullptr) {
             mapping_reader->need_casting_ = true;
         }
+        // A differing nested type needs the AlignArrayToReadType reshape below.
+        if (!mapping_reader->non_partition_info_.non_partition_data_schema[i].Type()->Equals(
+                *mapping_reader->non_partition_info_.non_partition_read_schema[i].Type())) {
+            mapping_reader->need_casting_ = true;
+        }
         // Field name change (RENAME COLUMN) also requires mapping: data schema
         // carries the file's physical name while read schema carries the
         // post-rename logical name. If we skipped mapping, the inner reader's
@@ -203,6 +208,7 @@ Result> FieldMappingReader::CastNonPartitionArrayI
     casted_array.reserve(field_count);
     casted_field_names.reserve(field_count);
     for (int32_t i = 0; i < field_count; i++) {
+        std::shared_ptr column;
         if (non_partition_info_.cast_executors[i] != nullptr) {
             auto single_column_array = struct_array->field(i);
             // if src array is dict, cast to string first
@@ -215,18 +221,27 @@ Result> FieldMappingReader::CastNonPartitionArrayI
                                        arrow::compute::CastOptions::Safe(), arrow_pool_.get()));
             }
             PAIMON_ASSIGN_OR_RAISE(
-                std::shared_ptr casted,
+                column,
                 non_partition_info_.cast_executors[i]->Cast(
                     single_column_array, non_partition_info_.non_partition_read_schema[i].Type(),
                     arrow_pool_.get()));
-            casted_array.push_back(casted);
-            casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name());
         } else {
             // read and data type may both be string type, but after adapter transform, type may be
             // dictionary, need reconstruct struct type
-            casted_array.push_back(struct_array->field(i));
-            casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name());
+            column = struct_array->field(i);
+        }
+        // Null-fill nested fields added by schema evolution. Only when the data and
+        // read types differ -- the reader may hand back a dictionary-encoded array
+        // for an unchanged type, which is not a reshape target.
+        if (!non_partition_info_.non_partition_data_schema[i].Type()->Equals(
+                *non_partition_info_.non_partition_read_schema[i].Type())) {
+            PAIMON_ASSIGN_OR_RAISE(
+                column, NestedProjectionUtils::AlignArrayToReadType(
+                            column, non_partition_info_.non_partition_read_schema[i].Type(),
+                            arrow_pool_.get()));
         }
+        casted_array.push_back(column);
+        casted_field_names.push_back(non_partition_info_.non_partition_data_schema[i].Name());
     }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array,
                                       arrow::StructArray::Make(casted_array, casted_field_names));
diff --git a/src/paimon/core/io/field_mapping_reader_test.cpp b/src/paimon/core/io/field_mapping_reader_test.cpp
index 15fc669d..233a8587 100644
--- a/src/paimon/core/io/field_mapping_reader_test.cpp
+++ b/src/paimon/core/io/field_mapping_reader_test.cpp
@@ -450,6 +450,91 @@ TEST_F(FieldMappingReaderTest, TestDictionaryTypeWithSchemaEvolution) {
                 partition, expected_array);
 }
 
+TEST_F(FieldMappingReaderTest, TestSchemaEvolutionAddedFieldInsideList) {
+    // A field `c`(id=12) was added inside the list's struct after the file was
+    // written. Reading the old file with the new schema must null-fill `c`.
+    auto id_field = [](const std::string& name, const std::shared_ptr& type,
+                       int32_t id) {
+        return DataField::ConvertDataFieldToArrowField(DataField(id, arrow::field(name, type)));
+    };
+    auto data_struct =
+        arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::utf8(), 11)});
+    auto read_struct =
+        arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::utf8(), 11),
+                        id_field("c", arrow::int32(), 12)});
+    std::vector data_fields = {
+        DataField(100, arrow::field("items", arrow::list(arrow::field("item", data_struct))))};
+    std::vector read_fields = {
+        DataField(100, arrow::field("items", arrow::list(arrow::field("item", read_struct))))};
+    auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields);
+    auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields);
+
+    auto data_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(data_schema->fields()), R"([
+        [[[1, "x"], [2, "y"]]],
+        [[[3, "z"]]]
+    ])")
+            .ValueOrDie());
+
+    ASSERT_OK_AND_ASSIGN(auto mapping_builder,
+                         FieldMappingBuilder::Create(read_schema, /*partition_keys=*/{},
+                                                     /*predicate=*/nullptr));
+    ASSERT_OK_AND_ASSIGN(auto mapping, mapping_builder->CreateFieldMapping(data_fields));
+    auto mock = std::make_unique(
+        data_array, arrow::struct_(data_schema->fields()), /*read_batch_size=*/8);
+    ASSERT_OK_AND_ASSIGN(auto reader, FieldMappingReader::Create(
+                                          read_schema->num_fields(), std::move(mock),
+                                          BinaryRow::EmptyRow(), std::move(mapping),
+                                          /*skip_map_selected_keys_filter_field_ids=*/{}, pool_));
+    ASSERT_OK_AND_ASSIGN(auto result_array, ReadResultCollector::CollectResult(reader.get()));
+
+    auto expect_array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), R"([
+        [[[1, "x", null], [2, "y", null]]],
+        [[[3, "z", null]]]
+    ])")
+            .ValueOrDie();
+    auto expected_chunk = std::make_shared(arrow::ArrayVector({expect_array}));
+    ASSERT_TRUE(result_array->type()->Equals(expected_chunk->type()))
+        << result_array->type()->ToString() << " vs " << expected_chunk->type()->ToString();
+    ASSERT_TRUE(result_array->Equals(expected_chunk))
+        << result_array->ToString() << " vs " << expected_chunk->ToString();
+}
+
+TEST_F(FieldMappingReaderTest, TestSchemaEvolutionAddedFieldInsideListOrc) {
+    // ORC round-trip: added field inside a list's struct is null-filled.
+    auto id_field = [](const std::string& name, const std::shared_ptr& type,
+                       int32_t id) {
+        return DataField::ConvertDataFieldToArrowField(DataField(id, arrow::field(name, type)));
+    };
+    auto data_struct =
+        arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::int32(), 11)});
+    auto read_struct =
+        arrow::struct_({id_field("a", arrow::int32(), 10), id_field("b", arrow::int32(), 11),
+                        id_field("c", arrow::int32(), 12)});
+    std::vector data_fields = {
+        DataField(100, arrow::field("items", arrow::list(arrow::field("item", data_struct))))};
+    std::vector read_fields = {
+        DataField(100, arrow::field("items", arrow::list(arrow::field("item", read_struct))))};
+    auto data_schema = DataField::ConvertDataFieldsToArrowSchema(data_fields);
+    auto read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields);
+
+    auto data_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(data_schema->fields()), R"([
+        [[[1, 2], [3, 4]]],
+        [[[5, 6]]]
+    ])")
+            .ValueOrDie());
+    auto expect_array =
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(read_schema->fields()), R"([
+        [[[1, 2, null], [3, 4, null]]],
+        [[[5, 6, null]]]
+    ])")
+            .ValueOrDie();
+    CheckResult(data_schema, data_array, read_schema, /*predicate=*/nullptr, /*partition_keys=*/{},
+                BinaryRow::EmptyRow(), expect_array);
+}
+
 TEST_F(FieldMappingReaderTest, TestSchemaEvolutionWithModifyType) {
     std::vector data_fields = {DataField(0, arrow::field("f0", arrow::utf8())),
                                           DataField(1, arrow::field("f1", arrow::float32())),
diff --git a/src/paimon/core/utils/field_mapping.cpp b/src/paimon/core/utils/field_mapping.cpp
index 34934a95..df3791e3 100644
--- a/src/paimon/core/utils/field_mapping.cpp
+++ b/src/paimon/core/utils/field_mapping.cpp
@@ -175,10 +175,11 @@ Result>> FieldMappingBuilder::CreateDa
                                FieldTypeUtils::ConvertToFieldType(data_fields[i].Type()->id()));
 
         if (!read_fields[i].Type()->Equals(data_fields[i].Type())) {
-            if (read_type == FieldType::STRUCT) {
-                // STRUCT may still differ by nested pruning shape. No cast is
-                // needed — type pruning is handled by PruneDataType during
-                // field mapping construction.
+            auto read_type_id = read_fields[i].Type()->id();
+            if (read_type_id == arrow::Type::STRUCT || read_type_id == arrow::Type::LIST ||
+                read_type_id == arrow::Type::MAP) {
+                // Nested type differs by pruning/evolution; the reader's reshape
+                // handles it, no scalar cast.
                 cast_executors.push_back(nullptr);
                 continue;
             }
diff --git a/src/paimon/core/utils/nested_projection_utils.cpp b/src/paimon/core/utils/nested_projection_utils.cpp
index bd64ee14..97c08e9a 100644
--- a/src/paimon/core/utils/nested_projection_utils.cpp
+++ b/src/paimon/core/utils/nested_projection_utils.cpp
@@ -29,11 +29,14 @@
 #include "arrow/array/array_primitive.h"
 #include "arrow/array/builder_primitive.h"
 #include "arrow/array/concatenate.h"
+#include "arrow/array/util.h"
+#include "arrow/compute/cast.h"
 #include "arrow/type.h"
 #include "fmt/format.h"
 #include "paimon/common/data/variant/variant_access_utils.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/utils/string_utils.h"
+#include "paimon/core/casting/casting_utils.h"
 #include "paimon/status.h"
 
 namespace paimon {
@@ -153,12 +156,47 @@ Result NestedProjectionUtils::HasNestedSubfieldProjectionType(
 
 namespace {
 
+// Structural equality that also compares paimon field IDs on STRUCT children, so a
+// drop+add of a same-name/same-type field (new ID) is not treated as a no-op.
+Result EqualWithFieldIds(const std::shared_ptr& a,
+                               const std::shared_ptr& b) {
+    if (a->id() != b->id() || a->num_fields() != b->num_fields()) {
+        return false;
+    }
+    if (a->num_fields() == 0) {
+        return a->Equals(*b);
+    }
+    for (int32_t i = 0; i < a->num_fields(); ++i) {
+        const auto& fa = a->field(i);
+        const auto& fb = b->field(i);
+        if (fa->nullable() != fb->nullable()) {
+            return false;
+        }
+        if (a->id() == arrow::Type::STRUCT) {
+            if (fa->name() != fb->name()) {
+                return false;
+            }
+            // Compare IDs only when present (a map entry's key/value carry none).
+            auto id_a = NestedProjectionUtils::GetPaimonFieldId(fa);
+            auto id_b = NestedProjectionUtils::GetPaimonFieldId(fb);
+            if (id_a.ok() && id_b.ok() && id_a.value() != id_b.value()) {
+                return false;
+            }
+        }
+        PAIMON_ASSIGN_OR_RAISE(bool child_equal, EqualWithFieldIds(fa->type(), fb->type()));
+        if (!child_equal) {
+            return false;
+        }
+    }
+    return true;
+}
+
 /// Whether `read_type` is `data_type` with variant columns replaced by their variant-access
-/// projections and nothing else changed. Such a read drops no field, so it is not a partial
-/// projection of an enclosing repeated group and may pass through where a real one must fail.
-bool IsVariantAccessSubstitution(const std::shared_ptr& read_type,
-                                 const std::shared_ptr& data_type) {
-    if (read_type->Equals(data_type)) {
+/// projections and nothing else changed (matching paimon field IDs).
+Result IsVariantAccessSubstitution(const std::shared_ptr& read_type,
+                                         const std::shared_ptr& data_type) {
+    PAIMON_ASSIGN_OR_RAISE(bool equal, EqualWithFieldIds(read_type, data_type));
+    if (equal) {
         return true;
     }
     if (VariantAccessUtils::IsVariantAccessType(read_type) &&
@@ -172,25 +210,113 @@ bool IsVariantAccessSubstitution(const std::shared_ptr& read_ty
     for (int32_t i = 0; i < read_type->num_fields(); ++i) {
         const std::shared_ptr& read_child = read_type->field(i);
         const std::shared_ptr& data_child = data_type->field(i);
-        // LIST and MAP name their children by format convention, so only STRUCT is matched
-        // by name.
-        if (read_type->id() == arrow::Type::STRUCT && read_child->name() != data_child->name()) {
-            return false;
+        // LIST and MAP name their children by format convention, so only STRUCT is
+        // matched by name and field ID.
+        if (read_type->id() == arrow::Type::STRUCT) {
+            if (read_child->name() != data_child->name()) {
+                return false;
+            }
+            auto id_r = NestedProjectionUtils::GetPaimonFieldId(read_child);
+            auto id_d = NestedProjectionUtils::GetPaimonFieldId(data_child);
+            if (id_r.ok() && id_d.ok() && id_r.value() != id_d.value()) {
+                return false;
+            }
         }
-        if (!IsVariantAccessSubstitution(read_child->type(), data_child->type())) {
+        PAIMON_ASSIGN_OR_RAISE(bool sub,
+                               IsVariantAccessSubstitution(read_child->type(), data_child->type()));
+        if (!sub) {
             return false;
         }
     }
     return true;
 }
 
+// Reconcile a LIST/MAP item: read may ADD fields (evolution, null-filled
+// downstream) but must not DROP one. Returns the file-readable item type;
+// `container` names the container ("list"/"map") for the error message.
+Result> PruneRepeatedItemType(
+    const std::shared_ptr& read_type,
+    const std::shared_ptr& data_type, const char* container) {
+    PAIMON_ASSIGN_OR_RAISE(bool same, EqualWithFieldIds(read_type, data_type));
+    if (same) {
+        return data_type;
+    }
+    PAIMON_ASSIGN_OR_RAISE(bool substitution, IsVariantAccessSubstitution(read_type, data_type));
+    if (substitution) {
+        return read_type;
+    }
+    if (read_type->id() != data_type->id()) {
+        return Status::Invalid(
+            fmt::format("PruneDataType nested item type mismatch inside {}: read {} vs data {}",
+                        container, read_type->ToString(), data_type->ToString()));
+    }
+    switch (data_type->id()) {
+        case arrow::Type::STRUCT: {
+            arrow::FieldVector item_fields;
+            for (const auto& data_child : data_type->fields()) {
+                PAIMON_ASSIGN_OR_RAISE(int32_t data_child_id,
+                                       NestedProjectionUtils::GetPaimonFieldId(data_child));
+                std::shared_ptr read_child;
+                for (const auto& candidate : read_type->fields()) {
+                    PAIMON_ASSIGN_OR_RAISE(int32_t candidate_id,
+                                           NestedProjectionUtils::GetPaimonFieldId(candidate));
+                    if (candidate_id == data_child_id) {
+                        read_child = candidate;
+                        break;
+                    }
+                }
+                if (!read_child) {
+                    // A file field is dropped -- a real partial projection.
+                    return Status::Invalid(fmt::format(
+                        "PruneDataType does not support partial projection inside {}: src {} vs "
+                        "target {}",
+                        container, data_type->ToString(), read_type->ToString()));
+                }
+                if (read_child->name() != data_child->name()) {
+                    return Status::Invalid(fmt::format(
+                        "PruneDataType does not support renaming inside {}: field id {} read '{}' "
+                        "vs data '{}'",
+                        container, data_child_id, read_child->name(), data_child->name()));
+                }
+                PAIMON_ASSIGN_OR_RAISE(
+                    std::shared_ptr item_child_type,
+                    PruneRepeatedItemType(read_child->type(), data_child->type(), container));
+                item_fields.push_back(data_child->WithType(item_child_type));
+            }
+            return arrow::struct_(item_fields);
+        }
+        case arrow::Type::LIST: {
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr item,
+                                   PruneRepeatedItemType(read_type->field(0)->type(),
+                                                         data_type->field(0)->type(), container));
+            return arrow::list(data_type->field(0)->WithType(item));
+        }
+        case arrow::Type::MAP: {
+            auto read_map = std::static_pointer_cast(read_type);
+            auto data_map = std::static_pointer_cast(data_type);
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr key,
+                PruneRepeatedItemType(read_map->key_type(), data_map->key_type(), container));
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr item,
+                PruneRepeatedItemType(read_map->item_type(), data_map->item_type(), container));
+            return std::static_pointer_cast(std::make_shared(
+                data_map->key_field()->WithType(key), data_map->item_field()->WithType(item),
+                data_map->keys_sorted()));
+        }
+        default:
+            return data_type;
+    }
+}
+
 }  // namespace
 
 Result>> NestedProjectionUtils::PruneDataType(
     const std::shared_ptr& read_type,
     const std::shared_ptr& data_type) {
-    // Identical types need no pruning.
-    if (read_type->Equals(data_type)) {
+    // Identical types (including paimon field IDs) need no pruning.
+    PAIMON_ASSIGN_OR_RAISE(bool same, EqualWithFieldIds(read_type, data_type));
+    if (same) {
         return std::optional>(data_type);
     }
 
@@ -235,25 +361,35 @@ Result>> NestedProjectionUtils::P
             return std::optional>(arrow::struct_(pruned_fields));
         }
         case arrow::Type::LIST: {
-            if (IsVariantAccessSubstitution(read_type, data_type)) {
+            PAIMON_ASSIGN_OR_RAISE(bool list_substitution,
+                                   IsVariantAccessSubstitution(read_type, data_type));
+            if (list_substitution) {
                 return std::optional>(read_type);
             }
-            // Keep behavior aligned with format readers: partial projection inside
-            // LIST is unsupported and must fail fast.
-            return Status::Invalid(
-                fmt::format("PruneDataType does not support partial projection inside list: src {} "
-                            "vs target {}",
-                            data_type->ToString(), read_type->ToString()));
+            // Added fields (schema evolution) are allowed; dropped fields still fail.
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr item,
+                                   PruneRepeatedItemType(read_type->field(0)->type(),
+                                                         data_type->field(0)->type(), "list"));
+            return std::optional>(
+                arrow::list(data_type->field(0)->WithType(item)));
         }
         case arrow::Type::MAP: {
-            if (IsVariantAccessSubstitution(read_type, data_type)) {
+            PAIMON_ASSIGN_OR_RAISE(bool map_substitution,
+                                   IsVariantAccessSubstitution(read_type, data_type));
+            if (map_substitution) {
                 return std::optional>(read_type);
             }
-            // Keep behavior aligned with format readers: partial projection inside
-            // MAP is unsupported and must fail fast.
-            return Status::Invalid(fmt::format(
-                "PruneDataType does not support partial projection inside map: src {} vs target {}",
-                data_type->ToString(), read_type->ToString()));
+            auto read_map = std::static_pointer_cast(read_type);
+            auto data_map = std::static_pointer_cast(data_type);
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr key,
+                PruneRepeatedItemType(read_map->key_type(), data_map->key_type(), "map"));
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr item,
+                PruneRepeatedItemType(read_map->item_type(), data_map->item_type(), "map"));
+            return std::optional>(std::make_shared(
+                data_map->key_field()->WithType(key), data_map->item_field()->WithType(item),
+                data_map->keys_sorted()));
         }
         default:
             // Atomic type: return data_type as-is (type evolution is handled
@@ -455,4 +591,130 @@ Result> NestedProjectionUtils::FilterMapArrayBySel
     return result_map;
 }
 
+namespace {
+// Strips physical-only differences from a leaf type: ORC lazy decoding wraps
+// strings in a dictionary and may widen them to large_string. binary is not
+// dictionary-encoded and large_binary is blob's real type, so neither is
+// normalized. Two leaves with equal normalized types hold the same logical
+// values.
+std::shared_ptr NormalizeLeafRepresentation(
+    const std::shared_ptr& type) {
+    auto t = type;
+    if (t->id() == arrow::Type::DICTIONARY) {
+        t = std::static_pointer_cast(t)->value_type();
+    }
+    if (t->id() == arrow::Type::LARGE_STRING) {
+        return arrow::utf8();
+    }
+    return t;
+}
+}  // namespace
+
+Result> NestedProjectionUtils::AlignArrayToReadType(
+    const std::shared_ptr& array, const std::shared_ptr& read_type,
+    arrow::MemoryPool* pool) {
+    PAIMON_ASSIGN_OR_RAISE(bool same, EqualWithFieldIds(array->type(), read_type));
+    if (same) {
+        return array;
+    }
+    // Produce exactly `read_type` so every file yields the same output type: rebuild
+    // STRUCT/LIST/MAP with read-side types/nullability and cast a leaf (decodes dict).
+    const auto& data = array->data();
+    switch (read_type->id()) {
+        case arrow::Type::STRUCT: {
+            if (array->type()->id() != arrow::Type::STRUCT) {
+                return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}",
+                                                   array->type()->ToString(),
+                                                   read_type->ToString()));
+            }
+            const auto& array_type = array->type();
+            std::vector> children;
+            children.reserve(read_type->num_fields());
+            for (const auto& read_field : read_type->fields()) {
+                // Match by name (parquet drops nested field-id metadata); if both
+                // carry IDs they must agree, so a drop+add same-name field won't match.
+                auto read_id = GetPaimonFieldId(read_field);
+                int32_t match = -1;
+                for (int32_t j = 0; j < array_type->num_fields(); j++) {
+                    const auto& array_field = array_type->field(j);
+                    if (array_field->name() != read_field->name()) {
+                        continue;
+                    }
+                    auto data_id = GetPaimonFieldId(array_field);
+                    if (read_id.ok() && data_id.ok() && read_id.value() != data_id.value()) {
+                        continue;
+                    }
+                    match = j;
+                    break;
+                }
+                if (match >= 0) {
+                    auto child = arrow::MakeArray(data->child_data[match]);
+                    PAIMON_ASSIGN_OR_RAISE(child,
+                                           AlignArrayToReadType(child, read_field->type(), pool));
+                    children.push_back(child->data());
+                } else {
+                    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                        std::shared_ptr null_child,
+                        arrow::MakeArrayOfNull(read_field->type(), data->offset + data->length,
+                                               pool));
+                    children.push_back(null_child->data());
+                }
+            }
+            auto new_data = data->Copy();
+            new_data->type = read_type;
+            new_data->child_data = std::move(children);
+            return arrow::MakeArray(new_data);
+        }
+        case arrow::Type::LIST: {
+            if (array->type()->id() != arrow::Type::LIST) {
+                return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}",
+                                                   array->type()->ToString(),
+                                                   read_type->ToString()));
+            }
+            auto read_list = std::static_pointer_cast(read_type);
+            auto values = arrow::MakeArray(data->child_data[0]);
+            PAIMON_ASSIGN_OR_RAISE(values,
+                                   AlignArrayToReadType(values, read_list->value_type(), pool));
+            auto new_data = data->Copy();
+            new_data->type = read_type;
+            new_data->child_data = {values->data()};
+            return arrow::MakeArray(new_data);
+        }
+        case arrow::Type::MAP: {
+            if (array->type()->id() != arrow::Type::MAP) {
+                return Status::Invalid(fmt::format("AlignArrayToReadType cannot reconcile {} to {}",
+                                                   array->type()->ToString(),
+                                                   read_type->ToString()));
+            }
+            auto read_map = std::static_pointer_cast(read_type);
+            const auto& entries_data = data->child_data[0];
+            auto key = arrow::MakeArray(entries_data->child_data[0]);
+            auto value = arrow::MakeArray(entries_data->child_data[1]);
+            PAIMON_ASSIGN_OR_RAISE(key, AlignArrayToReadType(key, read_map->key_type(), pool));
+            PAIMON_ASSIGN_OR_RAISE(value, AlignArrayToReadType(value, read_map->item_type(), pool));
+            auto new_entries = entries_data->Copy();
+            new_entries->type = arrow::struct_({read_map->key_field(), read_map->item_field()});
+            new_entries->child_data = {key->data(), value->data()};
+            auto new_data = data->Copy();
+            new_data->type = read_type;
+            new_data->child_data = {new_entries};
+            return arrow::MakeArray(new_data);
+        }
+        default: {
+            // Leaf: only physical-representation differences are valid here (ORC
+            // dictionary encoding, string/binary offset width). Genuine type
+            // evolution is handled by FieldMappingReader's cast executors and
+            // rejected upstream in PruneDataType, so fail anything else.
+            if (!NormalizeLeafRepresentation(array->type())
+                     ->Equals(*NormalizeLeafRepresentation(read_type))) {
+                return Status::Invalid(
+                    fmt::format("AlignArrayToReadType unsupported leaf type change: data {} vs "
+                                "read {}",
+                                array->type()->ToString(), read_type->ToString()));
+            }
+            return CastingUtils::Cast(array, read_type, arrow::compute::CastOptions::Safe(), pool);
+        }
+    }
+}
+
 }  // namespace paimon
diff --git a/src/paimon/core/utils/nested_projection_utils.h b/src/paimon/core/utils/nested_projection_utils.h
index 983d7376..b0ab8fc2 100644
--- a/src/paimon/core/utils/nested_projection_utils.h
+++ b/src/paimon/core/utils/nested_projection_utils.h
@@ -88,6 +88,13 @@ class PAIMON_EXPORT NestedProjectionUtils {
         const std::shared_ptr& map_array,
         const std::vector& selected_keys, arrow::MemoryPool* pool);
 
+    /// Reshape `array` to `read_type`, null-filling nested fields added by schema
+    /// evolution. No-op when types match. STRUCT matches children by paimon field id;
+    /// LIST/MAP recurse into items, preserving offsets and validity.
+    static Result> AlignArrayToReadType(
+        const std::shared_ptr& array,
+        const std::shared_ptr& read_type, arrow::MemoryPool* pool);
+
  private:
     static Result HasNestedSubfieldProjectionType(
         const std::shared_ptr& file_type,
diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp b/src/paimon/core/utils/nested_projection_utils_test.cpp
index c91adff3..7905b2ce 100644
--- a/src/paimon/core/utils/nested_projection_utils_test.cpp
+++ b/src/paimon/core/utils/nested_projection_utils_test.cpp
@@ -19,11 +19,13 @@
 
 #include "paimon/core/utils/nested_projection_utils.h"
 
+#include "arrow/array/array_dict.h"
 #include "arrow/array/array_nested.h"
 #include "arrow/array/builder_binary.h"
 #include "arrow/array/builder_dict.h"
 #include "arrow/array/builder_nested.h"
 #include "arrow/array/builder_primitive.h"
+#include "arrow/ipc/json_simple.h"
 #include "arrow/memory_pool.h"
 #include "arrow/type.h"
 #include "gtest/gtest.h"
@@ -225,6 +227,177 @@ TEST(NestedProjectionUtilsTest, PruneDataTypeListDroppingSiblingOfVariantStillFa
                         "partial projection inside list");
 }
 
+TEST(NestedProjectionUtilsTest, PruneDataTypeListStructSchemaEvolutionAddedField) {
+    // Added field (id=12) inside the list's struct: return the file struct.
+    auto data_inner =
+        arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)});
+    auto read_inner =
+        arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11),
+                        MakeField("c", arrow::int32(), 12)});
+    auto data_type = arrow::list(arrow::field("item", data_inner));
+    auto read_type = arrow::list(arrow::field("item", read_inner));
+
+    ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type));
+    ASSERT_TRUE(result.has_value());
+    ASSERT_TRUE(result.value()->Equals(*data_type)) << result.value()->ToString();
+}
+
+TEST(NestedProjectionUtilsTest, PruneDataTypeMapStructSchemaEvolutionAddedField) {
+    // Added field inside a MAP value.
+    auto data_inner =
+        arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)});
+    auto read_inner =
+        arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11),
+                        MakeField("c", arrow::int32(), 12)});
+    auto data_type = arrow::map(arrow::utf8(), data_inner);
+    auto read_type = arrow::map(arrow::utf8(), read_inner);
+
+    ASSERT_OK_AND_ASSIGN(auto result, NestedProjectionUtils::PruneDataType(read_type, data_type));
+    ASSERT_TRUE(result.has_value());
+    ASSERT_TRUE(result.value()->Equals(*data_type)) << result.value()->ToString();
+}
+
+TEST(NestedProjectionUtilsTest, PruneDataTypeListStructDropAndAddStillFails) {
+    // Dropping a file field (b) is a real partial projection -- keep failing.
+    auto data_inner =
+        arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)});
+    auto read_inner =
+        arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("c", arrow::int32(), 12)});
+    auto data_type = arrow::list(arrow::field("item", data_inner));
+    auto read_type = arrow::list(arrow::field("item", read_inner));
+
+    ASSERT_NOK_WITH_MSG(NestedProjectionUtils::PruneDataType(read_type, data_type),
+                        "partial projection inside list");
+}
+
+TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeNullFillsAddedListStructField) {
+    auto* pool = arrow::default_memory_pool();
+    // file array: list> = [ [{1,"x"},{2,"y"}], [{3,"z"}] ]
+    arrow::Int32Builder ab(pool);
+    ASSERT_TRUE(ab.AppendValues({1, 2, 3}).ok());
+    std::shared_ptr a;
+    ASSERT_TRUE(ab.Finish(&a).ok());
+    arrow::StringBuilder bb(pool);
+    ASSERT_TRUE(bb.AppendValues({"x", "y", "z"}).ok());
+    std::shared_ptr b;
+    ASSERT_TRUE(bb.Finish(&b).ok());
+    auto data_struct_type =
+        arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11)});
+    std::shared_ptr struct_arr =
+        arrow::StructArray::Make({a, b}, data_struct_type->fields()).ValueOrDie();
+    arrow::Int32Builder offb(pool);
+    ASSERT_TRUE(offb.AppendValues({0, 2, 3}).ok());
+    std::shared_ptr offsets;
+    ASSERT_TRUE(offb.Finish(&offsets).ok());
+    std::shared_ptr list_arr =
+        arrow::ListArray::FromArrays(*offsets, *struct_arr, pool).ValueOrDie();
+
+    // read type adds c:int(12) inside the struct.
+    auto read_struct =
+        arrow::struct_({MakeField("a", arrow::int32(), 10), MakeField("b", arrow::utf8(), 11),
+                        MakeField("c", arrow::int32(), 12)});
+    auto read_type = arrow::list(arrow::field("item", read_struct));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned,
+                         NestedProjectionUtils::AlignArrayToReadType(list_arr, read_type, pool));
+    ASSERT_TRUE(aligned->type()->Equals(*read_type)) << aligned->type()->ToString();
+    auto out_struct = std::static_pointer_cast(
+        std::static_pointer_cast(aligned)->values());
+    auto c_col = out_struct->GetFieldByName("c");
+    ASSERT_NE(c_col, nullptr);
+    ASSERT_EQ(c_col->null_count(), c_col->length());  // added field is all null
+    auto a_col = std::static_pointer_cast(out_struct->GetFieldByName("a"));
+    ASSERT_EQ(a_col->Value(0), 1);
+    ASSERT_EQ(a_col->Value(2), 3);
+}
+
+TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeDecodesDictionaryLeafAndNullFills) {
+    // ORC lazy decoding returns dictionary; decode/cast to string.
+    auto* pool = arrow::default_memory_pool();
+    auto dict_type = arrow::dictionary(arrow::int64(), arrow::large_utf8());
+    auto a_dict =
+        arrow::ipc::internal::json::ArrayFromJSON(dict_type, R"(["x", "y", "x"])").ValueOrDie();
+    auto data_struct = arrow::struct_({MakeField("a", dict_type, 10)});
+    auto struct_arr = arrow::StructArray::Make({a_dict}, data_struct->fields()).ValueOrDie();
+
+    auto read_type =
+        arrow::struct_({MakeField("a", arrow::utf8(), 10), MakeField("b", arrow::int32(), 11)});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned,
+                         NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool));
+    ASSERT_TRUE(aligned->type()->Equals(*read_type)) << aligned->type()->ToString();
+    auto out = std::static_pointer_cast(aligned);
+    ASSERT_EQ(std::static_pointer_cast(out->GetFieldByName("a"))->GetString(0),
+              "x");
+    auto b = out->GetFieldByName("b");
+    ASSERT_EQ(b->null_count(), b->length());
+}
+
+TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeAppliesReadNullability) {
+    auto* pool = arrow::default_memory_pool();
+    arrow::Int32Builder ab(pool);
+    ASSERT_TRUE(ab.AppendValues({1, 2}).ok());
+    std::shared_ptr a_arr;
+    ASSERT_TRUE(ab.Finish(&a_arr).ok());
+    auto data_field = DataField::ConvertDataFieldToArrowField(
+        DataField(10, arrow::field("a", arrow::int32(), /*nullable=*/false)));
+    auto struct_arr = arrow::StructArray::Make({a_arr}, {data_field}).ValueOrDie();
+    auto read_type = arrow::struct_({MakeField("a", arrow::int32(), 10)});
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned,
+                         NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool));
+    ASSERT_TRUE(aligned->type()->field(0)->nullable());
+}
+
+TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeFieldIdChangeNullFillsNotLeak) {
+    // a(id=10) replaced by a(id=11), same name/type: new field must read null, not leak.
+    auto* pool = arrow::default_memory_pool();
+    arrow::Int32Builder ab(pool);
+    ASSERT_TRUE(ab.AppendValues({42}).ok());
+    std::shared_ptr a_arr;
+    ASSERT_TRUE(ab.Finish(&a_arr).ok());
+    auto data_struct = arrow::struct_({MakeField("a", arrow::int32(), 10)});
+    auto struct_arr = arrow::StructArray::Make({a_arr}, data_struct->fields()).ValueOrDie();
+    auto read_type = arrow::struct_({MakeField("a", arrow::int32(), 11)});
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned,
+                         NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool));
+    auto a_out = std::static_pointer_cast(aligned)->GetFieldByName("a");
+    ASSERT_NE(a_out, nullptr);
+    ASSERT_EQ(a_out->null_count(), a_out->length());
+}
+
+TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeRejectsLeafTypeChange) {
+    auto* pool = arrow::default_memory_pool();
+    arrow::Int32Builder ab(pool);
+    ASSERT_TRUE(ab.AppendValues({1}).ok());
+    std::shared_ptr a_arr;
+    ASSERT_TRUE(ab.Finish(&a_arr).ok());
+    auto data_struct = arrow::struct_({MakeField("a", arrow::int32(), 10)});
+    auto struct_arr = arrow::StructArray::Make({a_arr}, data_struct->fields()).ValueOrDie();
+    auto read_type = arrow::struct_({MakeField("a", arrow::int64(), 10)});
+
+    ASSERT_NOK_WITH_MSG(NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool),
+                        "AlignArrayToReadType unsupported leaf type change");
+}
+
+TEST(NestedProjectionUtilsTest, AlignArrayToReadTypeKeepsNestedLargeBinaryBlob) {
+    auto* pool = arrow::default_memory_pool();
+    arrow::LargeBinaryBuilder blobb(pool);
+    ASSERT_TRUE(blobb.AppendValues({"a", "b"}).ok());
+    std::shared_ptr blob;
+    ASSERT_TRUE(blobb.Finish(&blob).ok());
+    auto data_struct = arrow::struct_({MakeField("blob", arrow::large_binary(), 10)});
+    auto struct_arr = arrow::StructArray::Make({blob}, data_struct->fields()).ValueOrDie();
+    auto read_type = arrow::struct_(
+        {MakeField("blob", arrow::large_binary(), 10), MakeField("c", arrow::int32(), 11)});
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr aligned,
+                         NestedProjectionUtils::AlignArrayToReadType(struct_arr, read_type, pool));
+    auto blob_out = std::static_pointer_cast(aligned)->GetFieldByName("blob");
+    ASSERT_EQ(blob_out->type_id(), arrow::Type::LARGE_BINARY);
+    ASSERT_TRUE(blob_out->Equals(*blob));
+}
+
 TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) {
     auto file_schema = arrow::schema({
         MakeField("f0", arrow::int32(), 1),
diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp
index d27d08e3..04638892 100644
--- a/test/inte/write_and_read_inte_test.cpp
+++ b/test/inte/write_and_read_inte_test.cpp
@@ -422,6 +422,95 @@ TEST_P(WriteAndReadInteTest, TestNestedType) {
     ASSERT_TRUE(success);
 }
 
+TEST_P(WriteAndReadInteTest, TestSchemaEvolutionAddFieldInsideListAndMap) {
+    auto [file_format, file_system] = GetParam();
+    if (file_format == "lance" || file_format == "avro") {
+        return;
+    }
+    auto list_struct =
+        arrow::struct_({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())});
+    auto map_value = arrow::struct_({arrow::field("m1", arrow::int32())});
+    arrow::FieldVector fields = {
+        arrow::field("id", arrow::int32()),
+        arrow::field("items", arrow::list(arrow::field("item", list_struct))),
+        arrow::field("props", arrow::map(arrow::utf8(), map_value)),
+    };
+    std::map options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"},
+        {Options::BUCKET, "-1"},
+        {Options::FILE_SYSTEM, file_system},
+        // Exercise ORC lazy decoding, which returns nested strings as dictionaries.
+        {"orc.read.enable-lazy-decoding", "true"},
+    };
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_, arrow::schema(fields),
+                                                         /*partition_keys=*/{}, /*primary_keys=*/{},
+                                                         options, /*is_streaming_mode=*/false));
+    ASSERT_OK_AND_ASSIGN(auto batch, TestHelper::MakeRecordBatch(arrow::struct_(fields),
+                                                                 R"([
+                [1, [[10, "x"], [20, "y"]], [["k1", [100]]]],
+                [2, [[30, "z"]], [["k2", [200]]]]
+            ])",
+                                                                 /*partition_map=*/{},
+                                                                 /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    std::string table_path = PathUtil::JoinPath(test_dir_, "foo.db/bar");
+    SchemaManager schema_manager(dir_->GetFileSystem(), table_path);
+    ASSERT_OK_AND_ASSIGN(auto schema_v0, schema_manager.ReadSchema(0));
+    std::vector fields_v0 = schema_v0->Fields();
+    int32_t next_id = schema_v0->HighestFieldId();
+
+    auto items_field = fields_v0[1].ArrowField();
+    auto items_list = arrow::internal::checked_pointer_cast(items_field->type());
+    auto items_struct =
+        arrow::internal::checked_pointer_cast(items_list->value_type());
+    auto c_field = DataField::ConvertDataFieldToArrowField(
+        DataField(++next_id, arrow::field("c", arrow::int32())));
+    auto new_items_type = arrow::list(items_list->value_field()->WithType(
+        arrow::struct_({items_struct->field(0), items_struct->field(1), c_field})));
+
+    auto props_field = fields_v0[2].ArrowField();
+    auto props_map = arrow::internal::checked_pointer_cast(props_field->type());
+    auto props_value =
+        arrow::internal::checked_pointer_cast(props_map->item_type());
+    auto m2_field = DataField::ConvertDataFieldToArrowField(
+        DataField(++next_id, arrow::field("m2", arrow::int32())));
+    auto new_props_type = arrow::map(
+        props_map->key_type(),
+        props_map->item_field()->WithType(arrow::struct_({props_value->field(0), m2_field})));
+
+    std::vector fields_v1 = fields_v0;
+    fields_v1[1] = DataField(fields_v0[1].Id(), items_field->WithType(new_items_type));
+    fields_v1[2] = DataField(fields_v0[2].Id(), props_field->WithType(new_props_type));
+    ASSERT_OK(WriteNextSchema(fields_v1, next_id, options));
+
+    auto expected_type = arrow::struct_({
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        arrow::field("id", arrow::int32()),
+        arrow::field("items", arrow::list(arrow::field(
+                                  "item", arrow::struct_({arrow::field("a", arrow::int32()),
+                                                          arrow::field("b", arrow::utf8()),
+                                                          arrow::field("c", arrow::int32())})))),
+        arrow::field("props", arrow::map(arrow::utf8(),
+                                         arrow::struct_({arrow::field("m1", arrow::int32()),
+                                                         arrow::field("m2", arrow::int32())}))),
+    });
+    ASSERT_OK_AND_ASSIGN(auto splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits,
+                                                                  R"([
+                [0, 1, [[10, "x", null], [20, "y", null]], [["k1", [100, null]]]],
+                [0, 2, [[30, "z", null]], [["k2", [200, null]]]]
+            ])"));
+    ASSERT_TRUE(success);
+}
+
 TEST_P(WriteAndReadInteTest, TestAppendExternalPath) {
     arrow::FieldVector fields = {
         arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()),
@@ -2457,10 +2546,9 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionRea
     fields_with_changed_tag_value[1] =
         DataField(fields_v0[1].Id(), tag_field->WithType(changed_tag_value_type));
     ASSERT_OK(WriteNextSchema(fields_with_changed_tag_value, schema_v0->HighestFieldId(), options));
-    ASSERT_NOK_WITH_MSG(read_fields({"tags"}),
-                        "PruneDataType does not support partial projection inside map: src "
-                        "map> vs target "
-                        "map>");
+    ASSERT_NOK_WITH_MSG(
+        read_fields({"tags"}),
+        "PruneDataType nested item type mismatch inside map: read string vs data int64");
 
     auto profile_field = fields_v0[2].ArrowField();
     auto profile_struct =

From 0cd7ef1a29e106ece091ce651145ef0f9b62560c Mon Sep 17 00:00:00 2001
From: Wei Zhang 
Date: Tue, 28 Jul 2026 08:42:04 +0800
Subject: [PATCH 120/138] fix(abi): dynamic_cast error on Predicate across
 shared libraries

---
 include/paimon/predicate/predicate.h      |   2 +-
 src/paimon/CMakeLists.txt                 |   1 +
 src/paimon/common/predicate/predicate.cpp |  25 +++
 test/inte/CMakeLists.txt                  |   7 +
 test/inte/predicate_abi_inte_test.cpp     | 235 ++++++++++++++++++++++
 5 files changed, 269 insertions(+), 1 deletion(-)
 create mode 100644 src/paimon/common/predicate/predicate.cpp
 create mode 100644 test/inte/predicate_abi_inte_test.cpp

diff --git a/include/paimon/predicate/predicate.h b/include/paimon/predicate/predicate.h
index bd2dbb49..6c094eee 100644
--- a/include/paimon/predicate/predicate.h
+++ b/include/paimon/predicate/predicate.h
@@ -33,7 +33,7 @@ class Function;
 /// @see PredicateBuilder
 class PAIMON_EXPORT Predicate {
  public:
-    virtual ~Predicate() = default;
+    virtual ~Predicate();
     virtual bool operator==(const Predicate& other) const = 0;
 
     virtual const Function& GetFunction() const = 0;
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 2be943b4..46521534 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -123,6 +123,7 @@ set(PAIMON_COMMON_SRCS
     common/predicate/not_equal.cpp
     common/predicate/not_in.cpp
     common/predicate/or.cpp
+    common/predicate/predicate.cpp
     common/predicate/predicate_builder.cpp
     common/predicate/predicate_utils.cpp
     common/predicate/starts_with.cpp
diff --git a/src/paimon/common/predicate/predicate.cpp b/src/paimon/common/predicate/predicate.cpp
new file mode 100644
index 00000000..6d902080
--- /dev/null
+++ b/src/paimon/common/predicate/predicate.cpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/predicate/predicate.h"
+
+namespace paimon {
+/// key function technique for Predicate
+Predicate::~Predicate() = default;
+}  // namespace paimon
diff --git a/test/inte/CMakeLists.txt b/test/inte/CMakeLists.txt
index d50ae178..85fb8f30 100644
--- a/test/inte/CMakeLists.txt
+++ b/test/inte/CMakeLists.txt
@@ -113,4 +113,11 @@ if(PAIMON_BUILD_TESTS)
                     test_utils_static
                     ${GTEST_LINK_TOOLCHAIN})
 
+    add_paimon_test(predicate_abi_inte_test
+                    STATIC_LINK_LIBS
+                    paimon_shared
+                    ${TEST_STATIC_LINK_LIBS}
+                    test_utils_static
+                    ${GTEST_LINK_TOOLCHAIN})
+
 endif()
diff --git a/test/inte/predicate_abi_inte_test.cpp b/test/inte/predicate_abi_inte_test.cpp
new file mode 100644
index 00000000..3590eb22
--- /dev/null
+++ b/test/inte/predicate_abi_inte_test.cpp
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Regression test for the Predicate cross-DSO RTTI/ABI fix (the out-of-line
+// `Predicate::~Predicate` key function in common/predicate/predicate.cpp).
+//
+// `Predicate` is an abstract base with only pure virtuals. Without an
+// out-of-line virtual ("key function"), the compiler has no single home for its
+// vtable and typeinfo, so it emits them *weakly* in every translation unit that
+// uses them. Combined with `-fvisibility=hidden`, separately linked modules
+// (e.g. libpaimon.so vs libpaimon_parquet_file_format.so, or plugins loaded via
+// dlopen) can each end up with their own `Predicate` typeinfo. A cross-module
+// `dynamic_cast`/`dynamic_pointer_cast` then compares mismatched typeinfo and
+// fails. Giving `Predicate` an out-of-line destructor anchors a single,
+// exported definition of its RTTI that all other modules import.
+//
+// This file guards the fix on two levels:
+//   1. Behavior: `Predicate` objects are created inside libpaimon.so and cast to
+//      derived types here in the test module. This catches the failure on
+//      toolchains that compare std::type_info by pointer (e.g. libc++) and in
+//      downstream builds that link plugins aggressively.
+//   2. Symbol layout (Linux/ELF only): assert that `Predicate`'s typeinfo has a
+//      single home in libpaimon.so which the format plugins import, rather than
+//      each plugin emitting its own duplicate copy. This is the deterministic
+//      signature of the key function on glibc/libstdc++ toolchains, which merge
+//      weak typeinfo by name and therefore would not fail the behavioral cast
+//      even without the fix.
+
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/defs.h"
+#include "paimon/predicate/compound_predicate.h"
+#include "paimon/predicate/leaf_predicate.h"
+#include "paimon/predicate/literal.h"
+#include "paimon/predicate/predicate.h"
+#include "paimon/predicate/predicate_builder.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+// A leaf predicate built in libpaimon.so must survive dynamic_pointer_cast to
+// LeafPredicate across the module boundary.
+TEST(PredicateAbiInteTest, LeafPredicateCastsAcrossModule) {
+    std::shared_ptr predicate =
+        PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT,
+                                Literal(static_cast(5)));
+    ASSERT_NE(predicate, nullptr);
+
+    std::shared_ptr leaf = std::dynamic_pointer_cast(predicate);
+    ASSERT_NE(leaf, nullptr) << "dynamic_pointer_cast failed across the module "
+                                "boundary; Predicate RTTI is likely duplicated (missing key "
+                                "function)";
+    EXPECT_EQ(leaf->FieldName(), "f0");
+    EXPECT_EQ(leaf->FieldIndex(), 0);
+
+    // A leaf predicate must not be mistaken for a compound predicate.
+    EXPECT_EQ(std::dynamic_pointer_cast(predicate), nullptr);
+}
+
+// A compound predicate built in libpaimon.so must survive dynamic_pointer_cast
+// to CompoundPredicate, and its children must remain castable to LeafPredicate.
+TEST(PredicateAbiInteTest, CompoundPredicateCastsAcrossModule) {
+    std::shared_ptr left =
+        PredicateBuilder::Equal(/*field_index=*/0, /*field_name=*/"f0", FieldType::BIGINT,
+                                Literal(static_cast(5)));
+    std::shared_ptr right = PredicateBuilder::Equal(
+        /*field_index=*/1, /*field_name=*/"f1", FieldType::INT, Literal(10));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate,
+                         PredicateBuilder::And({left, right}));
+
+    std::shared_ptr compound =
+        std::dynamic_pointer_cast(predicate);
+    ASSERT_NE(compound, nullptr)
+        << "dynamic_pointer_cast failed across the module boundary";
+    ASSERT_EQ(compound->Children().size(), 2u);
+
+    EXPECT_EQ(std::dynamic_pointer_cast(predicate), nullptr);
+    for (const auto& child : compound->Children()) {
+        EXPECT_NE(std::dynamic_pointer_cast(child), nullptr);
+    }
+}
+
+}  // namespace paimon::test
+
+#if defined(__linux__) && defined(__ELF__)
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+
+namespace paimon::test {
+namespace {
+
+// Mangled name of `typeinfo for paimon::Predicate`.
+constexpr const char* kPredicateTypeInfoSymbol = "_ZTIN6paimon9PredicateE";
+
+// Result of looking up a symbol in an ELF dynamic symbol table.
+enum class DynSym {
+    kError,     // could not read the file
+    kAbsent,    // symbol not present in .dynsym
+    kDefined,   // symbol has a definition in this module (st_shndx != SHN_UNDEF)
+    kImported,  // symbol is undefined here and resolved from another module
+};
+
+// On-disk path of the loaded shared object whose path contains `needle`.
+std::string FindLoadedModule(const char* needle) {
+    struct Ctx {
+        const char* needle;
+        std::string path;
+    } ctx{needle, {}};
+    dl_iterate_phdr(
+        [](struct dl_phdr_info* info, size_t, void* data) -> int {
+            auto* c = static_cast(data);
+            if (info->dlpi_name != nullptr && std::strstr(info->dlpi_name, c->needle) != nullptr) {
+                c->path = info->dlpi_name;
+                return 1;
+            }
+            return 0;
+        },
+        &ctx);
+    return ctx.path;
+}
+
+// Whether `sym_name` is defined in, imported by, or absent from the dynamic
+// symbol table of the ELF file at `path`.
+//
+// We look at whether the symbol is *defined* (has a section index) rather than
+// at its binding (GLOBAL vs WEAK): a class with a key function has WEAK typeinfo
+// on GCC but GLOBAL typeinfo on Clang, so the binding is not portable. What both
+// compilers guarantee is that the key function makes the typeinfo a single
+// definition that other modules import.
+DynSym LookupDynSym(const std::string& path, const char* sym_name) {
+    int fd = ::open(path.c_str(), O_RDONLY);
+    if (fd < 0) {
+        return DynSym::kError;
+    }
+    struct stat st{};
+    if (::fstat(fd, &st) != 0) {
+        ::close(fd);
+        return DynSym::kError;
+    }
+    void* map = ::mmap(nullptr, static_cast(st.st_size), PROT_READ, MAP_PRIVATE, fd, 0);
+    ::close(fd);
+    if (map == MAP_FAILED) {
+        return DynSym::kError;
+    }
+
+    const auto* base = static_cast(map);
+    const auto* ehdr = reinterpret_cast(base);
+    DynSym result = DynSym::kAbsent;
+    if (std::memcmp(ehdr->e_ident, ELFMAG, SELFMAG) == 0 && ehdr->e_ident[EI_CLASS] == ELFCLASS64) {
+        const auto* shdrs = reinterpret_cast(base + ehdr->e_shoff);
+        for (int i = 0; i < ehdr->e_shnum; ++i) {
+            if (shdrs[i].sh_type != SHT_DYNSYM) {
+                continue;
+            }
+            const auto* syms = reinterpret_cast(base + shdrs[i].sh_offset);
+            const char* strtab =
+                reinterpret_cast(base + shdrs[shdrs[i].sh_link].sh_offset);
+            size_t count = shdrs[i].sh_size / sizeof(Elf64_Sym);
+            for (size_t s = 0; s < count; ++s) {
+                if (std::strcmp(strtab + syms[s].st_name, sym_name) == 0) {
+                    result = syms[s].st_shndx == SHN_UNDEF ? DynSym::kImported : DynSym::kDefined;
+                    break;
+                }
+            }
+            break;
+        }
+    }
+    ::munmap(map, static_cast(st.st_size));
+    return result;
+}
+
+}  // namespace
+
+// Deterministic guard (compiler-agnostic across GCC and Clang): the out-of-line
+// `~Predicate()` key function must give `Predicate`'s typeinfo a single home in
+// libpaimon.so, which the format plugins then import instead of each emitting
+// their own copy. Without the key function the typeinfo is weak and every
+// module that casts a Predicate defines its own duplicate -- exactly the
+// condition that breaks cross-DSO dynamic_cast. Reverting the fix flips this
+// test from PASS to FAIL.
+TEST(PredicateAbiInteTest, PredicateTypeInfoHasSingleHome) {
+    std::string core = FindLoadedModule("libpaimon.so");
+    ASSERT_FALSE(core.empty()) << "could not locate loaded libpaimon.so";
+
+    // The core library owns the one definition of the typeinfo.
+    EXPECT_EQ(LookupDynSym(core, kPredicateTypeInfoSymbol), DynSym::kDefined)
+        << "typeinfo for paimon::Predicate must be defined in " << core;
+
+    // A format plugin that casts predicates must import that definition rather
+    // than defining its own duplicate. The parquet plugin performs
+    // dynamic_pointer_cast/ on Predicate, so it
+    // references the typeinfo; with the key function that reference resolves to
+    // libpaimon.so (imported), without it the plugin carries its own weak copy.
+    std::string plugin = FindLoadedModule("libpaimon_parquet_file_format.so");
+    ASSERT_FALSE(plugin.empty()) << "could not locate loaded libpaimon_parquet_file_format.so";
+
+    DynSym in_plugin = LookupDynSym(plugin, kPredicateTypeInfoSymbol);
+    ASSERT_NE(in_plugin, DynSym::kError) << "could not read " << plugin;
+    ASSERT_NE(in_plugin, DynSym::kAbsent)
+        << "typeinfo for paimon::Predicate is not referenced by " << plugin
+        << "; the parquet plugin is expected to cast Predicate across the DSO boundary";
+    EXPECT_EQ(in_plugin, DynSym::kImported)
+        << "typeinfo for paimon::Predicate is defined inside " << plugin
+        << " instead of being imported from libpaimon.so; the Predicate RTTI is duplicated across "
+           "DSOs (the ~Predicate() key function is missing), which breaks cross-DSO dynamic_cast";
+}
+
+}  // namespace paimon::test
+#endif  // defined(__linux__) && defined(__ELF__)

From 0d8929d0cf9e442a86cb082f54b614c9de7080c8 Mon Sep 17 00:00:00 2001
From: "Mr Dk." 
Date: Tue, 28 Jul 2026 11:29:08 +0800
Subject: [PATCH 121/138] feat(fs): add object store and S3 file systems

Add a reusable read-only object store layer with a curl-based HTTP
transport.

Use the AWS C authentication components for credential resolution and
request signing while keeping S3 data access independent of the full AWS
SDK.
---
 .github/workflows/gcc8_test.yaml              |    2 +-
 CMakeLists.txt                                |   15 +
 ci/scripts/build_paimon.sh                    |    1 +
 cmake_modules/BuildAwsAuth.cmake              |  189 ++++
 cmake_modules/ThirdpartyToolchain.cmake       |    4 +
 cmake_modules/arrow.diff                      |    8 +
 src/paimon/CMakeLists.txt                     |   15 +
 src/paimon/common/fs/http_client.cpp          |  214 ++++
 src/paimon/common/fs/http_client.h            |   69 ++
 .../common/fs/object_store_file_system.cpp    |  535 +++++++++
 .../common/fs/object_store_file_system.h      |  120 ++
 .../fs/object_store_file_system_test.cpp      |  276 +++++
 src/paimon/fs/s3/CMakeLists.txt               |   51 +
 src/paimon/fs/s3/s3_file_system.cpp           | 1000 +++++++++++++++++
 src/paimon/fs/s3/s3_file_system.h             |   55 +
 src/paimon/fs/s3/s3_file_system_factory.cpp   |   36 +
 src/paimon/fs/s3/s3_file_system_factory.h     |   36 +
 src/paimon/fs/s3/s3_file_system_test.cpp      |  503 +++++++++
 third_party/versions.txt                      |   33 +
 19 files changed, 3161 insertions(+), 1 deletion(-)
 create mode 100644 cmake_modules/BuildAwsAuth.cmake
 create mode 100644 src/paimon/common/fs/http_client.cpp
 create mode 100644 src/paimon/common/fs/http_client.h
 create mode 100644 src/paimon/common/fs/object_store_file_system.cpp
 create mode 100644 src/paimon/common/fs/object_store_file_system.h
 create mode 100644 src/paimon/common/fs/object_store_file_system_test.cpp
 create mode 100644 src/paimon/fs/s3/CMakeLists.txt
 create mode 100644 src/paimon/fs/s3/s3_file_system.cpp
 create mode 100644 src/paimon/fs/s3/s3_file_system.h
 create mode 100644 src/paimon/fs/s3/s3_file_system_factory.cpp
 create mode 100644 src/paimon/fs/s3/s3_file_system_factory.h
 create mode 100644 src/paimon/fs/s3/s3_file_system_test.cpp

diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml
index 692f4585..aae2ce98 100644
--- a/.github/workflows/gcc8_test.yaml
+++ b/.github/workflows/gcc8_test.yaml
@@ -45,7 +45,7 @@ jobs:
       - name: Install dependencies
         run: |
           apt-get update
-          DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
+          DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git tar curl libcurl4-openssl-dev libssl-dev tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
           curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz
           tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local --strip-components=1
           rm cmake-3.28.3-linux-x86_64.tar.gz
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 511aa6f7..2e640713 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -61,6 +61,7 @@ option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
 option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
+option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
 option(PAIMON_ENABLE_NETWORK_TESTS
        "Whether to enable tests that access real remote services over the network" OFF)
 option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
@@ -75,6 +76,9 @@ endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
+if(PAIMON_ENABLE_S3)
+    add_definitions(-DPAIMON_ENABLE_S3)
+endif()
 if(PAIMON_ENABLE_NETWORK_TESTS)
     add_definitions(-DPAIMON_ENABLE_NETWORK_TESTS)
 endif()
@@ -439,6 +443,13 @@ if(PAIMON_BUILD_TESTS)
                                            paimon_jindo_file_system_shared)
         list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
+    if(PAIMON_ENABLE_S3)
+        paimon_link_libraries_whole_archive(PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS
+                                            paimon_s3_file_system_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_s3_file_system_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+    endif()
     if(PAIMON_ENABLE_LUMINA)
         paimon_link_libraries_whole_archive(PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS
                                             paimon_lumina_index_static)
@@ -486,11 +497,15 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake"
 
 config_summary_message()
 
+if(PAIMON_ENABLE_S3)
+    find_package(CURL REQUIRED)
+endif()
 add_subdirectory(src/paimon)
 add_subdirectory(src/paimon/fs/local)
 if(PAIMON_ENABLE_JINDO)
     add_subdirectory(src/paimon/fs/jindo)
 endif()
+add_subdirectory(src/paimon/fs/s3)
 add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
 add_subdirectory(src/paimon/format/parquet)
diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh
index 32350438..3df51435 100755
--- a/ci/scripts/build_paimon.sh
+++ b/ci/scripts/build_paimon.sh
@@ -142,6 +142,7 @@ CMAKE_ARGS=(
     "-DCMAKE_BUILD_TYPE=${build_type}"
     "-DPAIMON_BUILD_TESTS=ON"
     "-DPAIMON_ENABLE_JINDO=ON"
+    "-DPAIMON_ENABLE_S3=ON"
     "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
     "-DPAIMON_ENABLE_LUCENE=ON"
     "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}"
diff --git a/cmake_modules/BuildAwsAuth.cmake b/cmake_modules/BuildAwsAuth.cmake
new file mode 100644
index 00000000..62e749d9
--- /dev/null
+++ b/cmake_modules/BuildAwsAuth.cmake
@@ -0,0 +1,189 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include(ExternalProject)
+include(GNUInstallDirs)
+
+function(paimon_add_s2n_project)
+    if(DEFINED ENV{PAIMON_AWS_S2N_URL})
+        set(S2N_URL "$ENV{PAIMON_AWS_S2N_URL}")
+    elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
+        set(S2N_URL "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
+    else()
+        set(S2N_URL
+            "${THIRDPARTY_MIRROR_URL}https://github.com/aws/s2n-tls/archive/refs/tags/${PAIMON_AWS_S2N_BUILD_VERSION}.zip"
+        )
+    endif()
+    set(S2N_C_FLAGS "${EP_C_FLAGS}")
+    set(S2N_CXX_FLAGS "${EP_CXX_FLAGS}")
+    string(REPLACE "-Wdocumentation" "" S2N_C_FLAGS "${S2N_C_FLAGS}")
+    string(REPLACE "-Wdocumentation" "" S2N_CXX_FLAGS "${S2N_CXX_FLAGS}")
+    externalproject_add(s2n_ep
+                        URL ${S2N_URL}
+                        URL_HASH SHA256=${PAIMON_AWS_S2N_BUILD_SHA256_CHECKSUM}
+                        DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+                        CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS}
+                                   -DCMAKE_C_FLAGS=${S2N_C_FLAGS}
+                                   -DCMAKE_CXX_FLAGS=${S2N_CXX_FLAGS}
+                                   -DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
+                                   -DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
+                                   -DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
+                                   -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+                                   -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
+                                   -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                                   -DS2N_INTERN_LIBCRYPTO=OFF
+                        BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/libs2n.a
+                                         ${THIRDPARTY_LOG_OPTIONS})
+endfunction()
+
+function(paimon_add_aws_c_project
+         NAME
+         VERSION
+         CHECKSUM
+         DEPENDS)
+    string(REPLACE "aws-c-" "AWS_C_" URL_NAME "${NAME}")
+    string(TOUPPER "${URL_NAME}" URL_NAME)
+    set(URL_VAR "PAIMON_${URL_NAME}_URL")
+    set(PKG_NAME_VAR "PAIMON_${URL_NAME}_PKG_NAME")
+    if(DEFINED ENV{${URL_VAR}})
+        set(URL "$ENV{${URL_VAR}}")
+    elseif(EXISTS "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
+        set(URL "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
+    else()
+        set(URL
+            "${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/${NAME}/archive/refs/tags/${VERSION}.tar.gz"
+        )
+    endif()
+    set(AWS_PLATFORM_CMAKE_ARGS)
+    list(APPEND
+         AWS_PLATFORM_CMAKE_ARGS
+         -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+         -DOPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}
+         -DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
+         -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+         -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY})
+    if(NAME STREQUAL "aws-c-cal")
+        list(APPEND AWS_PLATFORM_CMAKE_ARGS -DUSE_OPENSSL=ON)
+    endif()
+    if(APPLE)
+        list(APPEND AWS_PLATFORM_CMAKE_ARGS -DAWS_USE_SECITEM=ON)
+    endif()
+    # aws-c-io's installed package unconditionally looks for s2n on Unix even
+    # when it was built with Apple's native TLS backend.
+    if(APPLE AND (NAME STREQUAL "aws-c-http" OR NAME STREQUAL "aws-c-auth"))
+        list(APPEND AWS_PLATFORM_CMAKE_ARGS -DBYO_CRYPTO=ON)
+    endif()
+    externalproject_add(${NAME}_ep
+                        URL ${URL}
+                        URL_HASH SHA256=${CHECKSUM}
+                        DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+                        CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS}
+                                   -DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
+                                   -DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
+                                   -DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
+                                   -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                                   -DENABLE_TESTING=OFF
+                                   ${AWS_PLATFORM_CMAKE_ARGS}
+                        DEPENDS ${DEPENDS}
+                        BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/lib${NAME}.a
+                                         ${THIRDPARTY_LOG_OPTIONS})
+endfunction()
+
+function(build_aws_auth)
+    set(AWS_AUTH_PREFIX
+        "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install"
+        PARENT_SCOPE)
+    set(AWS_AUTH_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install")
+    set(AWS_AUTH_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}")
+    set(AWS_AUTH_LIB_DIR "${AWS_AUTH_PREFIX}/${AWS_AUTH_INSTALL_LIBDIR}")
+    file(MAKE_DIRECTORY "${AWS_AUTH_PREFIX}/include")
+    file(MAKE_DIRECTORY "${AWS_AUTH_LIB_DIR}")
+
+    find_package(OpenSSL REQUIRED)
+
+    set(AWS_IO_DEPENDS "aws-c-common_ep;aws-c-cal_ep")
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        paimon_add_s2n_project()
+        list(APPEND AWS_IO_DEPENDS s2n_ep)
+    endif()
+
+    paimon_add_aws_c_project(aws-c-common ${PAIMON_AWS_C_COMMON_BUILD_VERSION}
+                             ${PAIMON_AWS_C_COMMON_BUILD_SHA256_CHECKSUM} "")
+    paimon_add_aws_c_project(aws-c-sdkutils ${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION}
+                             ${PAIMON_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}
+                             aws-c-common_ep)
+    paimon_add_aws_c_project(aws-c-cal ${PAIMON_AWS_C_CAL_BUILD_VERSION}
+                             ${PAIMON_AWS_C_CAL_BUILD_SHA256_CHECKSUM} aws-c-common_ep)
+    paimon_add_aws_c_project(aws-c-compression ${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION}
+                             ${PAIMON_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}
+                             aws-c-common_ep)
+    paimon_add_aws_c_project(aws-c-io ${PAIMON_AWS_C_IO_BUILD_VERSION}
+                             ${PAIMON_AWS_C_IO_BUILD_SHA256_CHECKSUM} "${AWS_IO_DEPENDS}")
+    paimon_add_aws_c_project(aws-c-http ${PAIMON_AWS_C_HTTP_BUILD_VERSION}
+                             ${PAIMON_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}
+                             "aws-c-common_ep;aws-c-io_ep;aws-c-compression_ep")
+    paimon_add_aws_c_project(aws-c-auth
+                             ${PAIMON_AWS_C_AUTH_BUILD_VERSION}
+                             ${PAIMON_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}
+                             "aws-c-common_ep;aws-c-sdkutils_ep;aws-c-cal_ep;aws-c-io_ep;aws-c-http_ep"
+    )
+
+    find_package(Threads REQUIRED)
+    set(AWS_AUTH_PLATFORM_LIBS Threads::Threads ${CMAKE_DL_LIBS})
+    if(APPLE)
+        list(APPEND
+             AWS_AUTH_PLATFORM_LIBS
+             "-framework Security"
+             "-framework CoreFoundation"
+             "-framework Network")
+    endif()
+    set(AWS_AUTH_TLS_LIBS)
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        list(APPEND AWS_AUTH_TLS_LIBS "${AWS_AUTH_LIB_DIR}/libs2n.a")
+    endif()
+    set(AWS_AUTH_RUNTIME_LIBS
+        "${AWS_AUTH_LIB_DIR}/libaws-c-http.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-io.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-compression.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-cal.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-sdkutils.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-common.a"
+        ${AWS_AUTH_TLS_LIBS})
+    if(NOT APPLE)
+        list(PREPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--start-group")
+        list(APPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--end-group")
+    endif()
+    # Keep OpenSSL after the static AWS archives. In embedded builds CMake can
+    # resolve these targets to static archives, where link order matters.
+    list(APPEND AWS_AUTH_RUNTIME_LIBS "${OPENSSL_SSL_LIBRARY}"
+         "${OPENSSL_CRYPTO_LIBRARY}")
+    add_library(aws_auth_minimal STATIC IMPORTED GLOBAL)
+    set_target_properties(aws_auth_minimal
+                          PROPERTIES IMPORTED_LOCATION
+                                     "${AWS_AUTH_LIB_DIR}/libaws-c-auth.a"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${AWS_AUTH_PREFIX}/include"
+                                     INTERFACE_LINK_LIBRARIES
+                                     "${AWS_AUTH_RUNTIME_LIBS};${AWS_AUTH_PLATFORM_LIBS}")
+    add_dependencies(aws_auth_minimal aws-c-auth_ep)
+
+    set(AWS_AUTH_INCLUDE_DIR
+        "${AWS_AUTH_PREFIX}/include"
+        PARENT_SCOPE)
+    set(AWS_AUTH_LIB_DIR
+        "${AWS_AUTH_LIB_DIR}"
+        PARENT_SCOPE)
+endfunction()
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index 255a89ab..32d7b1d1 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -1913,6 +1913,10 @@ if(PAIMON_ENABLE_JINDO)
     build_jindosdk_c()
     build_jindosdk_nextarch()
 endif()
+if(PAIMON_ENABLE_S3)
+    include(BuildAwsAuth)
+    build_aws_auth()
+endif()
 if(PAIMON_ENABLE_LUMINA)
     build_lumina()
 endif()
diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff
index ae775209..c50b00c1 100644
--- a/cmake_modules/arrow.diff
+++ b/cmake_modules/arrow.diff
@@ -415,6 +415,14 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/Thi
  # and crosscompiling emulator (for try_run() )
  if(CMAKE_CROSSCOMPILING_EMULATOR)
    string(REPLACE ";" ${EP_LIST_SEPARATOR} EP_CMAKE_CROSSCOMPILING_EMULATOR
+@@ -1720,6 +1725,7 @@ macro(build_thrift)
+       -DWITH_JAVASCRIPT=OFF
+       -DWITH_LIBEVENT=OFF
+       -DWITH_NODEJS=OFF
++      -DWITH_OPENSSL=OFF
+       -DWITH_PYTHON=OFF
+       -DWITH_QT5=OFF
+       -DWITH_ZLIB=OFF)
 diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
 --- a/cpp/cmake_modules/BuildUtils.cmake
 +++ b/cpp/cmake_modules/BuildUtils.cmake
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 46521534..d56700b8 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -180,6 +180,12 @@ set(PAIMON_COMMON_SRCS
     common/utils/status.cpp
     common/utils/string_utils.cpp)
 
+if(PAIMON_ENABLE_S3)
+    list(APPEND PAIMON_COMMON_SRCS common/fs/http_client.cpp
+         common/fs/object_store_file_system.cpp)
+    set(PAIMON_OBJECT_STORE_LINK_LIBS CURL::libcurl)
+endif()
+
 set(PAIMON_CORE_SRCS
     core/disk/file_io_channel.cpp
     core/mergetree/spill_reader.cpp
@@ -412,6 +418,7 @@ add_paimon_lib(paimon
                xxhash
                Threads::Threads
                RapidJSON
+               ${PAIMON_OBJECT_STORE_LINK_LIBS}
                STATIC_LINK_LIBS
                arrow
                tbb
@@ -421,6 +428,7 @@ add_paimon_lib(paimon
                xxhash
                Threads::Threads
                RapidJSON
+               ${PAIMON_OBJECT_STORE_LINK_LIBS}
                SHARED_LINK_FLAGS
                ${PAIMON_VERSION_SCRIPT_FLAGS})
 
@@ -840,10 +848,17 @@ if(PAIMON_BUILD_TESTS)
              ${PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS})
     endif()
 
+    set(PAIMON_OBJECT_STORE_FS_TEST_SOURCES)
+    if(PAIMON_ENABLE_S3)
+        list(APPEND PAIMON_OBJECT_STORE_FS_TEST_SOURCES
+             common/fs/object_store_file_system_test.cpp)
+    endif()
+
     add_paimon_test(fs_test
                     SOURCES
                     common/fs/file_system_test.cpp
                     common/fs/resolving_file_system_test.cpp
+                    ${PAIMON_OBJECT_STORE_FS_TEST_SOURCES}
                     fs/local/local_file_test.cpp
                     ${PAIMON_JINDO_FS_TEST_SOURCES}
                     STATIC_LINK_LIBS
diff --git a/src/paimon/common/fs/http_client.cpp b/src/paimon/common/fs/http_client.cpp
new file mode 100644
index 00000000..3ff589b4
--- /dev/null
+++ b/src/paimon/common/fs/http_client.cpp
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/fs/http_client.h"
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "fmt/format.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/common/utils/string_utils.h"
+
+namespace paimon {
+
+namespace {
+
+constexpr int32_t kMaxAttempts = 3;
+
+class CurlGlobalGuard {
+ public:
+    CurlGlobalGuard() {
+        curl_global_init(CURL_GLOBAL_DEFAULT);
+    }
+    ~CurlGlobalGuard() {
+        curl_global_cleanup();
+    }
+};
+
+std::shared_ptr GetCurlGlobalGuard() {
+    static auto guard = std::make_shared();
+    return guard;
+}
+
+void TrimHttpWhitespace(std::string* value) {
+    constexpr char kHttpWhitespace[] = " \t\r\n";
+    size_t begin = value->find_first_not_of(kHttpWhitespace);
+    if (begin == std::string::npos) {
+        value->clear();
+        return;
+    }
+    size_t end = value->find_last_not_of(kHttpWhitespace);
+    value->erase(end + 1);
+    value->erase(0, begin);
+}
+
+struct TransferContext {
+    CURL* handle;
+    const HttpBodyConsumer* consumer;
+    HttpResponse response;
+    Status status = Status::OK();
+};
+
+size_t WriteCallback(char* data, size_t size, size_t count, void* user_data) {
+    auto* context = static_cast(user_data);
+    size_t bytes = size * count;
+    if (!context->status.ok()) {
+        return 0;
+    }
+    long status_code = 0;  // NOLINT(runtime/int, google-runtime-int): required by curl.
+    curl_easy_getinfo(context->handle, CURLINFO_RESPONSE_CODE, &status_code);
+    context->response.status_code = static_cast(status_code);
+    if (status_code >= 300) {
+        context->response.body_size += static_cast(bytes);
+        return bytes;
+    }
+    context->status = (*context->consumer)(data, static_cast(bytes));
+    if (!context->status.ok()) {
+        return 0;
+    }
+    context->response.body_size += static_cast(bytes);
+    return bytes;
+}
+
+size_t HeaderCallback(char* data, size_t size, size_t count, void* user_data) {
+    auto* context = static_cast(user_data);
+    size_t bytes = size * count;
+    std::string line(data, bytes);
+    size_t colon = line.find(':');
+    if (colon != std::string::npos) {
+        std::string name = line.substr(0, colon);
+        TrimHttpWhitespace(&name);
+        name = StringUtils::ToLowerCase(name);
+        std::string value = line.substr(colon + 1);
+        TrimHttpWhitespace(&value);
+        context->response.headers[name] = std::move(value);
+    }
+    return bytes;
+}
+
+bool IsRetryable(CURLcode code, int64_t status_code) {
+    if (code == CURLE_COULDNT_RESOLVE_HOST || code == CURLE_COULDNT_CONNECT ||
+        code == CURLE_OPERATION_TIMEDOUT || code == CURLE_SEND_ERROR || code == CURLE_RECV_ERROR ||
+        code == CURLE_PARTIAL_FILE) {
+        return true;
+    }
+    return status_code == 429 || status_code >= 500;
+}
+
+}  // namespace
+
+class CurlHttpClient::Impl {
+ public:
+    Impl() : guard_(GetCurlGlobalGuard()) {}
+
+    ~Impl() {
+        for (CURL* handle : handles_) {
+            curl_easy_cleanup(handle);
+        }
+    }
+
+    CURL* Acquire() const {
+        std::scoped_lock lock(mutex_);
+        if (handles_.empty()) {
+            return curl_easy_init();
+        }
+        CURL* handle = handles_.back();
+        handles_.pop_back();
+        return handle;
+    }
+
+    void Release(CURL* handle) const {
+        curl_easy_reset(handle);
+        std::scoped_lock lock(mutex_);
+        handles_.push_back(handle);
+    }
+
+ private:
+    std::shared_ptr guard_;
+    mutable std::mutex mutex_;
+    mutable std::vector handles_;
+};
+
+CurlHttpClient::CurlHttpClient() : impl_(std::make_unique()) {}
+CurlHttpClient::~CurlHttpClient() = default;
+
+Result CurlHttpClient::Execute(const HttpRequest& request,
+                                             const HttpBodyConsumer& consumer) const {
+    for (int32_t attempt = 0; attempt < kMaxAttempts; ++attempt) {
+        CURL* handle = impl_->Acquire();
+        if (handle == nullptr) {
+            return Status::IOError("failed to create curl easy handle");
+        }
+        ScopeGuard release_handle([this, handle] { impl_->Release(handle); });
+        TransferContext context{handle, &consumer, {}, Status::OK()};
+        curl_slist* headers = nullptr;
+        ScopeGuard release_headers([&headers] { curl_slist_free_all(headers); });
+        for (const auto& [name, value] : request.headers) {
+            curl_slist* updated_headers = curl_slist_append(headers, (name + ": " + value).c_str());
+            if (updated_headers == nullptr) {
+                return Status::IOError("failed to create HTTP headers");
+            }
+            headers = updated_headers;
+        }
+        curl_easy_setopt(handle, CURLOPT_URL, request.url.c_str());
+        curl_easy_setopt(handle, CURLOPT_HTTPHEADER, headers);
+        curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L);
+        curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 30000L);
+        curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, WriteCallback);
+        curl_easy_setopt(handle, CURLOPT_WRITEDATA, &context);
+        curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, HeaderCallback);
+        curl_easy_setopt(handle, CURLOPT_HEADERDATA, &context);
+        curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 0L);
+        if (request.method == HttpMethod::HEAD) {
+            curl_easy_setopt(handle, CURLOPT_NOBODY, 1L);
+        }
+        CURLcode code = curl_easy_perform(handle);
+        long response_code = 0;  // NOLINT(runtime/int, google-runtime-int): required by curl.
+        curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &response_code);
+        context.response.status_code = static_cast(response_code);
+
+        if (!context.status.ok()) {
+            return context.status;
+        }
+        if (code == CURLE_OK && !IsRetryable(code, response_code)) {
+            return context.response;
+        }
+        if (!IsRetryable(code, response_code) ||
+            (context.response.body_size > 0 && response_code < 300) ||
+            attempt + 1 == kMaxAttempts) {
+            if (code == CURLE_OK) {
+                return Status::IOError(fmt::format("HTTP request to {} returned status {}",
+                                                   request.url, response_code));
+            }
+            return Status::IOError(fmt::format("HTTP request to {} failed: {} (status {})",
+                                               request.url, curl_easy_strerror(code),
+                                               response_code));
+        }
+        std::this_thread::sleep_for(std::chrono::milliseconds(100 * (1 << attempt)));
+    }
+    return Status::IOError("HTTP request failed");
+}
+
+}  // namespace paimon
diff --git a/src/paimon/common/fs/http_client.h b/src/paimon/common/fs/http_client.h
new file mode 100644
index 00000000..5dd8d029
--- /dev/null
+++ b/src/paimon/common/fs/http_client.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/result.h"
+#include "paimon/visibility.h"
+
+namespace paimon {
+
+enum class HttpMethod { HEAD, GET };
+using HttpHeaders = std::map;
+using HttpBodyConsumer = std::function;
+
+struct HttpRequest {
+    HttpMethod method = HttpMethod::GET;
+    std::string url;
+    HttpHeaders headers;
+};
+
+struct HttpResponse {
+    int32_t status_code = 0;
+    HttpHeaders headers;
+    int64_t body_size = 0;
+};
+
+class PAIMON_EXPORT HttpClient {
+ public:
+    virtual ~HttpClient() = default;
+    virtual Result Execute(const HttpRequest& request,
+                                         const HttpBodyConsumer& consumer) const = 0;
+};
+
+class PAIMON_EXPORT CurlHttpClient : public HttpClient {
+ public:
+    CurlHttpClient();
+    ~CurlHttpClient() override;
+
+    Result Execute(const HttpRequest& request,
+                                 const HttpBodyConsumer& consumer) const override;
+
+ private:
+    class Impl;
+    std::unique_ptr impl_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/fs/object_store_file_system.cpp b/src/paimon/common/fs/object_store_file_system.cpp
new file mode 100644
index 00000000..2eb2b9be
--- /dev/null
+++ b/src/paimon/common/fs/object_store_file_system.cpp
@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/fs/object_store_file_system.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "fmt/format.h"
+#include "paimon/common/utils/math.h"
+#include "paimon/common/utils/path_util.h"
+
+namespace paimon {
+namespace {
+
+constexpr int64_t kMaxReadAheadMemory = 64LL * 1024LL * 1024LL;
+constexpr int64_t kInitialReadAheadSize = 64LL * 1024LL;
+constexpr int64_t kMaxReadAheadSize = 8LL * 1024LL * 1024LL;
+
+std::string NormalizeDirectoryPrefix(const std::string& key) {
+    if (key.empty() || key.back() == '/') {
+        return key;
+    }
+    return key + "/";
+}
+
+class ObjectStoreBasicFileStatus : public BasicFileStatus {
+ public:
+    ObjectStoreBasicFileStatus(std::string path, bool is_dir)
+        : path_(std::move(path)), is_dir_(is_dir) {}
+    bool IsDir() const override {
+        return is_dir_;
+    }
+    std::string GetPath() const override {
+        return path_;
+    }
+
+ private:
+    std::string path_;
+    bool is_dir_;
+};
+
+class ObjectStoreFileStatus : public FileStatus {
+ public:
+    ObjectStoreFileStatus(std::string path, int64_t size, int64_t modification_time, bool is_dir)
+        : path_(std::move(path)),
+          size_(size),
+          modification_time_(modification_time),
+          is_dir_(is_dir) {}
+    int64_t GetLen() const override {
+        return size_;
+    }
+    bool IsDir() const override {
+        return is_dir_;
+    }
+    std::string GetPath() const override {
+        return path_;
+    }
+    int64_t GetModificationTime() const override {
+        return modification_time_;
+    }
+
+ private:
+    std::string path_;
+    int64_t size_;
+    int64_t modification_time_;
+    bool is_dir_;
+};
+
+class ObjectStoreInputStream : public InputStream {
+ public:
+    ObjectStoreInputStream(std::shared_ptr client,
+                           std::shared_ptr limiter, ObjectStorePath path,
+                           std::string uri, int64_t length)
+        : client_(std::move(client)),
+          limiter_(std::move(limiter)),
+          path_(std::move(path)),
+          uri_(std::move(uri)),
+          length_(length) {}
+
+    ~ObjectStoreInputStream() override {
+        CloseInternal();
+    }
+
+    Status Seek(int64_t offset, SeekOrigin origin) override {
+        std::scoped_lock lock(mutex_);
+        if (closed_) {
+            return Status::IOError(fmt::format("{} is closed", uri_));
+        }
+        int64_t base;
+        if (origin == FS_SEEK_SET) {
+            base = 0;
+        } else if (origin == FS_SEEK_CUR) {
+            base = position_;
+        } else if (origin == FS_SEEK_END) {
+            base = length_;
+        } else {
+            return Status::Invalid("unsupported seek origin");
+        }
+        if ((offset > 0 && base > std::numeric_limits::max() - offset) ||
+            (offset < 0 && base < std::numeric_limits::min() - offset)) {
+            return Status::Invalid("object store input position overflows");
+        }
+        int64_t position = base + offset;
+        PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(position, "object store input position"));
+        position_ = position;
+        return Status::OK();
+    }
+
+    Result GetPos() const override {
+        std::scoped_lock lock(mutex_);
+        if (closed_) {
+            return Status::IOError(fmt::format("{} is closed", uri_));
+        }
+        return position_;
+    }
+
+    Result Read(char* buffer, int64_t size) override {
+        std::scoped_lock read_lock(read_mutex_);
+        int64_t offset;
+        {
+            std::scoped_lock lock(mutex_);
+            if (closed_) {
+                return Status::IOError(fmt::format("{} is closed", uri_));
+            }
+            offset = position_;
+        }
+        PAIMON_ASSIGN_OR_RAISE(int64_t bytes_read, ReadWithReadAhead(buffer, size, offset));
+        {
+            std::scoped_lock lock(mutex_);
+            position_ += bytes_read;
+        }
+        return bytes_read;
+    }
+
+    Result Read(char* buffer, int64_t size, int64_t offset) override {
+        std::scoped_lock read_lock(read_mutex_);
+        return ReadWithReadAhead(buffer, size, offset);
+    }
+
+    void ReadAsync(char* buffer, int64_t size, int64_t offset,
+                   std::function&& callback) override {
+        Status status = ValidateRead(buffer, size, offset);
+        if (!status.ok()) {
+            callback(std::move(status));
+            return;
+        }
+        if (offset > length_ || size > length_ - offset) {
+            callback(Status::Invalid(
+                fmt::format("object store async read size {} at offset {} exceeds length {}", size,
+                            offset, length_)));
+            return;
+        }
+        if (size == 0) {
+            callback(Status::OK());
+            return;
+        }
+        client_->GetObjectRangeAsync(path_, offset, size, buffer,
+                                     [callback = std::move(callback)](Status status) mutable {
+                                         if (!status.ok()) {
+                                             callback(std::move(status));
+                                             return;
+                                         }
+                                         callback(Status::OK());
+                                     });
+    }
+
+    Status Close() override {
+        CloseInternal();
+        return Status::OK();
+    }
+    Result GetUri() const override {
+        return uri_;
+    }
+    Result Length() const override {
+        return length_;
+    }
+
+ private:
+    Status ValidateRead(char* buffer, int64_t size, int64_t offset) const {
+        PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(size, "read length"));
+        PAIMON_RETURN_NOT_OK(ValidateValueNonNegative(offset, "read offset"));
+        {
+            std::scoped_lock lock(mutex_);
+            if (closed_) {
+                return Status::IOError(fmt::format("{} is closed", uri_));
+            }
+        }
+        if (size > 0 && buffer == nullptr) {
+            return Status::Invalid("read buffer is null");
+        }
+        return Status::OK();
+    }
+
+    void EvictReadAhead() {
+        int64_t reserved = 0;
+        {
+            std::scoped_lock lock(mutex_);
+            std::vector().swap(read_ahead_buffer_);
+            reserved = reserved_read_ahead_size_;
+            reserved_read_ahead_size_ = 0;
+        }
+        limiter_->Release(reserved);
+    }
+
+    Result ReadWithReadAhead(char* buffer, int64_t size, int64_t offset) {
+        PAIMON_RETURN_NOT_OK(ValidateRead(buffer, size, offset));
+        if (size == 0 || offset >= length_) {
+            return 0;
+        }
+        int64_t read_size = std::min(size, length_ - offset);
+        bool continues_previous = false;
+        {
+            std::scoped_lock lock(mutex_);
+            int64_t relative = offset - read_ahead_offset_;
+            if (relative >= 0 && relative <= static_cast(read_ahead_buffer_.size()) &&
+                read_size <= static_cast(read_ahead_buffer_.size()) - relative) {
+                std::memcpy(buffer, read_ahead_buffer_.data() + relative,
+                            static_cast(read_size));
+                return read_size;
+            }
+            continues_previous =
+                !read_ahead_buffer_.empty() &&
+                offset == read_ahead_offset_ + static_cast(read_ahead_buffer_.size());
+        }
+        EvictReadAhead();
+        if (read_size >= kMaxReadAheadSize) {
+            next_read_ahead_size_ = kInitialReadAheadSize;
+            return client_->GetObjectRange(path_, offset, read_size, buffer);
+        }
+        next_read_ahead_size_ = continues_previous
+                                    ? std::min(kMaxReadAheadSize, next_read_ahead_size_ * 2)
+                                    : kInitialReadAheadSize;
+        int64_t fetch_size = std::min(std::max(read_size, next_read_ahead_size_), length_ - offset);
+        fetch_size = limiter_->ReserveUpTo(read_size, fetch_size);
+        if (fetch_size == 0) {
+            return client_->GetObjectRange(path_, offset, read_size, buffer);
+        }
+        std::vector fetched(static_cast(fetch_size));
+        Result result = client_->GetObjectRange(path_, offset, fetch_size, fetched.data());
+        if (!result.ok()) {
+            limiter_->Release(fetch_size);
+            return result.status();
+        }
+        int64_t bytes_read = std::move(result).value();
+        if (bytes_read != fetch_size) {
+            limiter_->Release(fetch_size);
+            return Status::IOError(fmt::format("range read for {} returned {} bytes, expected {}",
+                                               uri_, bytes_read, fetch_size));
+        }
+        {
+            std::scoped_lock lock(mutex_);
+            if (closed_) {
+                limiter_->Release(fetch_size);
+                return Status::IOError(fmt::format("{} is closed", uri_));
+            }
+            read_ahead_offset_ = offset;
+            read_ahead_buffer_ = std::move(fetched);
+            reserved_read_ahead_size_ = fetch_size;
+            std::memcpy(buffer, read_ahead_buffer_.data(), static_cast(read_size));
+        }
+        return read_size;
+    }
+
+    void CloseInternal() {
+        int64_t reserved = 0;
+        {
+            std::scoped_lock lock(mutex_);
+            closed_ = true;
+            std::vector().swap(read_ahead_buffer_);
+            reserved = reserved_read_ahead_size_;
+            reserved_read_ahead_size_ = 0;
+        }
+        limiter_->Release(reserved);
+    }
+
+    std::shared_ptr client_;
+    std::shared_ptr limiter_;
+    ObjectStorePath path_;
+    std::string uri_;
+    int64_t length_;
+    int64_t position_ = 0;
+    int64_t read_ahead_offset_ = 0;
+    int64_t next_read_ahead_size_ = kInitialReadAheadSize;
+    int64_t reserved_read_ahead_size_ = 0;
+    std::vector read_ahead_buffer_;
+    bool closed_ = false;
+    mutable std::mutex read_mutex_;
+    mutable std::mutex mutex_;
+};
+
+}  // namespace
+
+ReadAheadMemoryLimiter::ReadAheadMemoryLimiter(int64_t limit) : limit_(limit) {}
+
+int64_t ReadAheadMemoryLimiter::ReserveUpTo(int64_t min_size, int64_t max_size) {
+    std::scoped_lock lock(mutex_);
+    int64_t available = limit_ - used_;
+    if (available < min_size) {
+        return 0;
+    }
+    int64_t size = std::min(max_size, available);
+    used_ += size;
+    return size;
+}
+
+void ReadAheadMemoryLimiter::Release(int64_t size) {
+    std::scoped_lock lock(mutex_);
+    assert(size >= 0 && size <= used_);
+    used_ -= std::clamp(size, int64_t{0}, used_);
+}
+
+ObjectStoreFileSystem::ObjectStoreFileSystem(std::string scheme,
+                                             std::shared_ptr client)
+    : ObjectStoreFileSystem(std::move(scheme), std::move(client), kMaxReadAheadMemory) {}
+
+ObjectStoreFileSystem::ObjectStoreFileSystem(std::string scheme,
+                                             std::shared_ptr client,
+                                             int64_t read_ahead_memory_limit)
+    : scheme_(std::move(scheme)),
+      client_(std::move(client)),
+      read_ahead_limiter_(std::make_shared(read_ahead_memory_limit)) {}
+
+Result ObjectStoreFileSystem::ParsePath(const std::string& path) const {
+    PAIMON_ASSIGN_OR_RAISE(Path parsed, PathUtil::ToPath(path));
+    if (parsed.scheme != scheme_) {
+        return Status::Invalid(fmt::format("path must use {} scheme: {}", scheme_, path));
+    }
+    if (parsed.authority.empty()) {
+        return Status::Invalid(fmt::format("{} path must include bucket: {}", scheme_, path));
+    }
+    std::string key = parsed.path;
+    key.erase(0, key.find_first_not_of('/'));
+    return ObjectStorePath{parsed.authority, key};
+}
+
+std::string ObjectStoreFileSystem::ToUri(const ObjectStorePath& path, bool is_dir) const {
+    std::string uri = scheme_ + "://" + path.bucket + "/";
+    uri += path.key;
+    if (is_dir && uri.back() != '/') {
+        uri += "/";
+    }
+    return uri;
+}
+
+Result ObjectStoreFileSystem::DirectoryExists(const ObjectStorePath& path) const {
+    ObjectStorePath directory{path.bucket, NormalizeDirectoryPrefix(path.key)};
+    PAIMON_ASSIGN_OR_RAISE(ListObjectsResult result, client_->ListObjects(directory, "", 1));
+    return path.key.empty() || !result.objects.empty() || !result.common_prefixes.empty();
+}
+
+Result> ObjectStoreFileSystem::Open(const std::string& path) const {
+    PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path));
+    if (object_path.key.empty()) {
+        return Status::Invalid(fmt::format("{} is a directory", path));
+    }
+    Result metadata = client_->HeadObject(object_path);
+    if (!metadata.ok()) {
+        if (!metadata.status().IsNotExist()) {
+            return metadata.status();
+        }
+        PAIMON_ASSIGN_OR_RAISE(bool is_dir, DirectoryExists(object_path));
+        if (is_dir) {
+            return Status::Invalid(fmt::format("{} is a directory", path));
+        }
+        return metadata.status();
+    }
+    return std::make_unique(client_, read_ahead_limiter_, object_path,
+                                                    ToUri(object_path), metadata.value().size);
+}
+
+Result> ObjectStoreFileSystem::GetFileStatus(
+    const std::string& path) const {
+    PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path));
+    if (!object_path.key.empty()) {
+        Result metadata = client_->HeadObject(object_path);
+        if (metadata.ok()) {
+            return std::make_unique(
+                ToUri(object_path), metadata.value().size, metadata.value().modification_time,
+                false);
+        }
+        if (!metadata.status().IsNotExist()) {
+            return metadata.status();
+        }
+    }
+    PAIMON_ASSIGN_OR_RAISE(bool exists, DirectoryExists(object_path));
+    if (!exists) {
+        return Status::NotExist(fmt::format("{} does not exist", path));
+    }
+    return std::make_unique(ToUri(object_path, true), 0, 0, true);
+}
+
+Status ObjectStoreFileSystem::ListDirectory(
+    const ObjectStorePath& path, std::vector>* basic_statuses,
+    std::vector>* statuses) const {
+    if (basic_statuses == nullptr && statuses == nullptr) {
+        return Status::Invalid("a destination status list is required");
+    }
+    ObjectStorePath directory{path.bucket, NormalizeDirectoryPrefix(path.key)};
+    std::string token;
+    do {
+        PAIMON_ASSIGN_OR_RAISE(ListObjectsResult result, client_->ListObjects(directory, token, 0));
+        if (result.is_truncated && result.continuation_token.empty()) {
+            return Status::IOError(
+                fmt::format("truncated listing for {} did not include a continuation token",
+                            ToUri(directory, true)));
+        }
+        for (const auto& object : result.objects) {
+            if (object.key == directory.key) {
+                continue;
+            }
+            ObjectStorePath child{path.bucket, object.key};
+            if (basic_statuses) {
+                basic_statuses->push_back(
+                    std::make_unique(ToUri(child), false));
+            } else {
+                statuses->push_back(std::make_unique(
+                    ToUri(child), object.size, object.modification_time, false));
+            }
+        }
+        for (const auto& prefix : result.common_prefixes) {
+            ObjectStorePath child{path.bucket, prefix};
+            if (basic_statuses) {
+                basic_statuses->push_back(
+                    std::make_unique(ToUri(child, true), true));
+            } else {
+                statuses->push_back(
+                    std::make_unique(ToUri(child, true), 0, 0, true));
+            }
+        }
+        token = result.continuation_token;
+        if (!result.is_truncated) {
+            break;
+        }
+    } while (true);
+    return Status::OK();
+}
+
+Status ObjectStoreFileSystem::ListDir(
+    const std::string& directory,
+    std::vector>* file_status_list) const {
+    PAIMON_ASSIGN_OR_RAISE(ObjectStorePath path, ParsePath(directory));
+    if (!path.key.empty() && path.key.back() != '/') {
+        Result metadata = client_->HeadObject(path);
+        if (metadata.ok()) {
+            return Status::Invalid(fmt::format("file {} exists and is not a directory", directory));
+        }
+        if (!metadata.status().IsNotExist()) {
+            return metadata.status();
+        }
+    }
+    return ListDirectory(path, file_status_list, nullptr);
+}
+
+Status ObjectStoreFileSystem::ListFileStatus(
+    const std::string& path, std::vector>* file_status_list) const {
+    PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path));
+    if (!object_path.key.empty() && object_path.key.back() != '/') {
+        Result metadata = client_->HeadObject(object_path);
+        if (metadata.ok()) {
+            file_status_list->push_back(
+                std::make_unique(ToUri(object_path), metadata.value().size,
+                                                        metadata.value().modification_time, false));
+            return Status::OK();
+        }
+        if (!metadata.status().IsNotExist()) {
+            return metadata.status();
+        }
+    }
+    return ListDirectory(object_path, nullptr, file_status_list);
+}
+
+Result ObjectStoreFileSystem::Exists(const std::string& path) const {
+    PAIMON_ASSIGN_OR_RAISE(ObjectStorePath object_path, ParsePath(path));
+    if (!object_path.key.empty()) {
+        Result metadata = client_->HeadObject(object_path);
+        if (metadata.ok()) {
+            return true;
+        }
+        if (!metadata.status().IsNotExist()) {
+            return metadata.status();
+        }
+    }
+    return DirectoryExists(object_path);
+}
+
+Status ObjectStoreFileSystem::ReadOnlyStatus() const {
+    return Status::NotImplemented(fmt::format("{} object store file system is read-only", scheme_));
+}
+
+Result> ObjectStoreFileSystem::Create(const std::string&,
+                                                                    bool) const {
+    return ReadOnlyStatus();
+}
+
+Status ObjectStoreFileSystem::Mkdirs(const std::string&) const {
+    return ReadOnlyStatus();
+}
+
+Status ObjectStoreFileSystem::Rename(const std::string&, const std::string&) const {
+    return ReadOnlyStatus();
+}
+
+Status ObjectStoreFileSystem::Delete(const std::string&, bool) const {
+    return ReadOnlyStatus();
+}
+
+Status ObjectStoreFileSystem::WriteFile(const std::string&, const std::string&, bool) {
+    return ReadOnlyStatus();
+}
+
+Status ObjectStoreFileSystem::AtomicStore(const std::string&, const std::string&) {
+    return ReadOnlyStatus();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/common/fs/object_store_file_system.h b/src/paimon/common/fs/object_store_file_system.h
new file mode 100644
index 00000000..9ef30528
--- /dev/null
+++ b/src/paimon/common/fs/object_store_file_system.h
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/fs/file_system.h"
+#include "paimon/visibility.h"
+
+namespace paimon {
+
+struct ObjectStorePath {
+    std::string bucket;
+    std::string key;
+};
+
+struct ObjectMetadata {
+    std::string key;
+    int64_t size = 0;
+    int64_t modification_time = 0;
+};
+
+struct ListObjectsResult {
+    std::vector objects;
+    std::vector common_prefixes;
+    std::string continuation_token;
+    bool is_truncated = false;
+};
+
+class PAIMON_EXPORT ObjectStoreClient {
+ public:
+    virtual ~ObjectStoreClient() = default;
+
+    virtual Result HeadObject(const ObjectStorePath& path) const = 0;
+    virtual Result ListObjects(const ObjectStorePath& path,
+                                                  const std::string& continuation_token,
+                                                  int32_t max_keys) const = 0;
+    virtual Result GetObjectRange(const ObjectStorePath& path, int64_t offset,
+                                           int64_t size, char* buffer) const = 0;
+    virtual void GetObjectRangeAsync(const ObjectStorePath& path, int64_t offset, int64_t size,
+                                     char* buffer,
+                                     std::function&& callback) const = 0;
+};
+
+class ReadAheadMemoryLimiter {
+ public:
+    explicit ReadAheadMemoryLimiter(int64_t limit);
+
+    int64_t ReserveUpTo(int64_t min_size, int64_t max_size);
+    void Release(int64_t size);
+
+ private:
+    int64_t limit_;
+    int64_t used_ = 0;
+    std::mutex mutex_;
+};
+
+class PAIMON_EXPORT ObjectStoreFileSystem : public FileSystem {
+ public:
+    ObjectStoreFileSystem(std::string scheme, std::shared_ptr client);
+    ObjectStoreFileSystem(std::string scheme, std::shared_ptr client,
+                          int64_t read_ahead_memory_limit);
+    ~ObjectStoreFileSystem() override = default;
+
+    Result> Open(const std::string& path) const override;
+    Result> GetFileStatus(const std::string& path) const override;
+    Status ListDir(const std::string& directory,
+                   std::vector>* file_status_list) const override;
+    Status ListFileStatus(
+        const std::string& path,
+        std::vector>* file_status_list) const override;
+    Result Exists(const std::string& path) const override;
+
+    Result> Create(const std::string& path,
+                                                 bool overwrite) const override;
+    Status Mkdirs(const std::string& path) const override;
+    Status Rename(const std::string& src, const std::string& dst) const override;
+    Status Delete(const std::string& path, bool recursive = true) const override;
+    Status WriteFile(const std::string& path, const std::string& content, bool overwrite) override;
+    Status AtomicStore(const std::string& path, const std::string& content) override;
+
+ protected:
+    Result ParsePath(const std::string& path) const;
+    std::string ToUri(const ObjectStorePath& path, bool is_dir = false) const;
+
+ private:
+    Result DirectoryExists(const ObjectStorePath& path) const;
+    Status ListDirectory(const ObjectStorePath& path,
+                         std::vector>* basic_statuses,
+                         std::vector>* statuses) const;
+    Status ReadOnlyStatus() const;
+
+    std::string scheme_;
+    std::shared_ptr client_;
+    std::shared_ptr read_ahead_limiter_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/fs/object_store_file_system_test.cpp b/src/paimon/common/fs/object_store_file_system_test.cpp
new file mode 100644
index 00000000..1b34917a
--- /dev/null
+++ b/src/paimon/common/fs/object_store_file_system_test.cpp
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/fs/object_store_file_system.h"
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+namespace {
+
+using Range = std::pair;
+
+class MockObjectStoreClient : public ObjectStoreClient {
+ public:
+    Result HeadObject(const ObjectStorePath& path) const override {
+        ++head_calls_;
+        if (!head_error_.ok()) {
+            return head_error_;
+        }
+        auto iter = objects_.find(path.key);
+        if (iter == objects_.end()) {
+            return Status::NotExist("not found");
+        }
+        return ObjectMetadata{path.key, static_cast(iter->second.size()), 1};
+    }
+
+    Result ListObjects(const ObjectStorePath& path, const std::string& token,
+                                          int32_t) const override {
+        ++list_calls_;
+        if (!list_error_.ok()) {
+            return list_error_;
+        }
+        if (!pages_.empty()) {
+            return pages_.at(token);
+        }
+        ListObjectsResult result;
+        for (const auto& [key, value] : objects_) {
+            if (key.rfind(path.key, 0) == 0) {
+                result.objects.push_back(
+                    ObjectMetadata{key, static_cast(value.size()), 1});
+            }
+        }
+        return result;
+    }
+
+    Result GetObjectRange(const ObjectStorePath& path, int64_t offset, int64_t size,
+                                   char* buffer) const override {
+        ranges_.emplace_back(offset, size);
+        const std::string& value = objects_.at(path.key);
+        int64_t available = std::min(size, static_cast(value.size()) - offset);
+        if (short_read_) {
+            --available;
+        }
+        std::memcpy(buffer, value.data() + offset, available);
+        return available;
+    }
+
+    void GetObjectRangeAsync(const ObjectStorePath& path, int64_t offset, int64_t size,
+                             char* buffer, std::function&& callback) const override {
+        Result result = GetObjectRange(path, offset, size, buffer);
+        callback(result.ok() && result.value() == size ? Status::OK()
+                                                       : Status::IOError("short read"));
+    }
+
+    std::map objects_;
+    std::map pages_;
+    Status head_error_ = Status::OK();
+    Status list_error_ = Status::OK();
+    mutable int32_t list_calls_ = 0;
+    mutable int32_t head_calls_ = 0;
+    mutable std::vector ranges_;
+    bool short_read_ = false;
+};
+
+TEST(ObjectStoreFileSystemTest, TestObjectWinsOverPrefix) {
+    auto client = std::make_shared();
+    client->objects_["foo"] = "file";
+    client->objects_["foo/bar"] = "child";
+    ObjectStoreFileSystem fs("s3", client);
+    ASSERT_OK_AND_ASSIGN(auto status, fs.GetFileStatus("s3://bucket/foo"));
+    ASSERT_FALSE(status->IsDir());
+    std::vector> statuses;
+    ASSERT_OK(fs.ListFileStatus("s3://bucket/foo", &statuses));
+    ASSERT_EQ(statuses.size(), 1);
+    ASSERT_FALSE(statuses[0]->IsDir());
+}
+
+TEST(ObjectStoreFileSystemTest, TestHeadErrorIsNotMasked) {
+    auto client = std::make_shared();
+    client->head_error_ = Status::IOError("access denied");
+    client->objects_["foo/bar"] = "child";
+    ObjectStoreFileSystem fs("s3", client);
+    auto status = fs.GetFileStatus("s3://bucket/foo");
+    ASSERT_TRUE(status.status().IsIOError());
+    ASSERT_EQ(client->list_calls_, 0);
+}
+
+TEST(ObjectStoreFileSystemTest, TestPaginationAndDirectoryMarker) {
+    auto client = std::make_shared();
+    ListObjectsResult first;
+    first.objects.push_back({"dir/", 0, 0});
+    first.objects.push_back({"dir/a", 1, 1});
+    first.is_truncated = true;
+    first.continuation_token = "next";
+    ListObjectsResult second;
+    second.common_prefixes.push_back("dir/sub/");
+    client->pages_[""] = first;
+    client->pages_["next"] = second;
+    ObjectStoreFileSystem fs("s3", client);
+    std::vector> statuses;
+    ASSERT_OK(fs.ListFileStatus("s3://bucket/dir/", &statuses));
+    ASSERT_EQ(statuses.size(), 2);
+    ASSERT_EQ(statuses[0]->GetPath(), "s3://bucket/dir/a");
+    ASSERT_TRUE(statuses[1]->IsDir());
+}
+
+TEST(ObjectStoreFileSystemTest, TestTruncatedPageRequiresContinuationToken) {
+    auto client = std::make_shared();
+    ListObjectsResult page;
+    page.objects.push_back({"dir/a", 1, 1});
+    page.is_truncated = true;
+    client->pages_[""] = page;
+    ObjectStoreFileSystem fs("s3", client);
+    std::vector> statuses;
+    ASSERT_TRUE(fs.ListFileStatus("s3://bucket/dir/", &statuses).IsIOError());
+    ASSERT_TRUE(statuses.empty());
+}
+
+TEST(ObjectStoreFileSystemTest, TestOpenBucketRootIsDirectory) {
+    auto client = std::make_shared();
+    ObjectStoreFileSystem fs("s3", client);
+    ASSERT_TRUE(fs.Open("s3://bucket/").status().IsInvalid());
+    ASSERT_EQ(client->head_calls_, 0);
+    ASSERT_EQ(client->list_calls_, 0);
+}
+
+TEST(ObjectStoreFileSystemTest, TestPathWithLeadingSlashes) {
+    auto client = std::make_shared();
+    client->objects_["file"] = "data";
+    ObjectStoreFileSystem fs("s3", client);
+    ASSERT_OK_AND_ASSIGN(auto status, fs.GetFileStatus("s3://bucket///file"));
+    ASSERT_EQ(status->GetPath(), "s3://bucket/file");
+}
+
+TEST(ObjectStoreInputStreamTest, TestBoundsCloseAndSeekOverflow) {
+    auto client = std::make_shared();
+    client->objects_["file"] = std::string(128 * 1024, 'x');
+    ObjectStoreFileSystem fs("s3", client, 64 * 1024);
+    ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file"));
+    char data[4];
+    bool called = false;
+    stream->ReadAsync(data, 4, 128 * 1024 - 2, [&called](Status status) {
+        called = true;
+        ASSERT_TRUE(status.IsInvalid());
+    });
+    ASSERT_TRUE(called);
+    ASSERT_TRUE(stream->Seek(std::numeric_limits::max(), FS_SEEK_END).IsInvalid());
+    ASSERT_OK(stream->Close());
+    called = false;
+    stream->ReadAsync(data, 1, 0, [&called, &stream](Status status) {
+        called = true;
+        ASSERT_TRUE(status.IsIOError());
+        ASSERT_TRUE(stream->GetPos().status().IsIOError());
+    });
+    ASSERT_TRUE(called);
+}
+
+TEST(ObjectStoreInputStreamTest, TestShortReadFails) {
+    auto client = std::make_shared();
+    client->objects_["file"] = std::string(128 * 1024, 'x');
+    client->short_read_ = true;
+    ObjectStoreFileSystem fs("s3", client);
+    ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file"));
+    char data[4];
+    ASSERT_TRUE(stream->Read(data, sizeof(data)).status().IsIOError());
+}
+
+TEST(ObjectStoreInputStreamTest, TestSequentialReadAheadGrows) {
+    auto client = std::make_shared();
+    client->objects_["file"] = std::string(1024 * 1024, 'x');
+    ObjectStoreFileSystem fs("s3", client);
+    ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file"));
+    std::array buffer{};
+    ASSERT_OK(stream->Read(buffer.data(), buffer.size()));
+    ASSERT_EQ(client->ranges_.back(), Range(0, 64 * 1024));
+    ASSERT_OK(stream->Seek(64 * 1024, FS_SEEK_SET));
+    ASSERT_OK(stream->Read(buffer.data(), buffer.size()));
+    ASSERT_EQ(client->ranges_.back(), Range(64 * 1024, 128 * 1024));
+}
+
+TEST(ObjectStoreInputStreamTest, TestConsumedBufferReleasesBudget) {
+    auto client = std::make_shared();
+    client->objects_["file"] = std::string(1024 * 1024, 'x');
+    ObjectStoreFileSystem fs("s3", client, 64 * 1024);
+    ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file"));
+    std::array buffer{};
+    for (int32_t i = 0; i < 17; ++i) {
+        ASSERT_OK(stream->Read(buffer.data(), buffer.size()));
+    }
+    ASSERT_EQ(client->ranges_.size(), 2);
+    ASSERT_EQ(client->ranges_.back(), Range(64 * 1024, 64 * 1024));
+}
+
+TEST(ObjectStoreInputStreamTest, TestBackwardSeekReleasesBudget) {
+    auto client = std::make_shared();
+    client->objects_["file"] = std::string(1024 * 1024, 'x');
+    ObjectStoreFileSystem fs("s3", client, 64 * 1024);
+    ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file"));
+    std::array buffer{};
+    ASSERT_OK(stream->Seek(64 * 1024, FS_SEEK_SET));
+    ASSERT_OK(stream->Read(buffer.data(), buffer.size()));
+    ASSERT_OK(stream->Seek(0, FS_SEEK_SET));
+    ASSERT_OK(stream->Read(buffer.data(), buffer.size()));
+    ASSERT_EQ(client->ranges_.back(), Range(0, 64 * 1024));
+}
+
+TEST(ObjectStoreInputStreamTest, TestLargeDirectReadReleasesBudget) {
+    auto client = std::make_shared();
+    client->objects_["file"] = std::string(16 * 1024 * 1024, 'x');
+    ObjectStoreFileSystem fs("s3", client, 64 * 1024);
+    ASSERT_OK_AND_ASSIGN(auto stream, fs.Open("s3://bucket/file"));
+    std::array small_buffer{};
+    ASSERT_OK(stream->Seek(64 * 1024, FS_SEEK_SET));
+    ASSERT_OK(stream->Read(small_buffer.data(), small_buffer.size()));
+    std::vector large_buffer(8 * 1024 * 1024);
+    ASSERT_OK(stream->Seek(1024 * 1024, FS_SEEK_SET));
+    ASSERT_OK(stream->Read(large_buffer.data(), large_buffer.size()));
+    ASSERT_OK(stream->Seek(0, FS_SEEK_SET));
+    ASSERT_OK(stream->Read(small_buffer.data(), small_buffer.size()));
+    ASSERT_EQ(client->ranges_.back(), Range(0, 64 * 1024));
+}
+
+TEST(ObjectStoreInputStreamTest, TestCompetingStreamsShareBudget) {
+    auto client = std::make_shared();
+    client->objects_["first"] = std::string(1024 * 1024, 'x');
+    client->objects_["second"] = std::string(1024 * 1024, 'x');
+    ObjectStoreFileSystem fs("s3", client, 64 * 1024);
+    ASSERT_OK_AND_ASSIGN(auto first, fs.Open("s3://bucket/first"));
+    ASSERT_OK_AND_ASSIGN(auto second, fs.Open("s3://bucket/second"));
+    std::array buffer{};
+    ASSERT_OK(first->Read(buffer.data(), buffer.size()));
+    ASSERT_OK(second->Read(buffer.data(), buffer.size()));
+    ASSERT_EQ(client->ranges_.back(), Range(0, 4096));
+    ASSERT_OK(first->Close());
+    ASSERT_OK(second->Seek(100000, FS_SEEK_SET));
+    ASSERT_OK(second->Read(buffer.data(), buffer.size()));
+    ASSERT_EQ(client->ranges_.back(), Range(100000, 64 * 1024));
+}
+
+}  // namespace
+}  // namespace paimon::test
diff --git a/src/paimon/fs/s3/CMakeLists.txt b/src/paimon/fs/s3/CMakeLists.txt
new file mode 100644
index 00000000..4c6a0d0e
--- /dev/null
+++ b/src/paimon/fs/s3/CMakeLists.txt
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if(PAIMON_ENABLE_S3)
+    add_paimon_lib(paimon_s3_file_system
+                   SOURCES
+                   s3_file_system.cpp
+                   s3_file_system_factory.cpp
+                   EXTRA_INCLUDES
+                   ${AWS_AUTH_INCLUDE_DIR}
+                   DEPENDENCIES
+                   paimon_shared
+                   CURL::libcurl
+                   STATIC_LINK_LIBS
+                   CURL::libcurl
+                   aws_auth_minimal
+                   fmt
+                   SHARED_LINK_LIBS
+                   paimon_shared
+                   SHARED_LINK_FLAGS
+                   ${PAIMON_VERSION_SCRIPT_FLAGS})
+
+    add_dependencies(paimon_s3_file_system_objlib aws-c-auth_ep)
+
+    if(PAIMON_BUILD_TESTS)
+        add_paimon_test(s3_file_system_test
+                        SOURCES
+                        s3_file_system_test.cpp
+                        EXTRA_INCLUDES
+                        ${AWS_AUTH_INCLUDE_DIR}
+                        STATIC_LINK_LIBS
+                        paimon_shared
+                        test_utils_static
+                        ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS}
+                        ${PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS}
+                        ${GTEST_LINK_TOOLCHAIN})
+    endif()
+endif()
diff --git a/src/paimon/fs/s3/s3_file_system.cpp b/src/paimon/fs/s3/s3_file_system.cpp
new file mode 100644
index 00000000..4d896680
--- /dev/null
+++ b/src/paimon/fs/s3/s3_file_system.cpp
@@ -0,0 +1,1000 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/fs/s3/s3_file_system.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "fmt/format.h"
+#include "paimon/common/fs/http_client.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/executor.h"
+
+namespace paimon::s3 {
+namespace {
+
+Result PercentEncode(std::string_view value, bool preserve_slash) {
+    if (value.size() > std::numeric_limits::max()) {
+        return Status::IOError("S3 URL component is too large to encode");
+    }
+    char* encoded = curl_easy_escape(nullptr, value.data(), static_cast(value.size()));
+    if (encoded == nullptr) {
+        return Status::IOError("failed to URL encode S3 component");
+    }
+    ScopeGuard free_encoded([encoded] { curl_free(encoded); });
+    std::string result(encoded);
+    if (preserve_slash) {
+        result = StringUtils::Replace(result, "%2F", "/");
+    }
+    return result;
+}
+
+Result PercentDecode(std::string_view value, const std::string& field) {
+    for (size_t position = 0; position < value.size(); ++position) {
+        if (value[position] == '%' &&
+            (position + 2 >= value.size() ||
+             !std::isxdigit(static_cast(value[position + 1])) ||
+             !std::isxdigit(static_cast(value[position + 2])))) {
+            return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field));
+        }
+    }
+    if (value.size() > std::numeric_limits::max()) {
+        return Status::IOError(fmt::format("S3 {} is too large to URL decode", field));
+    }
+    int decoded_size = 0;
+    char* decoded =
+        curl_easy_unescape(nullptr, value.data(), static_cast(value.size()), &decoded_size);
+    if (decoded == nullptr) {
+        return Status::IOError(fmt::format("failed to URL decode S3 {}", field));
+    }
+    ScopeGuard free_decoded([decoded] { curl_free(decoded); });
+    std::string result(decoded, decoded_size);
+    return result;
+}
+
+Result ParseNonNegativeInt64(const std::string& value, const std::string& field) {
+    std::optional result = StringUtils::StringToValue(value);
+    if (!result || *result < 0 || (!value.empty() && value.front() == '-')) {
+        return Status::IOError(fmt::format("S3 {} is not a non-negative integer", field));
+    }
+    return *result;
+}
+
+int64_t ParseModificationTime(const std::string& value) {
+    time_t seconds = curl_getdate(value.c_str(), nullptr);
+    return seconds == static_cast(-1) ? 0 : static_cast(seconds) * 1000;
+}
+
+std::string XmlUnescape(const std::string& value) {
+    const std::pair entities[] = {
+        {"&", "&"}, {"<", "<"}, {">", ">"}, {""", "\""}, {"'", "'"}};
+
+    std::string result;
+    result.reserve(value.size());
+    for (size_t position = 0; position < value.size();) {
+        bool matched = false;
+        if (value[position] == '&') {
+            for (const auto& [entity, replacement] : entities) {
+                size_t entity_size = std::strlen(entity);
+                if (value.compare(position, entity_size, entity) == 0) {
+                    result.append(replacement);
+                    position += entity_size;
+                    matched = true;
+                    break;
+                }
+            }
+        }
+        if (!matched) {
+            result.push_back(value[position++]);
+        }
+    }
+    return result;
+}
+
+std::optional TagValue(const std::string& xml, const std::string& tag,
+                                    size_t offset = 0) {
+    std::string begin = "<" + tag + ">";
+    std::string end = "";
+    size_t begin_position = xml.find(begin, offset);
+    if (begin_position == std::string::npos) {
+        return std::nullopt;
+    }
+    begin_position += begin.size();
+    size_t end_position = xml.find(end, begin_position);
+    if (end_position == std::string::npos) {
+        return std::nullopt;
+    }
+    return XmlUnescape(xml.substr(begin_position, end_position - begin_position));
+}
+
+Result> TagBlocks(const std::string& xml, const std::string& tag) {
+    std::vector blocks;
+    std::string begin = "<" + tag + ">";
+    std::string end = "";
+    size_t position = 0;
+    while (true) {
+        size_t begin_position = xml.find(begin, position);
+        size_t unexpected_end = xml.find(end, position);
+        if (begin_position == std::string::npos) {
+            if (unexpected_end != std::string::npos) {
+                return Status::IOError(fmt::format("malformed S3 XML element {}", tag));
+            }
+            break;
+        }
+        if (unexpected_end != std::string::npos && unexpected_end < begin_position) {
+            return Status::IOError(fmt::format("malformed S3 XML element {}", tag));
+        }
+        size_t end_position = xml.find(end, begin_position + begin.size());
+        if (end_position == std::string::npos) {
+            return Status::IOError(fmt::format("malformed S3 XML element {}", tag));
+        }
+        size_t nested_begin = xml.find(begin, begin_position + begin.size());
+        if (nested_begin != std::string::npos && nested_begin < end_position) {
+            return Status::IOError(fmt::format("malformed S3 XML element {}", tag));
+        }
+        end_position += end.size();
+        blocks.push_back(xml.substr(begin_position, end_position - begin_position));
+        position = end_position;
+    }
+    return blocks;
+}
+
+class AwsAuthRuntime {
+ public:
+    static Result> Create() {
+        std::unique_ptr runtime(new AwsAuthRuntime());
+        PAIMON_RETURN_NOT_OK(runtime->Initialize());
+        return runtime;
+    }
+
+    ~AwsAuthRuntime() {
+        if (tls_context_ != nullptr) {
+            aws_tls_ctx_release(tls_context_);
+        }
+        if (bootstrap_ != nullptr) {
+            aws_client_bootstrap_release(bootstrap_);
+        }
+        if (resolver_ != nullptr) {
+            aws_host_resolver_release(resolver_);
+        }
+        if (event_loop_group_ != nullptr) {
+            aws_event_loop_group_release(event_loop_group_);
+        }
+        if (library_initialized_) {
+            aws_auth_library_clean_up();
+        }
+    }
+
+    aws_allocator* allocator() const {
+        return allocator_;
+    }
+    aws_client_bootstrap* bootstrap() const {
+        return bootstrap_;
+    }
+    aws_tls_ctx* tls_context() const {
+        return tls_context_;
+    }
+
+ private:
+    AwsAuthRuntime() : allocator_(aws_default_allocator()) {}
+
+    Status Initialize() {
+        aws_auth_library_init(allocator_);
+        library_initialized_ = true;
+        event_loop_group_ = aws_event_loop_group_new_default(allocator_, 1, nullptr);
+        if (event_loop_group_ == nullptr) {
+            return InitializationError("event loop group");
+        }
+        aws_host_resolver_default_options resolver_options{};
+        resolver_options.el_group = event_loop_group_;
+        resolver_options.max_entries = 8;
+        resolver_ = aws_host_resolver_new_default(allocator_, &resolver_options);
+        if (resolver_ == nullptr) {
+            return InitializationError("host resolver");
+        }
+        aws_client_bootstrap_options bootstrap_options{};
+        bootstrap_options.event_loop_group = event_loop_group_;
+        bootstrap_options.host_resolver = resolver_;
+        bootstrap_ = aws_client_bootstrap_new(allocator_, &bootstrap_options);
+        if (bootstrap_ == nullptr) {
+            return InitializationError("client bootstrap");
+        }
+        aws_tls_ctx_options tls_options;
+        aws_tls_ctx_options_init_default_client(&tls_options, allocator_);
+        tls_context_ = aws_tls_client_ctx_new(allocator_, &tls_options);
+        aws_tls_ctx_options_clean_up(&tls_options);
+        if (tls_context_ == nullptr) {
+            return InitializationError("TLS context");
+        }
+        return Status::OK();
+    }
+
+    Status InitializationError(const std::string& component) const {
+        return Status::IOError(fmt::format("failed to initialize AWS {}: {}", component,
+                                           aws_error_debug_str(aws_last_error())));
+    }
+
+    aws_allocator* allocator_;
+    aws_event_loop_group* event_loop_group_ = nullptr;
+    aws_host_resolver* resolver_ = nullptr;
+    aws_client_bootstrap* bootstrap_ = nullptr;
+    aws_tls_ctx* tls_context_ = nullptr;
+    bool library_initialized_ = false;
+};
+
+Result GetAwsAuthRuntime() {
+    static const Result runtime = [] {
+        Result> created = AwsAuthRuntime::Create();
+        if (!created.ok()) {
+            return Result(created.status());
+        }
+        return Result(std::move(created).value().release());
+    }();
+    return runtime;
+}
+
+aws_byte_cursor Cursor(const std::string& value) {
+    return aws_byte_cursor_from_array(value.data(), value.size());
+}
+
+const char* CanonicalS3OptionName(const std::string& option) {
+    if (option == "access-key" || option == "access.key" || option == "accessKeyId") {
+        return kS3AccessKeyOption;
+    }
+    if (option == "secret-key" || option == "secret.key" || option == "accessKeySecret") {
+        return kS3SecretKeyOption;
+    }
+    if (option == "session.token" || option == "session-token" || option == "security.token" ||
+        option == "security-token" || option == "securityToken") {
+        return kS3SessionTokenOption;
+    }
+    if (option == "endpoint") {
+        return kS3EndpointOption;
+    }
+    if (option == "region") {
+        return kS3RegionOption;
+    }
+    if (option == "path-style-access" || option == "path.style.access") {
+        return kS3PathStyleAccessOption;
+    }
+    if (option == "profile") {
+        return kS3ProfileOption;
+    }
+    return nullptr;
+}
+
+std::map NormalizeS3Options(
+    const std::map& options) {
+    std::map normalized = options;
+    for (const auto& [key, value] : options) {
+        for (const char* prefix : {"s3a.", "fs.s3.", "fs.s3a."}) {
+            if (!StringUtils::StartsWith(key, prefix)) {
+                continue;
+            }
+            const char* canonical = CanonicalS3OptionName(key.substr(std::strlen(prefix)));
+            if (canonical != nullptr && normalized.find(canonical) == normalized.end()) {
+                normalized.emplace(canonical, value);
+            }
+            break;
+        }
+        if (StringUtils::StartsWith(key, "s3.")) {
+            const char* canonical = CanonicalS3OptionName(key.substr(std::strlen("s3.")));
+            if (canonical != nullptr && normalized.find(canonical) == normalized.end()) {
+                normalized.emplace(canonical, value);
+            }
+        }
+    }
+    return normalized;
+}
+
+std::shared_ptr WrapProvider(aws_credentials_provider* provider) {
+    return std::shared_ptr(provider, aws_credentials_provider_release);
+}
+
+Result ResolveRegion(const std::map& options) {
+    auto region = options.find(kS3RegionOption);
+    if (region != options.end() && !region->second.empty()) {
+        return region->second;
+    }
+    const char* environment_region = std::getenv("AWS_REGION");
+    if (environment_region != nullptr && environment_region[0] != '\0') {
+        return std::string(environment_region);
+    }
+    environment_region = std::getenv("AWS_DEFAULT_REGION");
+    if (environment_region != nullptr && environment_region[0] != '\0') {
+        return std::string(environment_region);
+    }
+
+    PAIMON_ASSIGN_OR_RAISE(AwsAuthRuntime * runtime, GetAwsAuthRuntime());
+    aws_byte_cursor profile_override{};
+    const aws_byte_cursor* profile_override_ptr = nullptr;
+    auto profile = options.find(kS3ProfileOption);
+    if (profile != options.end() && !profile->second.empty()) {
+        profile_override = Cursor(profile->second);
+        profile_override_ptr = &profile_override;
+    }
+    aws_string* config_path = aws_get_config_file_path(runtime->allocator(), nullptr);
+    aws_string* profile_name = aws_get_profile_name(runtime->allocator(), profile_override_ptr);
+    aws_profile_collection* profiles = config_path == nullptr
+                                           ? nullptr
+                                           : aws_profile_collection_new_from_file(
+                                                 runtime->allocator(), config_path, AWS_PST_CONFIG);
+    const aws_profile* selected_profile =
+        profiles == nullptr || profile_name == nullptr
+            ? nullptr
+            : aws_profile_collection_get_profile(profiles, profile_name);
+    aws_string* region_name = aws_string_new_from_c_str(runtime->allocator(), "region");
+    const aws_profile_property* property =
+        selected_profile == nullptr || region_name == nullptr
+            ? nullptr
+            : aws_profile_get_property(selected_profile, region_name);
+    const aws_string* value =
+        property == nullptr ? nullptr : aws_profile_property_get_value(property);
+    std::string resolved = value == nullptr ? "" : aws_string_c_str(value);
+    aws_string_destroy(region_name);
+    aws_profile_collection_release(profiles);
+    aws_string_destroy(profile_name);
+    aws_string_destroy(config_path);
+    return resolved.empty() ? "us-east-1" : resolved;
+}
+
+Result> MakeCredentialsProvider(
+    const std::map& options) {
+    PAIMON_ASSIGN_OR_RAISE(AwsAuthRuntime * runtime, GetAwsAuthRuntime());
+    auto access = options.find(kS3AccessKeyOption);
+    if (access != options.end()) {
+        const std::string& secret = options.at(kS3SecretKeyOption);
+        std::string token;
+        auto configured_token = options.find(kS3SessionTokenOption);
+        if (configured_token != options.end()) {
+            token = configured_token->second;
+        }
+        aws_credentials_provider_static_options static_options{};
+        static_options.access_key_id = Cursor(access->second);
+        static_options.secret_access_key = Cursor(secret);
+        static_options.session_token = Cursor(token);
+        return WrapProvider(
+            aws_credentials_provider_new_static(runtime->allocator(), &static_options));
+    }
+
+    aws_byte_cursor profile_override{};
+    auto profile_iter = options.find(kS3ProfileOption);
+    if (profile_iter != options.end() && !profile_iter->second.empty()) {
+        profile_override = Cursor(profile_iter->second);
+    }
+    std::string region;
+    auto region_iter = options.find(kS3RegionOption);
+    if (region_iter != options.end()) {
+        region = region_iter->second;
+    }
+
+    std::vector providers;
+    aws_credentials_provider_environment_options environment_options{};
+    providers.push_back(
+        aws_credentials_provider_new_environment(runtime->allocator(), &environment_options));
+
+    aws_credentials_provider_profile_options profile_options{};
+    profile_options.profile_name_override = profile_override;
+    profile_options.bootstrap = runtime->bootstrap();
+    profile_options.tls_ctx = runtime->tls_context();
+    providers.push_back(
+        aws_credentials_provider_new_profile(runtime->allocator(), &profile_options));
+
+    aws_credentials_provider_sts_web_identity_options web_options{};
+    web_options.profile_name_override = profile_override;
+    web_options.region = Cursor(region);
+    web_options.bootstrap = runtime->bootstrap();
+    web_options.tls_ctx = runtime->tls_context();
+    providers.push_back(
+        aws_credentials_provider_new_sts_web_identity(runtime->allocator(), &web_options));
+
+    aws_credentials_provider_sso_options sso_options{};
+    sso_options.profile_name_override = profile_override;
+    sso_options.bootstrap = runtime->bootstrap();
+    sso_options.tls_ctx = runtime->tls_context();
+    providers.push_back(aws_credentials_provider_new_sso(runtime->allocator(), &sso_options));
+
+    aws_credentials_provider_login_options login_options{};
+    login_options.profile_name_override = profile_override;
+    login_options.login_region = Cursor(region);
+    login_options.bootstrap = runtime->bootstrap();
+    login_options.tls_ctx = runtime->tls_context();
+    providers.push_back(aws_credentials_provider_new_login(runtime->allocator(), &login_options));
+
+    aws_credentials_provider_chain_default_options default_options{};
+    default_options.profile_name_override = profile_override;
+    default_options.bootstrap = runtime->bootstrap();
+    default_options.tls_ctx = runtime->tls_context();
+    default_options.skip_environment_credentials_provider = true;
+    providers.push_back(
+        aws_credentials_provider_new_chain_default(runtime->allocator(), &default_options));
+
+    providers.erase(std::remove(providers.begin(), providers.end(), nullptr), providers.end());
+    aws_credentials_provider_chain_options chain_options{};
+    chain_options.providers = providers.data();
+    chain_options.provider_count = providers.size();
+    aws_credentials_provider* chain =
+        aws_credentials_provider_new_chain(runtime->allocator(), &chain_options);
+    for (aws_credentials_provider* provider : providers) {
+        aws_credentials_provider_release(provider);
+    }
+    if (chain == nullptr) {
+        return std::shared_ptr();
+    }
+    aws_credentials_provider_cached_options cached_options{};
+    cached_options.source = chain;
+    cached_options.refresh_time_in_milliseconds = 15 * 60 * 1000;
+    aws_credentials_provider* cached =
+        aws_credentials_provider_new_cached(runtime->allocator(), &cached_options);
+    aws_credentials_provider_release(chain);
+    return WrapProvider(cached);
+}
+
+struct Endpoint {
+    std::string scheme;
+    std::string authority;
+    std::string base_path;
+};
+
+Result ParseEndpoint(std::string endpoint) {
+    if (endpoint.find("://") == std::string::npos) {
+        endpoint = "https://" + endpoint;
+    }
+    CURLU* url = curl_url();
+    if (url == nullptr) {
+        return Status::IOError("failed to create S3 endpoint parser");
+    }
+    ScopeGuard cleanup_url([url] { curl_url_cleanup(url); });
+    CURLUcode code = curl_url_set(url, CURLUPART_URL, endpoint.c_str(), 0);
+    if (code != CURLUE_OK) {
+        return Status::Invalid(
+            fmt::format("invalid S3 endpoint {}: code {}", endpoint, static_cast(code)));
+    }
+    auto get_part = [url, &endpoint](CURLUPart part, CURLUcode no_value,
+                                     const char* name) -> Result> {
+        char* value = nullptr;
+        CURLUcode result = curl_url_get(url, part, &value, 0);
+        if (result == no_value) {
+            return std::optional();
+        }
+        if (result != CURLUE_OK) {
+            return Status::Invalid(fmt::format("invalid S3 endpoint {} {}: code {}", endpoint, name,
+                                               static_cast(result)));
+        }
+        ScopeGuard free_value([value] { curl_free(value); });
+        return std::optional(value);
+    };
+    PAIMON_ASSIGN_OR_RAISE(std::optional scheme,
+                           get_part(CURLUPART_SCHEME, CURLUE_NO_SCHEME, "scheme"));
+    PAIMON_ASSIGN_OR_RAISE(std::optional host,
+                           get_part(CURLUPART_HOST, CURLUE_NO_HOST, "host"));
+    std::string normalized_scheme = scheme ? StringUtils::ToLowerCase(*scheme) : "";
+    if (!host || (normalized_scheme != "http" && normalized_scheme != "https")) {
+        return Status::Invalid(fmt::format("invalid S3 endpoint {}", endpoint));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::optional user,
+                           get_part(CURLUPART_USER, CURLUE_NO_USER, "user"));
+    PAIMON_ASSIGN_OR_RAISE(std::optional password,
+                           get_part(CURLUPART_PASSWORD, CURLUE_NO_PASSWORD, "password"));
+    PAIMON_ASSIGN_OR_RAISE(std::optional query,
+                           get_part(CURLUPART_QUERY, CURLUE_NO_QUERY, "query"));
+    PAIMON_ASSIGN_OR_RAISE(std::optional fragment,
+                           get_part(CURLUPART_FRAGMENT, CURLUE_NO_FRAGMENT, "fragment"));
+    if (user || password || query || fragment) {
+        return Status::Invalid(fmt::format(
+            "S3 endpoint {} must not contain user, password, query, or fragment", endpoint));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::optional port,
+                           get_part(CURLUPART_PORT, CURLUE_NO_PORT, "port"));
+    char* path = nullptr;
+    code = curl_url_get(url, CURLUPART_PATH, &path, 0);
+    if (code != CURLUE_OK) {
+        return Status::Invalid(
+            fmt::format("invalid S3 endpoint {} path: code {}", endpoint, static_cast(code)));
+    }
+    ScopeGuard free_path([path] { curl_free(path); });
+    std::string authority = *host;
+    if (authority.find(':') != std::string::npos) {
+        authority = "[" + authority + "]";
+    }
+    if (port) {
+        authority += ":" + *port;
+    }
+    return Endpoint{std::move(normalized_scheme), std::move(authority), path};
+}
+
+bool IsVirtualHostableS3Bucket(const std::string& bucket, bool allow_subdomains) {
+    if (bucket.size() < 3 || bucket.size() > 63) {
+        return false;
+    }
+    bool label_start = true;
+    for (size_t index = 0; index < bucket.size(); ++index) {
+        const auto character = static_cast(bucket[index]);
+        if (std::islower(character) || std::isdigit(character)) {
+            label_start = false;
+            continue;
+        }
+        if (character == '-') {
+            if (label_start || index + 1 == bucket.size() || bucket[index + 1] == '.') {
+                return false;
+            }
+            continue;
+        }
+        if (character == '.') {
+            if (!allow_subdomains || label_start || index + 1 == bucket.size()) {
+                return false;
+            }
+            label_start = true;
+            continue;
+        }
+        return false;
+    }
+    return !label_start;
+}
+
+bool IsIpAddressAuthority(const std::string& authority) {
+    if (!authority.empty() && authority.front() == '[') {
+        return true;
+    }
+    std::string_view host(authority);
+    size_t port_separator = host.find(':');
+    if (port_separator != std::string_view::npos) {
+        host = host.substr(0, port_separator);
+    }
+    size_t component_start = 0;
+    int component_count = 0;
+    while (component_start < host.size()) {
+        size_t component_end = host.find('.', component_start);
+        std::string_view component = host.substr(component_start, component_end - component_start);
+        if (component.empty() || component.size() > 3) {
+            return false;
+        }
+        int value = 0;
+        for (unsigned char character : component) {
+            if (!std::isdigit(character)) {
+                return false;
+            }
+            value = value * 10 + character - '0';
+        }
+        if (value > 255) {
+            return false;
+        }
+        ++component_count;
+        if (component_end == std::string_view::npos) {
+            break;
+        }
+        component_start = component_end + 1;
+    }
+    return component_count == 4;
+}
+
+const char* AwsDnsSuffixForRegion(const std::string& region) {
+    if (region.rfind("cn-", 0) == 0) {
+        return "amazonaws.com.cn";
+    }
+    if (region.rfind("eusc-de-", 0) == 0) {
+        return "amazonaws.eu";
+    }
+    if (region.rfind("us-iso-", 0) == 0) {
+        return "c2s.ic.gov";
+    }
+    if (region.rfind("us-isob-", 0) == 0) {
+        return "sc2s.sgov.gov";
+    }
+    if (region.rfind("eu-isoe-", 0) == 0) {
+        return "cloud.adc-e.uk";
+    }
+    if (region.rfind("us-isof-", 0) == 0) {
+        return "csp.hci.ic.gov";
+    }
+    return "amazonaws.com";
+}
+
+struct SigningContext {
+    SigningContext(aws_allocator* allocator, aws_http_message* message)
+        : allocator(allocator), message(message) {}
+
+    aws_allocator* allocator;
+    aws_http_message* message;
+    std::mutex mutex;
+    std::condition_variable condition;
+    int error_code = AWS_ERROR_SUCCESS;
+    bool complete = false;
+};
+
+void OnSigningComplete(aws_signing_result* result, int error_code, void* user_data) {
+    auto* context = static_cast(user_data);
+    if (error_code == AWS_ERROR_SUCCESS &&
+        aws_apply_signing_result_to_http_request(context->message, context->allocator, result)) {
+        error_code = aws_last_error();
+    }
+    {
+        std::scoped_lock lock(context->mutex);
+        context->error_code = error_code;
+        context->complete = true;
+    }
+    context->condition.notify_one();
+}
+
+class S3ObjectStoreClient : public ObjectStoreClient,
+                            public std::enable_shared_from_this {
+ public:
+    static Result> Create(
+        const std::map& options, std::shared_ptr http_client,
+        std::shared_ptr credentials, std::unique_ptr executor) {
+        PAIMON_ASSIGN_OR_RAISE(std::string region, ResolveRegion(options));
+        auto endpoint = options.find(kS3EndpointOption);
+        bool use_default_endpoint = endpoint == options.end() || endpoint->second.empty();
+        PAIMON_ASSIGN_OR_RAISE(
+            Endpoint parsed_endpoint,
+            ParseEndpoint(use_default_endpoint ? fmt::format("https://s3.{}.{}", region,
+                                                             AwsDnsSuffixForRegion(region))
+                                               : endpoint->second));
+        auto path_style = options.find(kS3PathStyleAccessOption);
+        bool use_path_style =
+            path_style != options.end() &&
+            OptionsUtils::GetValueFromMap(options, kS3PathStyleAccessOption).value();
+        return std::shared_ptr(new S3ObjectStoreClient(
+            std::move(http_client), std::move(credentials), std::move(executor),
+            std::move(parsed_endpoint), std::move(region), use_path_style, use_default_endpoint));
+    }
+
+    Result HeadObject(const ObjectStorePath& path) const override {
+        PAIMON_ASSIGN_OR_RAISE(HttpResponse response,
+                               Execute(path, HttpMethod::HEAD, "", {}, nullptr));
+        if (response.status_code == 404) {
+            return Status::NotExist(
+                fmt::format("s3://{}/{} does not exist", path.bucket, path.key));
+        }
+        PAIMON_RETURN_NOT_OK(CheckResponse(response, "HeadObject", path));
+        auto length = response.headers.find("content-length");
+        if (length == response.headers.end()) {
+            return Status::IOError("HeadObject response is missing Content-Length");
+        }
+        int64_t modification_time = 0;
+        auto modified = response.headers.find("last-modified");
+        if (modified != response.headers.end()) {
+            modification_time = ParseModificationTime(modified->second);
+        }
+        PAIMON_ASSIGN_OR_RAISE(int64_t object_size,
+                               ParseNonNegativeInt64(length->second, "Content-Length"));
+        return ObjectMetadata{path.key, object_size, modification_time};
+    }
+
+    Result ListObjects(const ObjectStorePath& path,
+                                          const std::string& continuation_token,
+                                          int32_t max_keys) const override {
+        std::string query = "list-type=2&delimiter=%2F&encoding-type=url";
+        if (!path.key.empty()) {
+            PAIMON_ASSIGN_OR_RAISE(std::string encoded_prefix, PercentEncode(path.key, false));
+            query += "&prefix=" + encoded_prefix;
+        }
+        if (!continuation_token.empty()) {
+            PAIMON_ASSIGN_OR_RAISE(std::string encoded_token,
+                                   PercentEncode(continuation_token, false));
+            query += "&continuation-token=" + encoded_token;
+        }
+        if (max_keys > 0) {
+            query += "&max-keys=" + std::to_string(max_keys);
+        }
+        std::string body;
+        HttpBodyConsumer consumer = [&body](const char* data, int64_t size) {
+            body.append(data, static_cast(size));
+            return Status::OK();
+        };
+        ObjectStorePath bucket_path{path.bucket, ""};
+        PAIMON_ASSIGN_OR_RAISE(HttpResponse response,
+                               Execute(bucket_path, HttpMethod::GET, query, {}, consumer));
+        if (response.status_code == 404) {
+            return Status::NotExist(fmt::format("S3 bucket {} does not exist", path.bucket));
+        }
+        PAIMON_RETURN_NOT_OK(CheckResponse(response, "ListObjectsV2", path));
+        if (body.find("") == std::string::npos) {
+            return Status::IOError("malformed S3 ListObjectsV2 XML response");
+        }
+        ListObjectsResult result;
+        PAIMON_ASSIGN_OR_RAISE(std::vector contents, TagBlocks(body, "Contents"));
+        for (const std::string& block : contents) {
+            auto key = TagValue(block, "Key");
+            auto size = TagValue(block, "Size");
+            if (!key || !size) {
+                return Status::IOError("S3 ListObjectsV2 Contents is missing Key or Size");
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::string decoded_key, PercentDecode(*key, "Key"));
+            PAIMON_ASSIGN_OR_RAISE(int64_t object_size,
+                                   ParseNonNegativeInt64(*size, "ListObjectsV2 Size"));
+            int64_t modified = 0;
+            auto last_modified = TagValue(block, "LastModified");
+            if (last_modified) {
+                modified = ParseModificationTime(*last_modified);
+            }
+            result.objects.push_back(ObjectMetadata{decoded_key, object_size, modified});
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::vector common_prefixes,
+                               TagBlocks(body, "CommonPrefixes"));
+        for (const std::string& block : common_prefixes) {
+            auto prefix = TagValue(block, "Prefix");
+            if (!prefix) {
+                return Status::IOError("S3 ListObjectsV2 CommonPrefixes is missing Prefix");
+            }
+            PAIMON_ASSIGN_OR_RAISE(std::string decoded_prefix, PercentDecode(*prefix, "Prefix"));
+            result.common_prefixes.push_back(std::move(decoded_prefix));
+        }
+        auto is_truncated = TagValue(body, "IsTruncated");
+        if (!is_truncated || (*is_truncated != "true" && *is_truncated != "false")) {
+            return Status::IOError("S3 ListObjectsV2 response has invalid IsTruncated");
+        }
+        result.is_truncated = *is_truncated == "true";
+        auto token = TagValue(body, "NextContinuationToken");
+        if (token) {
+            result.continuation_token = std::move(*token);
+        }
+        return result;
+    }
+
+    Result GetObjectRange(const ObjectStorePath& path, int64_t offset, int64_t size,
+                                   char* buffer) const override {
+        if (size == 0) {
+            return 0;
+        }
+        int64_t copied = 0;
+        HttpHeaders headers{{"range", fmt::format("bytes={}-{}", offset, offset + size - 1)}};
+        HttpBodyConsumer consumer = [&copied, buffer, size](const char* data, int64_t length) {
+            if (length > size - copied) {
+                return Status::IOError("S3 range response exceeds the requested length");
+            }
+            std::memcpy(buffer + copied, data, static_cast(length));
+            copied += length;
+            return Status::OK();
+        };
+        PAIMON_ASSIGN_OR_RAISE(HttpResponse response,
+                               Execute(path, HttpMethod::GET, "", headers, consumer));
+        if (response.status_code == 404) {
+            return Status::NotExist(
+                fmt::format("s3://{}/{} does not exist", path.bucket, path.key));
+        }
+        PAIMON_RETURN_NOT_OK(CheckResponse(response, "GetObject", path));
+        if (copied != size) {
+            return Status::IOError(
+                fmt::format("GetObject read {} bytes for s3://{}/{}, expected {}", copied,
+                            path.bucket, path.key, size));
+        }
+        return copied;
+    }
+
+    void GetObjectRangeAsync(const ObjectStorePath& path, int64_t offset, int64_t size,
+                             char* buffer, std::function&& callback) const override {
+        auto self = shared_from_this();
+        executor_->Add([self = std::move(self), path, offset, size, buffer,
+                        callback = std::move(callback)]() mutable {
+            Result result = self->GetObjectRange(path, offset, size, buffer);
+            callback(result.ok() ? Status::OK() : result.status());
+        });
+    }
+
+ private:
+    S3ObjectStoreClient(std::shared_ptr http_client,
+                        std::shared_ptr credentials,
+                        std::unique_ptr executor, Endpoint endpoint, std::string region,
+                        bool path_style, bool use_default_endpoint)
+        : http_client_(std::move(http_client)),
+          credentials_(std::move(credentials)),
+          endpoint_(std::move(endpoint)),
+          region_(std::move(region)),
+          path_style_(path_style),
+          use_default_endpoint_(use_default_endpoint),
+          executor_(std::move(executor)) {}
+    Status CheckResponse(const HttpResponse& response, const std::string& operation,
+                         const ObjectStorePath& path) const {
+        if (response.status_code >= 200 && response.status_code < 300) {
+            return Status::OK();
+        }
+        return Status::IOError(fmt::format("{} failed for s3://{}/{}: HTTP {}", operation,
+                                           path.bucket, path.key, response.status_code));
+    }
+
+    Result Execute(const ObjectStorePath& object, HttpMethod method,
+                                 const std::string& query, const HttpHeaders& headers,
+                                 const HttpBodyConsumer& consumer) const {
+        std::string authority = endpoint_.authority;
+        std::string request_path = endpoint_.base_path;
+        if (request_path.empty() || request_path.back() != '/') {
+            request_path += '/';
+        }
+        bool use_path_style =
+            path_style_ ||
+            (use_default_endpoint_
+                 ? !IsVirtualHostableS3Bucket(object.bucket, false)
+                 : endpoint_.scheme != "http" || IsIpAddressAuthority(endpoint_.authority) ||
+                       !IsVirtualHostableS3Bucket(object.bucket, true));
+        if (use_path_style) {
+            PAIMON_ASSIGN_OR_RAISE(std::string encoded_bucket, PercentEncode(object.bucket, false));
+            request_path += encoded_bucket + "/";
+        } else {
+            authority = object.bucket + "." + authority;
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::string encoded_key, PercentEncode(object.key, true));
+        request_path += encoded_key;
+        if (!query.empty()) {
+            request_path += "?" + query;
+        }
+
+        PAIMON_ASSIGN_OR_RAISE(AwsAuthRuntime * runtime, GetAwsAuthRuntime());
+        aws_http_message* message = aws_http_message_new_request(runtime->allocator());
+        if (message == nullptr) {
+            return Status::IOError("failed to create S3 HTTP request");
+        }
+        ScopeGuard release_message([message] { aws_http_message_release(message); });
+        std::string method_name = method == HttpMethod::HEAD ? "HEAD" : "GET";
+        aws_http_message_set_request_method(message, Cursor(method_name));
+        aws_http_message_set_request_path(message, Cursor(request_path));
+        aws_http_header host_header{};
+        host_header.name = aws_byte_cursor_from_c_str("host");
+        host_header.value = Cursor(authority);
+        aws_http_message_add_header(message, host_header);
+        for (const auto& [name, value] : headers) {
+            aws_http_header header{};
+            header.name = Cursor(name);
+            header.value = Cursor(value);
+            aws_http_message_add_header(message, header);
+        }
+        aws_signable* signable = aws_signable_new_http_request(runtime->allocator(), message);
+        if (signable == nullptr) {
+            return Status::IOError("failed to create S3 signable request");
+        }
+        ScopeGuard destroy_signable([signable] { aws_signable_destroy(signable); });
+        aws_signing_config_aws config{};
+        config.config_type = AWS_SIGNING_CONFIG_AWS;
+        config.algorithm = AWS_SIGNING_ALGORITHM_V4;
+        config.signature_type = AWS_ST_HTTP_REQUEST_HEADERS;
+        config.region = Cursor(region_);
+        config.service = aws_byte_cursor_from_c_str("s3");
+        aws_date_time_init_now(&config.date);
+        config.flags.use_double_uri_encode = false;
+        config.flags.should_normalize_uri_path = false;
+        config.signed_body_value = g_aws_signed_body_value_unsigned_payload;
+        config.signed_body_header = AWS_SBHT_X_AMZ_CONTENT_SHA256;
+        config.credentials_provider = credentials_.get();
+
+        SigningContext context(runtime->allocator(), message);
+        int result = aws_sign_request_aws(runtime->allocator(), signable,
+                                          reinterpret_cast(&config),
+                                          OnSigningComplete, &context);
+        if (result == AWS_OP_SUCCESS) {
+            std::unique_lock lock(context.mutex);
+            context.condition.wait(lock, [&context] { return context.complete; });
+        }
+        if (result != AWS_OP_SUCCESS || context.error_code != AWS_ERROR_SUCCESS) {
+            int error = result == AWS_OP_SUCCESS ? context.error_code : aws_last_error();
+            return Status::IOError(
+                fmt::format("failed to sign S3 request: {}", aws_error_debug_str(error)));
+        }
+
+        HttpRequest request;
+        request.method = method;
+        request.url = endpoint_.scheme + "://" + authority + request_path;
+        aws_http_headers* signed_headers = aws_http_message_get_headers(message);
+        for (size_t i = 0; i < aws_http_headers_count(signed_headers); ++i) {
+            aws_http_header header;
+            aws_http_headers_get_index(signed_headers, i, &header);
+            request.headers[std::string(reinterpret_cast(header.name.ptr),
+                                        header.name.len)] =
+                std::string(reinterpret_cast(header.value.ptr), header.value.len);
+        }
+        HttpBodyConsumer body_consumer = consumer;
+        if (!body_consumer) {
+            body_consumer = [](const char*, int64_t) { return Status::OK(); };
+        }
+        return http_client_->Execute(request, body_consumer);
+    }
+
+    std::shared_ptr http_client_;
+    std::shared_ptr credentials_;
+    Endpoint endpoint_;
+    std::string region_;
+    bool path_style_ = false;
+    bool use_default_endpoint_ = false;
+    std::unique_ptr executor_;
+};
+
+}  // namespace
+
+Status ValidateS3Options(const std::map& options) {
+    auto access = options.find(kS3AccessKeyOption);
+    auto secret = options.find(kS3SecretKeyOption);
+    auto token = options.find(kS3SessionTokenOption);
+    bool has_access = access != options.end();
+    bool has_secret = secret != options.end();
+    bool has_token = token != options.end();
+    if (has_access != has_secret) {
+        return Status::Invalid(fmt::format("{} and {} must be configured together",
+                                           kS3AccessKeyOption, kS3SecretKeyOption));
+    }
+    if (has_token && !has_access) {
+        return Status::Invalid(fmt::format("{} requires {} and {}", kS3SessionTokenOption,
+                                           kS3AccessKeyOption, kS3SecretKeyOption));
+    }
+    if (has_access && access->second.empty()) {
+        return Status::Invalid(fmt::format("{} must not be empty", kS3AccessKeyOption));
+    }
+    if (has_secret && secret->second.empty()) {
+        return Status::Invalid(fmt::format("{} must not be empty", kS3SecretKeyOption));
+    }
+    if (options.find(kS3PathStyleAccessOption) == options.end()) {
+        return Status::OK();
+    }
+    Result parsed = OptionsUtils::GetValueFromMap(options, kS3PathStyleAccessOption);
+    if (!parsed.ok()) {
+        return Status::Invalid(
+            fmt::format("{} {}", kS3PathStyleAccessOption, parsed.status().message()));
+    }
+    return Status::OK();
+}
+
+S3FileSystem::S3FileSystem(std::shared_ptr client)
+    : ObjectStoreFileSystem("s3", std::move(client)) {}
+
+Result> S3FileSystem::Create(
+    const std::map& options) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr client,
+                           MakeS3ObjectStoreClient(options, std::make_shared()));
+    return std::unique_ptr(new S3FileSystem(std::move(client)));
+}
+
+Result> MakeS3ObjectStoreClient(
+    const std::map& options, std::shared_ptr http_client) {
+    std::map normalized_options = NormalizeS3Options(options);
+    PAIMON_RETURN_NOT_OK(ValidateS3Options(normalized_options));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr credentials,
+                           MakeCredentialsProvider(normalized_options));
+    if (!credentials) {
+        return Status::IOError("failed to initialize S3 credentials provider");
+    }
+    std::unique_ptr executor = CreateDefaultExecutor();
+    return S3ObjectStoreClient::Create(normalized_options, std::move(http_client),
+                                       std::move(credentials), std::move(executor));
+}
+
+}  // namespace paimon::s3
diff --git a/src/paimon/fs/s3/s3_file_system.h b/src/paimon/fs/s3/s3_file_system.h
new file mode 100644
index 00000000..12caf8b7
--- /dev/null
+++ b/src/paimon/fs/s3/s3_file_system.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+
+#include "paimon/common/fs/http_client.h"
+#include "paimon/common/fs/object_store_file_system.h"
+
+namespace paimon::s3 {
+
+inline constexpr char kS3RegionOption[] = "s3.region";
+inline constexpr char kS3EndpointOption[] = "s3.endpoint";
+inline constexpr char kS3PathStyleAccessOption[] = "s3.path-style-access";
+inline constexpr char kS3ProfileOption[] = "s3.profile";
+inline constexpr char kS3AccessKeyOption[] = "s3.access-key";
+inline constexpr char kS3SecretKeyOption[] = "s3.secret-key";
+inline constexpr char kS3SessionTokenOption[] = "s3.session.token";
+
+Status ValidateS3Options(const std::map& options);
+Result> MakeS3ObjectStoreClient(
+    const std::map& options, std::shared_ptr http_client);
+
+class S3FileSystem : public ObjectStoreFileSystem {
+ public:
+    using ObjectStoreFileSystem::Create;
+
+    static Result> Create(
+        const std::map& options);
+    ~S3FileSystem() override = default;
+
+ private:
+    explicit S3FileSystem(std::shared_ptr client);
+};
+
+}  // namespace paimon::s3
diff --git a/src/paimon/fs/s3/s3_file_system_factory.cpp b/src/paimon/fs/s3/s3_file_system_factory.cpp
new file mode 100644
index 00000000..8a28e274
--- /dev/null
+++ b/src/paimon/fs/s3/s3_file_system_factory.cpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/fs/s3/s3_file_system_factory.h"
+
+#include "paimon/factories/factory.h"
+#include "paimon/fs/s3/s3_file_system.h"
+
+namespace paimon::s3 {
+
+const char S3FileSystemFactory::IDENTIFIER[] = "s3";
+
+Result> S3FileSystemFactory::Create(
+    const std::string&, const std::map& options) const {
+    return S3FileSystem::Create(options);
+}
+
+REGISTER_PAIMON_FACTORY(S3FileSystemFactory);
+
+}  // namespace paimon::s3
diff --git a/src/paimon/fs/s3/s3_file_system_factory.h b/src/paimon/fs/s3/s3_file_system_factory.h
new file mode 100644
index 00000000..189317cd
--- /dev/null
+++ b/src/paimon/fs/s3/s3_file_system_factory.h
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include "paimon/fs/file_system_factory.h"
+
+namespace paimon::s3 {
+
+class S3FileSystemFactory : public FileSystemFactory {
+ public:
+    static const char IDENTIFIER[];
+    const char* Identifier() const override {
+        return IDENTIFIER;
+    }
+    Result> Create(
+        const std::string& path, const std::map& options) const override;
+};
+
+}  // namespace paimon::s3
diff --git a/src/paimon/fs/s3/s3_file_system_test.cpp b/src/paimon/fs/s3/s3_file_system_test.cpp
new file mode 100644
index 00000000..09b1bb4b
--- /dev/null
+++ b/src/paimon/fs/s3/s3_file_system_test.cpp
@@ -0,0 +1,503 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/fs/s3/s3_file_system.h"
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/fs/s3/s3_file_system_factory.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::s3 {
+namespace {
+
+class MockHttpClient : public HttpClient {
+ public:
+    Result Execute(const HttpRequest& request,
+                                 const HttpBodyConsumer& consumer) const override {
+        request_ = request;
+        HttpResponse response;
+        response.status_code = status_code_;
+        response.headers = response_headers_;
+        if (!body_.empty()) {
+            PAIMON_RETURN_NOT_OK(consumer(body_.data(), body_.size()));
+            response.body_size = body_.size();
+        }
+        return response;
+    }
+
+    mutable HttpRequest request_;
+    int32_t status_code_ = 200;
+    HttpHeaders response_headers_;
+    std::string body_;
+};
+
+class ScopedEnvironmentVariable {
+ public:
+    ScopedEnvironmentVariable(const char* name, std::optional value) : name_(name) {
+        const char* previous = std::getenv(name);
+        if (previous != nullptr) {
+            previous_ = previous;
+        }
+        if (value) {
+            setenv(name, value->c_str(), 1);
+        } else {
+            unsetenv(name);
+        }
+    }
+
+    ~ScopedEnvironmentVariable() {
+        if (previous_) {
+            setenv(name_.c_str(), previous_->c_str(), 1);
+        } else {
+            unsetenv(name_.c_str());
+        }
+    }
+
+ private:
+    std::string name_;
+    std::optional previous_;
+};
+
+std::map StaticOptions() {
+    return {{kS3AccessKeyOption, "access"},
+            {kS3SecretKeyOption, "secret"},
+            {kS3SessionTokenOption, "token"},
+            {kS3RegionOption, "ap-northeast-2"}};
+}
+
+const std::string* FindHeader(const HttpHeaders& headers, const std::string& name) {
+    std::string normalized_name = StringUtils::ToLowerCase(name);
+    for (const auto& [header_name, value] : headers) {
+        if (StringUtils::ToLowerCase(header_name) == normalized_name) {
+            return &value;
+        }
+    }
+    return nullptr;
+}
+
+TEST(S3ObjectStoreClientTest, TestHeadAndSigning) {
+    auto http = std::make_shared();
+    http->response_headers_["content-length"] = "12";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(StaticOptions(), http));
+    ASSERT_OK_AND_ASSIGN(auto metadata, client->HeadObject({"bucket", "a b/file"}));
+    ASSERT_EQ(metadata.size, 12);
+    ASSERT_EQ(http->request_.url, "https://bucket.s3.ap-northeast-2.amazonaws.com/a%20b/file");
+    const std::string* authorization = FindHeader(http->request_.headers, "authorization");
+    ASSERT_NE(authorization, nullptr);
+    ASSERT_NE(authorization->find("/ap-northeast-2/s3/aws4_request"), std::string::npos);
+    ASSERT_NE(authorization->find(
+                  "SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token"),
+              std::string::npos);
+    ASSERT_EQ(http->request_.headers["host"], "bucket.s3.ap-northeast-2.amazonaws.com");
+    const std::string* session_token = FindHeader(http->request_.headers, "x-amz-security-token");
+    ASSERT_NE(session_token, nullptr);
+    ASSERT_EQ(*session_token, "token");
+}
+
+TEST(S3ObjectStoreClientTest, TestDottedBucketUsesPathStyleForDefaultEndpoint) {
+    auto http = std::make_shared();
+    http->response_headers_["content-length"] = "12";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(StaticOptions(), http));
+    ASSERT_OK(client->HeadObject({"paimon.prod.data", "file"}));
+    ASSERT_EQ(http->request_.url, "https://s3.ap-northeast-2.amazonaws.com/paimon.prod.data/file");
+    ASSERT_EQ(http->request_.headers["host"], "s3.ap-northeast-2.amazonaws.com");
+
+    auto path_style_options = StaticOptions();
+    path_style_options[kS3PathStyleAccessOption] = "false";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr path_style_client,
+                         MakeS3ObjectStoreClient(path_style_options, http));
+    ASSERT_OK(path_style_client->HeadObject({"paimon.prod.data", "file"}));
+    ASSERT_EQ(http->request_.url, "https://s3.ap-northeast-2.amazonaws.com/paimon.prod.data/file");
+}
+
+TEST(S3ObjectStoreClientTest, TestCustomEndpointAddressing) {
+    auto http = std::make_shared();
+    http->response_headers_["content-length"] = "12";
+
+    auto https_options = StaticOptions();
+    https_options[kS3EndpointOption] = "https://s3.example.com";
+    https_options[kS3PathStyleAccessOption] = "false";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr https_client,
+                         MakeS3ObjectStoreClient(https_options, http));
+    ASSERT_OK(https_client->HeadObject({"bucket", "file"}));
+    ASSERT_EQ(http->request_.url, "https://s3.example.com/bucket/file");
+    ASSERT_OK(https_client->HeadObject({"paimon.prod.data", "file"}));
+    ASSERT_EQ(http->request_.url, "https://s3.example.com/paimon.prod.data/file");
+
+    auto http_options = StaticOptions();
+    http_options[kS3EndpointOption] = "http://s3.example.com";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr http_client,
+                         MakeS3ObjectStoreClient(http_options, http));
+    ASSERT_OK(http_client->HeadObject({"paimon.prod.data", "file"}));
+    ASSERT_EQ(http->request_.url, "http://paimon.prod.data.s3.example.com/file");
+
+    auto ip_options = StaticOptions();
+    ip_options[kS3EndpointOption] = "http://127.0.0.1:9000";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr ip_client,
+                         MakeS3ObjectStoreClient(ip_options, http));
+    ASSERT_OK(ip_client->HeadObject({"bucket", "file"}));
+    ASSERT_EQ(http->request_.url, "http://127.0.0.1:9000/bucket/file");
+
+    auto base_path_options = StaticOptions();
+    base_path_options[kS3EndpointOption] = "HTTPS://s3.example.com/storage";
+    base_path_options[kS3PathStyleAccessOption] = "true";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr base_path_client,
+                         MakeS3ObjectStoreClient(base_path_options, http));
+    ASSERT_OK(base_path_client->HeadObject({"bucket", "file"}));
+    ASSERT_EQ(http->request_.url, "https://s3.example.com/storage/bucket/file");
+}
+
+TEST(S3ObjectStoreClientTest, TestInvalidCustomEndpoint) {
+    for (const char* endpoint :
+         {"ftp://s3.example.com", "https://user@s3.example.com", "https://s3.example.com?query",
+          "https://s3.example.com#fragment"}) {
+        auto options = StaticOptions();
+        options[kS3EndpointOption] = endpoint;
+        ASSERT_NOK(MakeS3ObjectStoreClient(options, std::make_shared()));
+    }
+}
+
+TEST(S3ObjectStoreClientTest, TestNonVirtualHostableBucketUsesPathStyle) {
+    auto http = std::make_shared();
+    http->response_headers_["content-length"] = "12";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(StaticOptions(), http));
+    ASSERT_OK(client->HeadObject({"aa", "file"}));
+    ASSERT_EQ(http->request_.url, "https://s3.ap-northeast-2.amazonaws.com/aa/file");
+
+    auto options = StaticOptions();
+    options[kS3EndpointOption] = "http://s3.example.com";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr custom_client,
+                         MakeS3ObjectStoreClient(options, http));
+    ASSERT_OK(custom_client->HeadObject({"BucketName", "file"}));
+    ASSERT_EQ(http->request_.url, "http://s3.example.com/BucketName/file");
+}
+
+TEST(S3ObjectStoreClientTest, TestPathStyleOptionUsesCommonBooleanParser) {
+    for (const auto& [value, path_style] :
+         std::vector>{{"t", true},
+                                                   {"y", true},
+                                                   {"yes", true},
+                                                   {"1", true},
+                                                   {"f", false},
+                                                   {"n", false},
+                                                   {"no", false},
+                                                   {"0", false}}) {
+        auto options = StaticOptions();
+        options[kS3PathStyleAccessOption] = value;
+        auto http = std::make_shared();
+        http->response_headers_["content-length"] = "12";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                             MakeS3ObjectStoreClient(options, http));
+        ASSERT_OK(client->HeadObject({"bucket", "file"}));
+        ASSERT_EQ(http->request_.url, path_style
+                                          ? "https://s3.ap-northeast-2.amazonaws.com/bucket/file"
+                                          : "https://bucket.s3.ap-northeast-2.amazonaws.com/file");
+    }
+}
+
+TEST(S3ObjectStoreClientTest, TestOptionAliases) {
+    auto http = std::make_shared();
+    http->response_headers_["content-length"] = "12";
+    std::map python_options{
+        {"fs.s3.accessKeyId", "python-access"},  {"fs.s3.accessKeySecret", "python-secret"},
+        {"fs.s3.securityToken", "python-token"}, {"fs.s3.endpoint", "http://s3.example.com"},
+        {"fs.s3.region", "us-west-2"},           {"fs.s3.path.style.access", "true"}};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr python_client,
+                         MakeS3ObjectStoreClient(python_options, http));
+    ASSERT_OK(python_client->HeadObject({"bucket", "file"}));
+    ASSERT_EQ(http->request_.url, "http://s3.example.com/bucket/file");
+    const std::string* authorization = FindHeader(http->request_.headers, "authorization");
+    ASSERT_NE(authorization, nullptr);
+    ASSERT_NE(authorization->find("Credential=python-access/"), std::string::npos);
+    const std::string* token = FindHeader(http->request_.headers, "x-amz-security-token");
+    ASSERT_NE(token, nullptr);
+    ASSERT_EQ(*token, "python-token");
+
+    std::map java_options{{"s3a.access.key", "java-access"},
+                                                    {"s3a.secret.key", "java-secret"},
+                                                    {"s3a.session.token", "java-token"},
+                                                    {"s3a.region", "eu-west-1"}};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr java_client,
+                         MakeS3ObjectStoreClient(java_options, http));
+    ASSERT_OK(java_client->HeadObject({"bucket", "file"}));
+    ASSERT_EQ(http->request_.url, "https://bucket.s3.eu-west-1.amazonaws.com/file");
+    authorization = FindHeader(http->request_.headers, "authorization");
+    ASSERT_NE(authorization, nullptr);
+    ASSERT_NE(authorization->find("Credential=java-access/"), std::string::npos);
+    token = FindHeader(http->request_.headers, "x-amz-security-token");
+    ASSERT_NE(token, nullptr);
+    ASSERT_EQ(*token, "java-token");
+}
+
+TEST(S3ObjectStoreClientTest, TestCanonicalOptionsTakePrecedenceOverAliases) {
+    auto http = std::make_shared();
+    http->response_headers_["content-length"] = "12";
+    std::map options{{kS3AccessKeyOption, "canonical-access"},
+                                               {kS3SecretKeyOption, "canonical-secret"},
+                                               {"s3a.access.key", "alias-access"},
+                                               {"s3a.secret.key", "alias-secret"},
+                                               {kS3RegionOption, "ap-northeast-2"}};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(options, http));
+    ASSERT_OK(client->HeadObject({"bucket", "file"}));
+    const std::string* authorization = FindHeader(http->request_.headers, "authorization");
+    ASSERT_NE(authorization, nullptr);
+    ASSERT_NE(authorization->find("Credential=canonical-access/"), std::string::npos);
+}
+
+TEST(S3ObjectStoreClientTest, TestInvalidContentLength) {
+    for (const std::string content_length :
+         {"", "invalid", "-1", "12abc", "999999999999999999999999"}) {
+        auto http = std::make_shared();
+        http->response_headers_["content-length"] = content_length;
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                             MakeS3ObjectStoreClient(StaticOptions(), http));
+        ASSERT_TRUE(client->HeadObject({"bucket", "file"}).status().IsIOError());
+    }
+}
+
+TEST(S3ObjectStoreClientTest, TestInvalidModificationTime) {
+    auto http = std::make_shared();
+    http->response_headers_["content-length"] = "12";
+    http->response_headers_["last-modified"] = "invalid";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(StaticOptions(), http));
+    ASSERT_OK_AND_ASSIGN(auto metadata, client->HeadObject({"bucket", "file"}));
+    ASSERT_EQ(metadata.modification_time, 0);
+
+    http->body_ =
+        "falsefile"
+        "invalid12";
+    ASSERT_OK_AND_ASSIGN(auto result, client->ListObjects({"bucket", ""}, "", 0));
+    ASSERT_EQ(result.objects[0].modification_time, 0);
+}
+
+TEST(S3ObjectStoreClientTest, TestRegionFromEnvironment) {
+    ScopedEnvironmentVariable region("AWS_REGION", "eu-west-1");
+    auto options = StaticOptions();
+    options.erase(kS3RegionOption);
+    auto http = std::make_shared();
+    http->response_headers_["content-length"] = "12";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(options, http));
+    ASSERT_OK(client->HeadObject({"bucket", "file"}));
+    ASSERT_EQ(http->request_.url, "https://bucket.s3.eu-west-1.amazonaws.com/file");
+}
+
+TEST(S3ObjectStoreClientTest, TestDefaultEndpointsForAwsPartitions) {
+    for (const auto& [region, endpoint] : std::vector>{
+             {"ap-northeast-2", "https://bucket.s3.ap-northeast-2.amazonaws.com/file"},
+             {"cn-north-1", "https://bucket.s3.cn-north-1.amazonaws.com.cn/file"},
+             {"eusc-de-east-1", "https://bucket.s3.eusc-de-east-1.amazonaws.eu/file"},
+             {"us-iso-east-1", "https://bucket.s3.us-iso-east-1.c2s.ic.gov/file"},
+             {"us-isob-east-1", "https://bucket.s3.us-isob-east-1.sc2s.sgov.gov/file"},
+             {"eu-isoe-west-1", "https://bucket.s3.eu-isoe-west-1.cloud.adc-e.uk/file"},
+             {"us-isof-south-1", "https://bucket.s3.us-isof-south-1.csp.hci.ic.gov/file"},
+             {"us-gov-west-1", "https://bucket.s3.us-gov-west-1.amazonaws.com/file"}}) {
+        auto options = StaticOptions();
+        options[kS3RegionOption] = region;
+        auto http = std::make_shared();
+        http->response_headers_["content-length"] = "12";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                             MakeS3ObjectStoreClient(options, http));
+        ASSERT_OK(client->HeadObject({"bucket", "file"}));
+        ASSERT_EQ(http->request_.url, endpoint);
+    }
+}
+
+TEST(S3ObjectStoreClientTest, TestRegionFromProfile) {
+    auto test_dir = paimon::test::UniqueTestDirectory::Create();
+    ASSERT_TRUE(test_dir);
+    std::filesystem::path config_path = std::filesystem::path(test_dir->Str()) / "region-config";
+    {
+        std::ofstream config(config_path);
+        config << "[profile test-profile]\nregion = ap-south-1\n";
+    }
+    ScopedEnvironmentVariable region("AWS_REGION", std::nullopt);
+    ScopedEnvironmentVariable default_region("AWS_DEFAULT_REGION", std::nullopt);
+    ScopedEnvironmentVariable config_file("AWS_CONFIG_FILE", config_path.string());
+    auto options = StaticOptions();
+    options.erase(kS3RegionOption);
+    options[kS3ProfileOption] = "test-profile";
+    auto http = std::make_shared();
+    http->response_headers_["content-length"] = "12";
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(options, http));
+    ASSERT_OK(client->HeadObject({"bucket", "file"}));
+    ASSERT_EQ(http->request_.url, "https://bucket.s3.ap-south-1.amazonaws.com/file");
+}
+
+TEST(S3ObjectStoreClientTest, TestCredentialsFromEnvironmentProfile) {
+    auto test_dir = paimon::test::UniqueTestDirectory::Create();
+    ASSERT_TRUE(test_dir);
+    std::filesystem::path credentials_path =
+        std::filesystem::path(test_dir->Str()) / "credentials-config";
+    {
+        std::ofstream credentials(credentials_path);
+        credentials << "[environment-profile]\n"
+                       "aws_access_key_id = profile-access\n"
+                       "aws_secret_access_key = profile-secret\n"
+                       "aws_session_token = profile-token\n"
+                       "[default]\n"
+                       "aws_access_key_id = default-access\n"
+                       "aws_secret_access_key = default-secret\n"
+                       "aws_session_token = default-token\n";
+    }
+    ScopedEnvironmentVariable credentials_file("AWS_SHARED_CREDENTIALS_FILE",
+                                               credentials_path.string());
+    ScopedEnvironmentVariable access_key("AWS_ACCESS_KEY_ID", std::nullopt);
+    ScopedEnvironmentVariable secret_key("AWS_SECRET_ACCESS_KEY", std::nullopt);
+    ScopedEnvironmentVariable session_token("AWS_SESSION_TOKEN", std::nullopt);
+    {
+        ScopedEnvironmentVariable profile("AWS_PROFILE", "environment-profile");
+        auto http = std::make_shared();
+        http->response_headers_["content-length"] = "12";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                             MakeS3ObjectStoreClient({{kS3RegionOption, "ap-northeast-2"}}, http));
+        ASSERT_OK(client->HeadObject({"bucket", "file"}));
+        const std::string* authorization = FindHeader(http->request_.headers, "authorization");
+        ASSERT_NE(authorization, nullptr);
+        ASSERT_NE(authorization->find("Credential=profile-access/"), std::string::npos);
+        const std::string* token = FindHeader(http->request_.headers, "x-amz-security-token");
+        ASSERT_NE(token, nullptr);
+        ASSERT_EQ(*token, "profile-token");
+    }
+    {
+        ScopedEnvironmentVariable profile("AWS_PROFILE", std::nullopt);
+        auto http = std::make_shared();
+        http->response_headers_["content-length"] = "12";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                             MakeS3ObjectStoreClient({{kS3RegionOption, "ap-northeast-2"}}, http));
+        ASSERT_OK(client->HeadObject({"bucket", "file"}));
+        const std::string* authorization = FindHeader(http->request_.headers, "authorization");
+        ASSERT_NE(authorization, nullptr);
+        ASSERT_NE(authorization->find("Credential=default-access/"), std::string::npos);
+        const std::string* token = FindHeader(http->request_.headers, "x-amz-security-token");
+        ASSERT_NE(token, nullptr);
+        ASSERT_EQ(*token, "default-token");
+    }
+}
+
+TEST(S3ObjectStoreClientTest, TestRangeAndListObjects) {
+    auto http = std::make_shared();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(StaticOptions(), http));
+    http->body_ = "data";
+    char buffer[4];
+    ASSERT_OK_AND_ASSIGN(auto size, client->GetObjectRange({"bucket", "key"}, 2, 4, buffer));
+    ASSERT_EQ(size, 4);
+    ASSERT_EQ(std::string(buffer, sizeof(buffer)), "data");
+    ASSERT_EQ(http->request_.headers["range"], "bytes=2-5");
+
+    http->body_ =
+        "true"
+        "dir/a&b2026-01-01T00:00:00Z"
+        "7dir/a&lt;b8"
+        "dir/sub/"
+        "next token";
+    ASSERT_OK_AND_ASSIGN(auto result, client->ListObjects({"bucket", "dir/"}, "old token", 10));
+    ASSERT_TRUE(result.is_truncated);
+    ASSERT_EQ(result.continuation_token, "next token");
+    ASSERT_EQ(result.objects[0].key, "dir/a&b");
+    ASSERT_EQ(result.objects[1].key, "dir/a<b");
+    ASSERT_EQ(result.common_prefixes[0], "dir/sub/");
+    ASSERT_NE(http->request_.url.find("amazonaws.com/?list-type=2"), std::string::npos);
+    ASSERT_NE(http->request_.url.find("encoding-type=url"), std::string::npos);
+    ASSERT_NE(http->request_.url.find("continuation-token=old%20token"), std::string::npos);
+}
+
+TEST(S3ObjectStoreClientTest, TestUrlEncodedListObjects) {
+    auto http = std::make_shared();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(StaticOptions(), http));
+    http->body_ =
+        "true"
+        "dir/a%26b%0D7"
+        "dir/sub%25/"
+        "next%2Ftoken";
+    ASSERT_OK_AND_ASSIGN(auto result, client->ListObjects({"bucket", "dir/"}, "", 0));
+    ASSERT_EQ(result.objects[0].key, "dir/a&b\r");
+    ASSERT_EQ(result.common_prefixes[0], "dir/sub%/");
+    ASSERT_EQ(result.continuation_token, "next%2Ftoken");
+
+    http->body_ = "false";
+    ASSERT_OK(client->ListObjects({"bucket", "dir/"}, result.continuation_token, 0));
+    ASSERT_NE(http->request_.url.find("continuation-token=next%252Ftoken"), std::string::npos);
+
+    http->body_ =
+        "false"
+        "dir/invalid%27";
+    ASSERT_TRUE(client->ListObjects({"bucket", "dir/"}, "", 0).status().IsIOError());
+}
+
+TEST(S3ObjectStoreClientTest, TestInvalidListObjectsResponse) {
+    auto http = std::make_shared();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr client,
+                         MakeS3ObjectStoreClient(StaticOptions(), http));
+    for (const std::string size : {"", "abc", "-1", "12abc", "9223372036854775808"}) {
+        http->body_ =
+            "falsekey"
+            "" +
+            size + "";
+        ASSERT_TRUE(client->ListObjects({"bucket", ""}, "", 0).status().IsIOError());
+    }
+    http->body_ =
+        "falsekey"
+        "1";
+    ASSERT_TRUE(client->ListObjects({"bucket", ""}, "", 0).status().IsIOError());
+    http->body_ =
+        "key1";
+    ASSERT_TRUE(client->ListObjects({"bucket", ""}, "", 0).status().IsIOError());
+}
+
+TEST(S3FileSystemFactoryTest, TestOptionValidation) {
+    S3FileSystemFactory factory;
+    ASSERT_TRUE(
+        factory.Create("s3://bucket", {{kS3AccessKeyOption, ""}, {kS3SecretKeyOption, "secret"}})
+            .status()
+            .IsInvalid());
+    ASSERT_TRUE(
+        factory.Create("s3://bucket", {{kS3PathStyleAccessOption, "treu"}}).status().IsInvalid());
+    ASSERT_TRUE(
+        factory.Create("s3://bucket", {{kS3PathStyleAccessOption, "on"}}).status().IsInvalid());
+    ASSERT_TRUE(
+        factory.Create("s3://bucket", {{kS3SessionTokenOption, "token"}}).status().IsInvalid());
+
+    auto http = std::make_shared();
+    ASSERT_TRUE(
+        MakeS3ObjectStoreClient({{kS3AccessKeyOption, "access"}}, http).status().IsInvalid());
+    ASSERT_TRUE(
+        MakeS3ObjectStoreClient({{kS3PathStyleAccessOption, "treu"}}, http).status().IsInvalid());
+}
+
+}  // namespace
+}  // namespace paimon::s3
diff --git a/third_party/versions.txt b/third_party/versions.txt
index 17fa9b3a..d27b9e81 100644
--- a/third_party/versions.txt
+++ b/third_party/versions.txt
@@ -72,6 +72,31 @@ PAIMON_AVRO_BUILD_VERSION=c499eefb48aa2db906c7bca14a047223806f36db
 PAIMON_AVRO_BUILD_SHA256_CHECKSUM=9771f1dcfe3c01aff7ff670e873e66d3406362f71941821d482de65f3d32d780
 PAIMON_AVRO_PKG_NAME=avro-${PAIMON_AVRO_BUILD_VERSION}.tar.gz
 
+PAIMON_AWS_C_AUTH_BUILD_VERSION=v0.10.3
+PAIMON_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=20fc5e75529fadd81fd38b25f9d83798b53ab235ebbac92cdfbb716cfcc7593d
+PAIMON_AWS_C_AUTH_PKG_NAME=aws-c-auth-${PAIMON_AWS_C_AUTH_BUILD_VERSION}.tar.gz
+PAIMON_AWS_C_CAL_BUILD_VERSION=v0.9.14
+PAIMON_AWS_C_CAL_BUILD_SHA256_CHECKSUM=0e96e0067fa921768e07b5b4ebad82011ccf474903e9286419ef428d68f317ea
+PAIMON_AWS_C_CAL_PKG_NAME=aws-c-cal-${PAIMON_AWS_C_CAL_BUILD_VERSION}.tar.gz
+PAIMON_AWS_C_COMMON_BUILD_VERSION=v0.14.0
+PAIMON_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=3684076ec5da899074336722ba58a01f7166a1a2e5ad72f846f6fd468ecdf2ec
+PAIMON_AWS_C_COMMON_PKG_NAME=aws-c-common-${PAIMON_AWS_C_COMMON_BUILD_VERSION}.tar.gz
+PAIMON_AWS_C_COMPRESSION_BUILD_VERSION=v0.3.2
+PAIMON_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=f93f5a5d8b3fee3a6d97b14ba279efacd4d4016ef9cc7dc4be7d43519ecfbe93
+PAIMON_AWS_C_COMPRESSION_PKG_NAME=aws-c-compression-${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz
+PAIMON_AWS_C_HTTP_BUILD_VERSION=v0.11.0
+PAIMON_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=4ccbdd33c798b590288330dec9e93abe2ff6cfb198b7a4db036c9d362f2e6506
+PAIMON_AWS_C_HTTP_PKG_NAME=aws-c-http-${PAIMON_AWS_C_HTTP_BUILD_VERSION}.tar.gz
+PAIMON_AWS_C_IO_BUILD_VERSION=v0.27.0
+PAIMON_AWS_C_IO_BUILD_SHA256_CHECKSUM=e89a1f784e7c97e4197031ffdcf30f67d66d7c14f8a391edf5764f17dae982ee
+PAIMON_AWS_C_IO_PKG_NAME=aws-c-io-${PAIMON_AWS_C_IO_BUILD_VERSION}.tar.gz
+PAIMON_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.5
+PAIMON_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=13a03ea87aa67c7db414bf245fbcc623555c783a34d8ba1d7d701fd42717c366
+PAIMON_AWS_C_SDKUTILS_PKG_NAME=aws-c-sdkutils-${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz
+PAIMON_AWS_S2N_BUILD_VERSION=v1.7.4
+PAIMON_AWS_S2N_BUILD_SHA256_CHECKSUM=af5ce0783fd9e05ed1899fda0c76e02fa5dd92128018e5bfe71634312d2ce8e7
+PAIMON_AWS_S2N_PKG_NAME=s2n-${PAIMON_AWS_S2N_BUILD_VERSION}.zip
+
 PAIMON_FMT_BUILD_VERSION=11.2.0
 PAIMON_FMT_BUILD_SHA256_CHECKSUM=bc23066d87ab3168f27cef3e97d545fa63314f5c79df5ea444d41d56f962c6af
 PAIMON_FMT_PKG_NAME=fmt-${PAIMON_FMT_BUILD_VERSION}.tar.gz
@@ -134,6 +159,14 @@ DEPENDENCIES=(
   "PAIMON_BENCHMARK_URL ${PAIMON_BENCHMARK_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/benchmark/archive/refs/tags/v${PAIMON_BENCHMARK_BUILD_VERSION}.tar.gz"
   "PAIMON_ARROW_URL ${PAIMON_ARROW_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/arrow/releases/download/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}/apache-arrow-${PAIMON_ARROW_BUILD_VERSION}.tar.gz"
   "PAIMON_AVRO_URL ${PAIMON_AVRO_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/apache/avro/archive/${PAIMON_AVRO_BUILD_VERSION}.tar.gz"
+  "PAIMON_AWS_C_AUTH_URL ${PAIMON_AWS_C_AUTH_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-auth/archive/refs/tags/${PAIMON_AWS_C_AUTH_BUILD_VERSION}.tar.gz"
+  "PAIMON_AWS_C_CAL_URL ${PAIMON_AWS_C_CAL_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-cal/archive/refs/tags/${PAIMON_AWS_C_CAL_BUILD_VERSION}.tar.gz"
+  "PAIMON_AWS_C_COMMON_URL ${PAIMON_AWS_C_COMMON_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-common/archive/refs/tags/${PAIMON_AWS_C_COMMON_BUILD_VERSION}.tar.gz"
+  "PAIMON_AWS_C_COMPRESSION_URL ${PAIMON_AWS_C_COMPRESSION_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-compression/archive/refs/tags/${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION}.tar.gz"
+  "PAIMON_AWS_C_HTTP_URL ${PAIMON_AWS_C_HTTP_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-http/archive/refs/tags/${PAIMON_AWS_C_HTTP_BUILD_VERSION}.tar.gz"
+  "PAIMON_AWS_C_IO_URL ${PAIMON_AWS_C_IO_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-io/archive/refs/tags/${PAIMON_AWS_C_IO_BUILD_VERSION}.tar.gz"
+  "PAIMON_AWS_C_SDKUTILS_URL ${PAIMON_AWS_C_SDKUTILS_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/aws-c-sdkutils/archive/refs/tags/${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION}.tar.gz"
+  "PAIMON_AWS_S2N_URL ${PAIMON_AWS_S2N_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/aws/s2n-tls/archive/refs/tags/${PAIMON_AWS_S2N_BUILD_VERSION}.zip"
   "PAIMON_FMT_URL ${PAIMON_FMT_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/fmtlib/fmt/archive/refs/tags/${PAIMON_FMT_BUILD_VERSION}.tar.gz"
   "PAIMON_GLOG_URL ${PAIMON_GLOG_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/google/glog/archive/${PAIMON_GLOG_BUILD_VERSION}.tar.gz"
   "PAIMON_RAPIDJSON_URL ${PAIMON_RAPIDJSON_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/miloyip/rapidjson/archive/${PAIMON_RAPIDJSON_BUILD_VERSION}.tar.gz"

From 83a9c414acc49b7bc6d5c6ca18314fc803319d5c Mon Sep 17 00:00:00 2001
From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com>
Date: Tue, 28 Jul 2026 13:17:03 +0800
Subject: [PATCH 122/138] feat(parquet): support page-level (page index)
 filtering for nested column types (patch arrow)

---
 cmake_modules/arrow.diff                      | 298 ++++++-
 .../format/parquet/file_reader_wrapper.cpp    |  37 +-
 .../format/parquet/file_reader_wrapper.h      |   8 -
 .../page_filtered_row_group_reader.cpp        | 247 +++---
 .../parquet/page_filtered_row_group_reader.h  |  51 +-
 .../page_filtered_row_group_reader_test.cpp   | 730 ++++++++++++------
 .../parquet/parquet_file_batch_reader.cpp     |  15 +-
 7 files changed, 976 insertions(+), 410 deletions(-)

diff --git a/cmake_modules/arrow.diff b/cmake_modules/arrow.diff
index c50b00c1..c8aa0dd7 100644
--- a/cmake_modules/arrow.diff
+++ b/cmake_modules/arrow.diff
@@ -439,9 +439,305 @@ diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.c
 diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h
 --- a/cpp/src/arrow/io/interfaces.h
 +++ b/cpp/src/arrow/io/interfaces.h
-@@ -211,7 +211,7 @@
+@@ -210,7 +210,7 @@
    /// \brief Advance or skip stream indicated number of bytes
    /// \param[in] nbytes the number to move forward
    /// \return Status
 -  Status Advance(int64_t nbytes);
 +  virtual Status Advance(int64_t nbytes);
+
+   /// \brief Return zero-copy string_view to upcoming bytes.
+   ///
+--- a/cpp/src/parquet/arrow/reader.cc
++++ b/cpp/src/parquet/arrow/reader.cc
+@@ -254,6 +254,11 @@
+     return GetColumn(i, AllRowGroupsFactory(), out);
+   }
+
++  ::arrow::Status GetColumn(
++      int i, const std::vector& column_indices,
++      FileColumnIteratorFactory iterator_factory,
++      std::unique_ptr* out) override;
++
+   Status GetSchema(std::shared_ptr<::arrow::Schema>* out) override {
+     return FromParquetSchema(reader_->metadata()->schema(), reader_properties_,
+                              reader_->metadata()->key_value_metadata(), out);
+@@ -493,10 +498,40 @@
+
+   ::arrow::Status BuildArray(int64_t length_upper_bound,
+                              std::shared_ptr<::arrow::ChunkedArray>* out) final {
++    if (!out_) {
++      BEGIN_PARQUET_CATCH_EXCEPTIONS
++      RETURN_NOT_OK(
++          TransferColumnData(record_reader_.get(), field_, descr_, ctx_->pool, &out_));
++      END_PARQUET_CATCH_EXCEPTIONS
++    }
+     *out = out_;
+     return Status::OK();
+   }
+
++  std::vector LeafColumnIndices() const final {
++    return {input_->column_index()};
++  }
++
++  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final {
++    if (col_idx != input_->column_index()) return Status::OK();
++    BEGIN_PARQUET_CATCH_EXCEPTIONS
++    out_ = nullptr;
++    record_reader_->Reset();
++    record_reader_->Reserve(reserve);
++    return Status::OK();
++    END_PARQUET_CATCH_EXCEPTIONS
++  }
++
++  int64_t SkipRecords(int col_idx, int64_t num_records) final {
++    if (col_idx != input_->column_index() || num_records <= 0) return 0;
++    return record_reader_->SkipRecords(num_records);
++  }
++
++  int64_t ReadRecords(int col_idx, int64_t num_records) final {
++    if (col_idx != input_->column_index() || num_records <= 0) return 0;
++    return record_reader_->ReadRecords(num_records);
++  }
++
+   const std::shared_ptr field() override { return field_; }
+
+  private:
+@@ -532,6 +567,22 @@
+     return storage_reader_->LoadBatch(number_of_records);
+   }
+
++  std::vector LeafColumnIndices() const final {
++    return storage_reader_->LeafColumnIndices();
++  }
++
++  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final {
++    return storage_reader_->ResetLeaf(col_idx, reserve);
++  }
++
++  int64_t SkipRecords(int col_idx, int64_t num_records) final {
++    return storage_reader_->SkipRecords(col_idx, num_records);
++  }
++
++  int64_t ReadRecords(int col_idx, int64_t num_records) final {
++    return storage_reader_->ReadRecords(col_idx, num_records);
++  }
++
+   Status BuildArray(int64_t length_upper_bound,
+                     std::shared_ptr* out) override {
+     std::shared_ptr storage;
+@@ -576,6 +627,22 @@
+     return item_reader_->LoadBatch(number_of_records);
+   }
+
++  std::vector LeafColumnIndices() const final {
++    return item_reader_->LeafColumnIndices();
++  }
++
++  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) final {
++    return item_reader_->ResetLeaf(col_idx, reserve);
++  }
++
++  int64_t SkipRecords(int col_idx, int64_t num_records) final {
++    return item_reader_->SkipRecords(col_idx, num_records);
++  }
++
++  int64_t ReadRecords(int col_idx, int64_t num_records) final {
++    return item_reader_->ReadRecords(col_idx, num_records);
++  }
++
+   virtual ::arrow::Result> AssembleArray(
+       std::shared_ptr data) {
+     if (field_->type()->id() == ::arrow::Type::MAP) {
+@@ -709,6 +776,39 @@
+     }
+     return Status::OK();
+   }
++
++  std::vector LeafColumnIndices() const override {
++    std::vector indices;
++    for (const std::unique_ptr& reader : children_) {
++      std::vector child_indices = reader->LeafColumnIndices();
++      indices.insert(indices.end(), child_indices.begin(), child_indices.end());
++    }
++    return indices;
++  }
++
++  ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) override {
++    for (const std::unique_ptr& reader : children_) {
++      RETURN_NOT_OK(reader->ResetLeaf(col_idx, reserve));
++    }
++    return Status::OK();
++  }
++
++  int64_t SkipRecords(int col_idx, int64_t num_records) override {
++    int64_t skipped = 0;
++    for (const std::unique_ptr& reader : children_) {
++      skipped += reader->SkipRecords(col_idx, num_records);
++    }
++    return skipped;
++  }
++
++  int64_t ReadRecords(int col_idx, int64_t num_records) override {
++    int64_t read = 0;
++    for (const std::unique_ptr& reader : children_) {
++      read += reader->ReadRecords(col_idx, num_records);
++    }
++    return read;
++  }
++
+   Status BuildArray(int64_t length_upper_bound,
+                     std::shared_ptr* out) override;
+   Status GetDefLevels(const int16_t** data, int64_t* length) override;
+@@ -1228,6 +1328,23 @@
+   std::unique_ptr result;
+   RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result));
+   *out = std::move(result);
++  return Status::OK();
++}
++
++::arrow::Status FileReaderImpl::GetColumn(
++    int i, const std::vector& column_indices,
++    FileColumnIteratorFactory iterator_factory,
++    std::unique_ptr* out) {
++  RETURN_NOT_OK(BoundsCheckColumn(i));
++  auto ctx = std::make_shared();
++  ctx->reader = reader_.get();
++  ctx->pool = pool_;
++  ctx->iterator_factory = iterator_factory;
++  ctx->filter_leaves = true;
++  ctx->included_leaves = VectorToSharedSet(column_indices);
++  std::unique_ptr result;
++  RETURN_NOT_OK(GetReader(manifest_.schema_fields[i], ctx, &result));
++  *out = std::move(result);
+   return Status::OK();
+ }
+
+--- a/cpp/src/parquet/arrow/reader.h
++++ b/cpp/src/parquet/arrow/reader.h
+@@ -21,6 +21,7 @@
+ // N.B. we don't include async_generator.h as it's relatively heavy
+ #include 
+ #include 
++#include 
+ #include 
+
+ #include "parquet/file_reader.h"
+@@ -48,9 +49,13 @@
+
+ class ColumnChunkReader;
+ class ColumnReader;
++class FileColumnIterator;
+ struct SchemaManifest;
+ class RowGroupReader;
+
++using FileColumnIteratorFactory =
++    std::function;
++
+ /// \brief Arrow read adapter class for deserializing Parquet files as Arrow row batches.
+ ///
+ /// This interfaces caters for different use cases and thus provides different
+@@ -136,6 +141,27 @@
+   // The indicated column index is relative to the schema
+   virtual ::arrow::Status GetColumn(int i, std::unique_ptr* out) = 0;
+
++  /// \brief Return a ColumnReader with a custom FileColumnIteratorFactory
++  /// and leaf column filtering.
++  ///
++  /// This allows callers to customize page reading behavior (e.g., setting
++  /// data_page_filter for page-level skipping) and to select only specific
++  /// leaf columns within a nested field. The factory is called once per leaf
++  /// column included in column_indices.
++  ///
++  /// \param i top-level field index (same as GetColumn(int i, ...))
++  /// \param column_indices leaf column indices to include (enables sub-column
++  ///        projection within nested types)
++  /// \param iterator_factory factory to create FileColumnIterator per leaf
++  /// \param[out] out the ColumnReader (may be nullptr if all leaves are pruned)
++  virtual ::arrow::Status GetColumn(
++      int i, const std::vector& column_indices,
++      FileColumnIteratorFactory iterator_factory,
++      std::unique_ptr* out) {
++    return ::arrow::Status::NotImplemented(
++        "GetColumn with factory not implemented");
++  }
++
+   /// \brief Return arrow schema for all the columns.
+   virtual ::arrow::Status GetSchema(std::shared_ptr<::arrow::Schema>* out) = 0;
+
+@@ -316,6 +342,43 @@
+   // the data available in the file.
+   virtual ::arrow::Status NextBatch(int64_t batch_size,
+                                     std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
++
++  /// \brief Leaf column indices covered by this (sub)tree, in leaf order.
++  ///
++  /// Used to drive per-leaf row filtering: after page-level skipping each leaf
++  /// lives in its own compressed coordinate space, so callers must reset and
++  /// skip/read each leaf independently rather than in lockstep.
++  virtual std::vector LeafColumnIndices() const { return {}; }
++
++  /// \brief Reset the leaf identified by col_idx and reserve space for
++  /// `reserve` records (in that leaf's post-page-filter compressed space).
++  /// Must be called before SkipRecords()/ReadRecords() for that leaf, and
++  /// followed by BuildArray() to get the result.
++  virtual ::arrow::Status ResetLeaf(int col_idx, int64_t reserve) {
++    return ::arrow::Status::NotImplemented("ResetLeaf not implemented");
++  }
++
++  /// \brief Skip num_records on the leaf identified by col_idx and return the
++  /// number of records actually skipped. Returns 0 when num_records <= 0 or
++  /// col_idx does not belong to this (sub)tree. May throw ParquetException on a
++  /// decode error; callers convert it to Status at the public boundary.
++  virtual int64_t SkipRecords(int col_idx, int64_t num_records) { return 0; }
++
++  /// \brief Read num_records on the leaf identified by col_idx and return the
++  /// number of records actually read. Values accumulate across successive calls
++  /// until BuildArray() is called. Returns 0 when num_records <= 0 or col_idx
++  /// does not belong to this (sub)tree. May throw ParquetException on a decode
++  /// error; callers convert it to Status at the public boundary.
++  virtual int64_t ReadRecords(int col_idx, int64_t num_records) { return 0; }
++
++  /// \brief Build the Arrow array from previously loaded data.
++  /// For leaf readers, calls TransferColumnData if not already done.
++  /// For nested readers, assembles the nested array from child arrays.
++  virtual ::arrow::Status BuildArray(
++      int64_t length_upper_bound,
++      std::shared_ptr<::arrow::ChunkedArray>* out) {
++    return ::arrow::Status::NotImplemented("BuildArray not implemented");
++  }
+ };
+
+ /// \brief Experimental helper class for bindings (like Python) that struggle
+--- a/cpp/src/parquet/arrow/reader_internal.h
++++ b/cpp/src/parquet/arrow/reader_internal.h
+@@ -26,6 +26,7 @@
+ #include 
+ #include 
+
++#include "parquet/arrow/reader.h"
+ #include "parquet/arrow/schema.h"
+ #include "parquet/column_reader.h"
+ #include "parquet/file_reader.h"
+@@ -70,7 +71,10 @@
+
+   virtual ~FileColumnIterator() {}
+
+-  std::unique_ptr<::parquet::PageReader> NextChunk() {
++  /// \brief Fetch the PageReader for the next row group in this iterator's
++  /// range. Virtual so subclasses can decorate the returned PageReader, e.g.
++  /// to install a data_page_filter for I/O-level page skipping.
++  virtual std::unique_ptr<::parquet::PageReader> NextChunk() {
+     if (row_groups_.empty()) {
+       return nullptr;
+     }
+@@ -95,9 +99,6 @@
+   std::deque row_groups_;
+ };
+
+-using FileColumnIteratorFactory =
+-    std::function;
+-
+ Status TransferColumnData(::parquet::internal::RecordReader* reader,
+                           const std::shared_ptr<::arrow::Field>& value_field,
+                           const ColumnDescriptor* descr, ::arrow::MemoryPool* pool,
diff --git a/src/paimon/format/parquet/file_reader_wrapper.cpp b/src/paimon/format/parquet/file_reader_wrapper.cpp
index 0f553c96..39664658 100644
--- a/src/paimon/format/parquet/file_reader_wrapper.cpp
+++ b/src/paimon/format/parquet/file_reader_wrapper.cpp
@@ -32,6 +32,7 @@
 #include "paimon/format/parquet/parquet_format_defs.h"
 #include "paimon/macros.h"
 #include "parquet/arrow/reader.h"
+#include "parquet/arrow/schema.h"
 #include "parquet/file_reader.h"
 #include "parquet/metadata.h"
 #include "parquet/page_index.h"
@@ -232,15 +233,14 @@ Result> FileReaderWrapper::NextPageFiltered(
     if (!current_page_filtered_reader_) {
         const auto& target_rg = target_row_groups_[current_row_group_idx_];
         auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges(
-            file_reader_->parquet_reader(), target_rg, target_column_indices_);
+            target_rg, target_column_indices_, file_reader_->parquet_reader());
         bool pre_buffered = !prebuffered_ranges_.empty();
         int64_t max_chunksize = batch_size_ > 0 ? batch_size_ : std::numeric_limits::max();
         PAIMON_ASSIGN_OR_RAISE(
             current_page_filtered_reader_,
             PageFilteredRowGroupReader::ReadFilteredRowGroup(
-                file_reader_->parquet_reader(), target_rg, target_column_indices_,
-                page_filtered_read_schema_, file_reader_->properties().cache_options(),
-                pre_buffered, page_ranges, max_chunksize, pool_));
+                target_rg, target_column_indices_, file_reader_->properties().cache_options(),
+                pre_buffered, page_ranges, max_chunksize, pool_, file_reader_.get()));
         current_filtered_row_ranges_ = target_rg.GetRowRanges();
         current_filtered_rg_start_ = all_row_group_ranges_[rg_id].first;
         filtered_global_offset_ = 0;
@@ -347,29 +347,6 @@ Status FileReaderWrapper::PrepareForReadingLazy(
     return Status::OK();
 }
 
-Status FileReaderWrapper::BuildPageFilteredSchema(const std::vector& column_indices) {
-    if (page_filtered_read_schema_) {
-        return Status::OK();
-    }
-    std::shared_ptr schema;
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(file_reader_->GetSchema(&schema));
-    auto parquet_schema = file_reader_->parquet_reader()->metadata()->schema();
-    std::vector> fields;
-    for (int32_t col_idx : column_indices) {
-        const std::string& col_name = parquet_schema->Column(col_idx)->name();
-        auto field = schema->GetFieldByName(col_name);
-        if (!field) {
-            return Status::Invalid(fmt::format(
-                "PrepareForReading: Parquet column {} ('{}') has no matching Arrow field in "
-                "file schema",
-                col_idx, col_name));
-        }
-        fields.push_back(field);
-    }
-    page_filtered_read_schema_ = arrow::schema(fields);
-    return Status::OK();
-}
-
 std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges(
     const std::vector& column_indices) {
     std::vector<::arrow::io::ReadRange> ranges;
@@ -381,7 +358,7 @@ std::vector<::arrow::io::ReadRange> FileReaderWrapper::CollectPreBufferRanges(
         if (trg.IsPartiallyMatched()) {
             // Page-filtered RGs: only matching page byte ranges.
             auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges(
-                file_reader_->parquet_reader(), trg, column_indices);
+                trg, column_indices, file_reader_->parquet_reader());
             ranges.insert(ranges.end(), std::make_move_iterator(page_ranges.begin()),
                           std::make_move_iterator(page_ranges.end()));
         } else {
@@ -418,7 +395,6 @@ Status FileReaderWrapper::PrepareForReading(const std::vector& t
     try {
         target_row_groups_ = target_row_groups;
         target_column_indices_ = column_indices;
-        page_filtered_read_schema_.reset();
 
         // Partition into fully-matched and page-filtered row groups, skipping excluded ones.
         std::vector fully_matched_row_groups;
@@ -434,9 +410,6 @@ Status FileReaderWrapper::PrepareForReading(const std::vector& t
         }
 
         bool has_partially_matched = fully_matched_row_groups.size() != active_count;
-        if (has_partially_matched) {
-            PAIMON_RETURN_NOT_OK(BuildPageFilteredSchema(column_indices));
-        }
 
         WaitForPendingPreBuffer();
 
diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h
index ddde5835..c62c1b96 100644
--- a/src/paimon/format/parquet/file_reader_wrapper.h
+++ b/src/paimon/format/parquet/file_reader_wrapper.h
@@ -160,9 +160,6 @@ class FileReaderWrapper {
     /// Read next batch from the fully-matched batch_reader_. Returns nullptr when exhausted.
     Result> NextFullyMatched();
 
-    /// Build page_filtered_read_schema_ from the given column indices. No-op if already built.
-    Status BuildPageFilteredSchema(const std::vector& column_indices);
-
     /// Collect all byte ranges that need pre-buffering (page-filtered + fully-matched).
     std::vector<::arrow::io::ReadRange> CollectPreBufferRanges(
         const std::vector& column_indices);
@@ -196,11 +193,6 @@ class FileReaderWrapper {
     // Target row groups with row ranges for none page-level filtering and page-level filtering
     std::vector target_row_groups_;
 
-    // Arrow schema covering target_column_indices_, used when constructing the per-RG
-    // page-filtered reader. Cached in PrepareForReading because it's identical across
-    // all page-filtered RGs in a session.
-    std::shared_ptr page_filtered_read_schema_;
-
     // Track pre-buffered ranges so we can wait on destruction
     std::vector<::arrow::io::ReadRange> prebuffered_ranges_;
 };
diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
index 9197fef0..6b8f9a26 100644
--- a/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
+++ b/src/paimon/format/parquet/page_filtered_row_group_reader.cpp
@@ -31,7 +31,9 @@
 #include "fmt/format.h"
 #include "paimon/common/utils/arrow/arrow_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "parquet/arrow/reader.h"
 #include "parquet/arrow/reader_internal.h"
+#include "parquet/arrow/schema.h"
 #include "parquet/metadata.h"
 #include "parquet/schema.h"
 
@@ -77,6 +79,29 @@ class TableRecordBatchReader : public arrow::RecordBatchReader {
     std::shared_ptr pool_;
 };
 
+/// A FileColumnIterator that installs a data_page_filter on every PageReader it
+/// produces, enabling I/O-level page skipping. The base class handles row group
+/// iteration; this subclass only decorates the PageReader returned by NextChunk().
+class PageFilteringColumnIterator : public ::parquet::arrow::FileColumnIterator {
+ public:
+    PageFilteringColumnIterator(
+        int column_index, ::parquet::ParquetFileReader* reader, std::vector row_groups,
+        std::function data_page_filter)
+        : FileColumnIterator(column_index, reader, std::move(row_groups)),
+          data_page_filter_(std::move(data_page_filter)) {}
+
+    std::unique_ptr<::parquet::PageReader> NextChunk() override {
+        std::unique_ptr<::parquet::PageReader> page_reader = FileColumnIterator::NextChunk();
+        if (page_reader && data_page_filter_) {
+            page_reader->set_data_page_filter(data_page_filter_);
+        }
+        return page_reader;
+    }
+
+ private:
+    std::function data_page_filter_;
+};
+
 }  // namespace
 
 std::pair PageFilteredRowGroupReader::GetPageRowRange(
@@ -142,92 +167,35 @@ std::pair PageFilteredRowGroupReader::ComputeCompressedRowRa
 }
 
 Status PageFilteredRowGroupReader::ExecuteSkipReadPattern(
-    const std::shared_ptr<::parquet::internal::RecordReader>& record_reader,
-    const RowRanges& ranges, int64_t total_row_count, int32_t row_group_index,
-    int32_t column_index) {
-    int64_t current_row = 0;
+    int col_idx, const RowRanges& ranges, int64_t total,
+    ::parquet::arrow::ColumnReader* column_reader) {
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(column_reader->ResetLeaf(col_idx, total));
+    int64_t current = 0;
     for (const auto& range : ranges.GetRanges()) {
-        if (range.from > current_row) {
-            int64_t to_skip = range.from - current_row;
-            int64_t skipped = record_reader->SkipRecords(to_skip);
-            if (skipped != to_skip) {
-                return Status::Invalid(fmt::format(
-                    "PageFilteredRowGroupReader: expected to skip {} records but skipped {} "
-                    "(row_group={}, column={})",
-                    to_skip, skipped, row_group_index, column_index));
-            }
-            current_row = range.from;
+        int64_t skip = range.from > current ? range.from - current : 0;
+        int64_t skipped = column_reader->SkipRecords(col_idx, skip);
+        if (skipped != skip) {
+            return Status::Invalid(fmt::format(
+                "PageFilteredRowGroupReader: leaf {} expected to skip {} records but skipped {}",
+                col_idx, skip, skipped));
         }
         int64_t to_read = range.Count();
-        int64_t read = record_reader->ReadRecords(to_read);
+        int64_t read = column_reader->ReadRecords(col_idx, to_read);
         if (read != to_read) {
-            return Status::Invalid(
-                fmt::format("PageFilteredRowGroupReader: expected to read {} records but read {} "
-                            "(row_group={}, column={}, range=[{},{}])",
-                            to_read, read, row_group_index, column_index, range.from, range.to));
+            return Status::Invalid(fmt::format(
+                "PageFilteredRowGroupReader: leaf {} expected to read {} records but read {}",
+                col_idx, to_read, read));
         }
-        current_row += to_read;
-    }
-    if (current_row < total_row_count) {
-        record_reader->SkipRecords(total_row_count - current_row);
+        current = range.to + 1;
     }
     return Status::OK();
 }
 
-Result> PageFilteredRowGroupReader::ReadFilteredColumn(
-    const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader,
-    ::parquet::ParquetFileReader* parquet_reader,
-    const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader,
-    int32_t row_group_index, int32_t column_index, const RowRanges& row_ranges,
-    const std::shared_ptr& field, int64_t row_group_row_count,
-    std::shared_ptr<::arrow::MemoryPool> pool) {
-    auto file_metadata = parquet_reader->metadata();
-    const auto* col_descriptor = file_metadata->schema()->Column(column_index);
-
-    // Try to get OffsetIndex for I/O-level page skipping
-    RowRanges effective_ranges = row_ranges;
-    int64_t effective_row_count = row_group_row_count;
-
-    std::shared_ptr<::parquet::OffsetIndex> offset_index;
-    if (rg_page_index_reader) {
-        offset_index = rg_page_index_reader->GetOffsetIndex(column_index);
-    }
-
-    auto page_reader = row_group_reader->GetColumnPageReader(column_index);
-
-    if (offset_index) {
-        // Set data_page_filter for I/O-level page skipping
-        page_reader->set_data_page_filter(
-            MakePageFilter(row_ranges, offset_index, row_group_row_count));
-        // Compute compressed RowRanges for the decode-level skip/read pattern
-        auto [compressed_ranges, compressed_total] =
-            ComputeCompressedRowRanges(row_ranges, offset_index, row_group_row_count);
-        effective_ranges = std::move(compressed_ranges);
-        effective_row_count = compressed_total;
-    }
-
-    // Create RecordReader
-    ::parquet::internal::LevelInfo leaf_info =
-        ::parquet::internal::LevelInfo::ComputeLevelInfo(col_descriptor);
-    auto record_reader =
-        ::parquet::internal::RecordReader::Make(col_descriptor, leaf_info, pool.get());
-    record_reader->SetPageReader(std::move(page_reader));
-
-    PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern(
-        record_reader, effective_ranges, effective_row_count, row_group_index, column_index));
-
-    std::shared_ptr chunked_array;
-    PAIMON_RETURN_NOT_OK_FROM_ARROW(::parquet::arrow::TransferColumnData(
-        record_reader.get(), field, col_descriptor, pool.get(), &chunked_array));
-
-    return chunked_array;
-}
-
 Status PageFilteredRowGroupReader::WaitForPreBuffer(
-    ::parquet::ParquetFileReader* parquet_reader, int32_t row_group_index,
-    const std::vector& column_indices, const ::arrow::io::CacheOptions& cache_options,
-    bool pre_buffered, const std::vector<::arrow::io::ReadRange>& page_ranges,
-    std::shared_ptr<::arrow::MemoryPool> pool) {
+    int32_t row_group_index, const std::vector& column_indices,
+    const ::arrow::io::CacheOptions& cache_options, bool pre_buffered,
+    const std::vector<::arrow::io::ReadRange>& page_ranges,
+    std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::ParquetFileReader* parquet_reader) {
     std::vector rg_vec = {row_group_index};
     std::vector col_vec(column_indices.begin(), column_indices.end());
     if (!pre_buffered) {
@@ -247,28 +215,84 @@ Status PageFilteredRowGroupReader::WaitForPreBuffer(
     return Status::OK();
 }
 
+Result> PageFilteredRowGroupReader::ReadFilteredField(
+    const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader,
+    int32_t row_group_index, int32_t field_index, const std::vector& column_indices,
+    const RowRanges& row_ranges, int64_t row_group_row_count,
+    ::parquet::arrow::FileReader* arrow_file_reader) {
+    // Factory: set data_page_filter on every leaf (per-leaf OffsetIndex).
+    // data_page_filter enables I/O-level page skipping for all leaves.
+    auto factory =
+        [row_group_index, &rg_page_index_reader, &row_ranges, row_group_row_count](
+            int col_idx,
+            ::parquet::ParquetFileReader* reader) -> ::parquet::arrow::FileColumnIterator* {
+        std::function data_page_filter;
+        if (rg_page_index_reader) {
+            auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx);
+            if (offset_index) {
+                data_page_filter = MakePageFilter(row_ranges, offset_index, row_group_row_count);
+            }
+        }
+        return new PageFilteringColumnIterator(col_idx, reader, std::vector{row_group_index},
+                                               std::move(data_page_filter));
+    };
+
+    // Build reader tree with leaf column filtering
+    std::unique_ptr<::parquet::arrow::ColumnReader> column_reader;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        arrow_file_reader->GetColumn(field_index, column_indices, factory, &column_reader));
+
+    if (!column_reader) {
+        return Status::Invalid(
+            fmt::format("PageFilteredRowGroupReader: field {} has no matching leaf columns "
+                        "(row_group={})",
+                        field_index, row_group_index));
+    }
+
+    // Since leaf columns may have misaligned pages, we compute compressed row ranges and drive each
+    // leaf column independently
+
+    for (int col_idx : column_reader->LeafColumnIndices()) {
+        RowRanges effective_ranges = row_ranges;
+        int64_t effective_total = row_group_row_count;
+        if (rg_page_index_reader) {
+            auto offset_index = rg_page_index_reader->GetOffsetIndex(col_idx);
+            if (offset_index) {
+                auto [compressed, total] =
+                    ComputeCompressedRowRanges(row_ranges, offset_index, row_group_row_count);
+                effective_ranges = std::move(compressed);
+                effective_total = total;
+            }
+        }
+
+        PAIMON_RETURN_NOT_OK(ExecuteSkipReadPattern(col_idx, effective_ranges, effective_total,
+                                                    column_reader.get()));
+    }
+
+    // Build the Arrow array (TransferColumnData for leaves + assemble for nested)
+    std::shared_ptr chunked_array;
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        column_reader->BuildArray(row_ranges.RowCount(), &chunked_array));
+
+    return chunked_array;
+}
+
 Result> PageFilteredRowGroupReader::ReadFilteredRowGroup(
-    ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group,
-    const std::vector& column_indices, const std::shared_ptr& arrow_schema,
+    const TargetRowGroup& target_row_group, const std::vector& column_indices,
     const ::arrow::io::CacheOptions& cache_options, bool pre_buffered,
     const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize,
-    std::shared_ptr<::arrow::MemoryPool> pool) {
+    std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::arrow::FileReader* arrow_file_reader) {
+    auto parquet_reader = arrow_file_reader->parquet_reader();
     const auto& row_ranges = target_row_group.GetRowRanges();
     int32_t row_group_index = target_row_group.GetRowGroupIndex();
 
-    if (row_ranges.IsEmpty()) {
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr empty_table,
-                                          arrow::Table::MakeEmpty(arrow_schema, pool.get()));
-        return std::make_unique(std::move(empty_table), max_chunksize,
-                                                        pool);
-    }
-
     int64_t expected_rows = row_ranges.RowCount();
 
-    PAIMON_RETURN_NOT_OK(WaitForPreBuffer(parquet_reader, row_group_index, column_indices,
-                                          cache_options, pre_buffered, page_ranges, pool));
+    if (!row_ranges.IsEmpty()) {
+        PAIMON_RETURN_NOT_OK(WaitForPreBuffer(row_group_index, column_indices, cache_options,
+                                              pre_buffered, page_ranges, pool, parquet_reader));
+    }
 
-    auto row_group_reader = parquet_reader->RowGroup(row_group_index);
     auto rg_metadata = parquet_reader->metadata()->RowGroup(row_group_index);
     int64_t row_group_row_count = rg_metadata->num_rows();
 
@@ -280,36 +304,45 @@ Result> PageFilteredRowGroupReader::Re
         rg_page_index_reader = page_index_reader->RowGroup(row_group_index);
     }
 
-    // Read each column with page filtering
-    std::vector> columns;
-    columns.reserve(column_indices.size());
+    const auto& manifest = arrow_file_reader->manifest();
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+        std::vector field_indices,
+        manifest.GetFieldIndices(std::vector(column_indices.begin(), column_indices.end())));
 
-    for (size_t i = 0; i < column_indices.size(); ++i) {
+    std::vector> result_arrays;
+    result_arrays.reserve(field_indices.size());
+
+    for (int field_idx : field_indices) {
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr chunked_array,
-            ReadFilteredColumn(row_group_reader, parquet_reader, rg_page_index_reader,
-                               row_group_index, column_indices[i], row_ranges,
-                               arrow_schema->field(static_cast(i)), row_group_row_count,
-                               pool));
+            ReadFilteredField(rg_page_index_reader, row_group_index, field_idx, column_indices,
+                              row_ranges, row_group_row_count, arrow_file_reader));
 
         if (chunked_array->length() != expected_rows) {
-            return Status::Invalid(fmt::format(
-                "PageFilteredRowGroupReader: column {} produced {} rows but expected {} "
-                "(row_group={})",
-                column_indices[i], chunked_array->length(), expected_rows, row_group_index));
+            return Status::Invalid(
+                fmt::format("PageFilteredRowGroupReader: field {} produced {} rows but expected {} "
+                            "(row_group={})",
+                            field_idx, chunked_array->length(), expected_rows, row_group_index));
         }
 
-        columns.push_back(std::move(chunked_array));
+        result_arrays.push_back(std::move(chunked_array));
+    }
+
+    std::vector> result_fields;
+    for (size_t i = 0; i < result_arrays.size(); ++i) {
+        const auto& field = manifest.schema_fields[field_indices[i]].field;
+        result_fields.push_back(arrow::field(field->name(), result_arrays[i]->type(),
+                                             field->nullable(), field->metadata()));
     }
+    auto result_schema = arrow::schema(result_fields);
 
-    auto table = arrow::Table::Make(arrow_schema, std::move(columns), expected_rows);
-    return std::make_unique(std::move(table), max_chunksize,
-                                                    std::move(pool));
+    auto table = arrow::Table::Make(result_schema, std::move(result_arrays), expected_rows);
+    return std::make_unique(std::move(table), max_chunksize, pool);
 }
 
 std::vector<::arrow::io::ReadRange> PageFilteredRowGroupReader::ComputePageRanges(
-    ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group,
-    const std::vector& column_indices) {
+    const TargetRowGroup& target_row_group, const std::vector& column_indices,
+    ::parquet::ParquetFileReader* parquet_reader) {
     int32_t row_group_index = target_row_group.GetRowGroupIndex();
     const auto& row_ranges = target_row_group.GetRowRanges();
 
diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader.h b/src/paimon/format/parquet/page_filtered_row_group_reader.h
index 6b8faf3d..1122ece1 100644
--- a/src/paimon/format/parquet/page_filtered_row_group_reader.h
+++ b/src/paimon/format/parquet/page_filtered_row_group_reader.h
@@ -32,6 +32,7 @@
 #include "paimon/format/parquet/row_ranges.h"
 #include "paimon/format/parquet/target_row_group.h"
 #include "paimon/result.h"
+#include "parquet/arrow/reader.h"
 #include "parquet/column_reader.h"
 #include "parquet/file_reader.h"
 #include "parquet/page_index.h"
@@ -41,39 +42,35 @@ namespace paimon::parquet {
 /// Reads a single row group using page-level filtering.
 /// Non-matching rows are skipped at the decoding level via RecordReader::SkipRecords,
 /// using RowRanges computed from the page index (ColumnIndex + OffsetIndex).
-/// MakePageFilter is available for future I/O-level page skipping optimization.
 class PageFilteredRowGroupReader {
  public:
     PageFilteredRowGroupReader() = delete;
     ~PageFilteredRowGroupReader() = delete;
 
     /// Read a row group with page-level filtering.
-    /// @param parquet_reader The underlying ParquetFileReader
     /// @param target_row_group Target row group with index and row ranges
     /// @param column_indices Leaf column indices to read
-    /// @param arrow_schema The target Arrow schema for output columns
     /// @param pool Memory pool
     /// @param cache_options Cache options for PreBuffer
     /// @param pre_buffered If true, assumes PreBuffer was already called externally
     ///        and only waits via WhenBuffered (no redundant PreBuffer).
     /// @param page_ranges If non-empty, wait via WhenBufferedRanges instead of WhenBuffered
     /// @param max_chunksize Per-batch row cap for the returned reader.
+    /// @param arrow_file_reader The Arrow FileReader for ColumnReader tree creation
     /// @return A RecordBatchReader streaming the filtered rows.
     static Result> ReadFilteredRowGroup(
-        ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group,
-        const std::vector& column_indices,
-        const std::shared_ptr& arrow_schema,
+        const TargetRowGroup& target_row_group, const std::vector& column_indices,
         const ::arrow::io::CacheOptions& cache_options, bool pre_buffered,
         const std::vector<::arrow::io::ReadRange>& page_ranges, int64_t max_chunksize,
-        std::shared_ptr<::arrow::MemoryPool> pool);
+        std::shared_ptr<::arrow::MemoryPool> pool, ::parquet::arrow::FileReader* arrow_file_reader);
 
     /// Compute the byte ranges of pages that overlap with the given RowRanges.
     /// Uses OffsetIndex to determine per-page file offsets and sizes.
     /// Includes dictionary pages unconditionally.
     /// Falls back to entire column chunk range if OffsetIndex is unavailable.
     static std::vector<::arrow::io::ReadRange> ComputePageRanges(
-        ::parquet::ParquetFileReader* parquet_reader, const TargetRowGroup& target_row_group,
-        const std::vector& column_indices);
+        const TargetRowGroup& target_row_group, const std::vector& column_indices,
+        ::parquet::ParquetFileReader* parquet_reader);
 
  private:
     /// Get the [first_row, last_row] range of a page given page locations.
@@ -82,38 +79,38 @@ class PageFilteredRowGroupReader {
         int64_t row_group_row_count);
 
     /// Wait for pre-buffered data to become available before reading.
-    static Status WaitForPreBuffer(::parquet::ParquetFileReader* parquet_reader,
-                                   int32_t row_group_index,
+    static Status WaitForPreBuffer(int32_t row_group_index,
                                    const std::vector& column_indices,
                                    const ::arrow::io::CacheOptions& cache_options,
                                    bool pre_buffered,
                                    const std::vector<::arrow::io::ReadRange>& page_ranges,
-                                   std::shared_ptr<::arrow::MemoryPool> pool);
-
-    /// Execute the skip/read pattern on a RecordReader based on RowRanges.
-    static Status ExecuteSkipReadPattern(
-        const std::shared_ptr<::parquet::internal::RecordReader>& record_reader,
-        const RowRanges& ranges, int64_t total_row_count, int32_t row_group_index,
-        int32_t column_index);
+                                   std::shared_ptr<::arrow::MemoryPool> pool,
+                                   ::parquet::ParquetFileReader* parquet_reader);
 
     /// Create a data_page_filter callback for a column based on RowRanges + OffsetIndex.
     static std::function MakePageFilter(
         const RowRanges& row_ranges, const std::shared_ptr<::parquet::OffsetIndex>& offset_index,
         int64_t row_group_row_count);
 
-    /// Read a single column using skip/read pattern driven by RowRanges.
-    static Result> ReadFilteredColumn(
-        const std::shared_ptr<::parquet::RowGroupReader>& row_group_reader,
-        ::parquet::ParquetFileReader* parquet_reader,
-        const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader,
-        int32_t row_group_index, int32_t column_index, const RowRanges& row_ranges,
-        const std::shared_ptr& field, int64_t row_group_row_count,
-        std::shared_ptr<::arrow::MemoryPool> pool);
-
     /// Compute compressed RowRanges after data_page_filter skips non-matching pages.
     static std::pair ComputeCompressedRowRanges(
         const RowRanges& original_ranges,
         const std::shared_ptr<::parquet::OffsetIndex>& offset_index, int64_t row_group_row_count);
+
+    /// Reset the given leaf and replay the skip/read pattern derived from `ranges`
+    /// directly against the ColumnReader (ResetLeaf + SkipRecords/ReadRecords).
+    static Status ExecuteSkipReadPattern(int col_idx, const RowRanges& ranges, int64_t total,
+                                         ::parquet::arrow::ColumnReader* column_reader);
+
+    /// Read a field (flat or nested) using ColumnReader tree.
+    /// Sets data_page_filter on all leaves via factory, then drives each leaf
+    /// independently via ResetLeaf/SkipRecords/ReadRecords using its own
+    /// compressed_ranges.
+    static Result> ReadFilteredField(
+        const std::shared_ptr<::parquet::RowGroupPageIndexReader>& rg_page_index_reader,
+        int32_t row_group_index, int32_t field_index, const std::vector& column_indices,
+        const RowRanges& row_ranges, int64_t row_group_row_count,
+        ::parquet::arrow::FileReader* arrow_file_reader);
 };
 
 }  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp
index e5a4c0e3..2ccdc4e4 100644
--- a/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp
+++ b/src/paimon/format/parquet/page_filtered_row_group_reader_test.cpp
@@ -20,6 +20,7 @@
 #include "paimon/format/parquet/page_filtered_row_group_reader.h"
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -79,7 +80,7 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test {
     void WriteTestFile(const std::string& file_name,
                        const std::shared_ptr& struct_array,
                        int32_t write_batch_size, int64_t max_row_group_length,
-                       bool enable_dictionary = false) {
+                       bool enable_dictionary = false, int64_t data_page_size = 1) {
         auto data_type = struct_array->struct_type();
         auto data_schema = arrow::schema(data_type->fields());
         auto data_arrow_array = std::make_unique();
@@ -95,10 +96,12 @@ class PageFilteredRowGroupReaderTest : public ::testing::Test {
             builder.disable_dictionary();  // Ensure page index min/max are meaningful
         }
         builder.enable_write_page_index();  // Enable page index for page-level filtering
-        // Set data page size to 1 byte to force a new page after every write_batch_size rows.
-        // The writer flushes a page when accumulated data exceeds data_pagesize, so setting
-        // it to 1 ensures each batch of write_batch_size rows becomes exactly one page.
-        builder.data_pagesize(1);
+        // Data page size controls when a page is flushed. The default of 1 byte forces a new
+        // page after every write_batch_size rows (each batch becomes one page), giving pages
+        // aligned across columns. A larger byte-based value combined with write_batch_size=1
+        // instead lets columns of different physical widths flush pages at different row
+        // counts, producing intentionally misaligned pages across leaves.
+        builder.data_pagesize(data_page_size);
         auto writer_properties = builder.build();
         ASSERT_OK_AND_ASSIGN(
             auto format_writer,
@@ -554,9 +557,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesPartialMatch) {
     row_ranges.Add(RowRanges::Range(50, 59));
 
     auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
-        parquet_reader.get(),
         TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges),
-        /*column_indices=*/{0});
+        /*column_indices=*/{0}, parquet_reader.get());
 
     // Should have exactly 1 range (page 5 of column 0, no dictionary since disabled)
     ASSERT_EQ(1, ranges.size());
@@ -580,8 +582,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesAllMatch) {
     row_ranges.Add(RowRanges::Range(0, 99));
 
     auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
-        parquet_reader.get(),
-        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0});
+        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0},
+        parquet_reader.get());
 
     // 10 pages, all matching
     ASSERT_EQ(10, ranges.size());
@@ -605,8 +607,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesNoMatch) {
     RowRanges row_ranges;  // empty
 
     auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
-        parquet_reader.get(),
-        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0});
+        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0},
+        parquet_reader.get());
 
     ASSERT_EQ(0, ranges.size());
 }
@@ -627,9 +629,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiColumn) {
     row_ranges.Add(RowRanges::Range(50, 59));
 
     auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
-        parquet_reader.get(),
         TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges),
-        {0, 1});
+        {0, 1}, parquet_reader.get());
 
     // 1 matching page per column = 2 ranges total
     ASSERT_EQ(2, ranges.size());
@@ -655,8 +656,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) {
     row_ranges.Add(RowRanges::Range(70, 79));
 
     auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
-        parquet_reader.get(),
-        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0});
+        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges), {0},
+        parquet_reader.get());
 
     // 2 matching pages for 1 column
     ASSERT_EQ(2, ranges.size());
@@ -840,9 +841,8 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding)
     row_ranges.Add(RowRanges::Range(0, 99));
 
     auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
-        parquet_reader.get(),
         TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, /*ranges=*/row_ranges),
-        /*column_indices=*/{0});
+        /*column_indices=*/{0}, parquet_reader.get());
 
     ASSERT_FALSE(ranges.empty());
 
@@ -917,56 +917,56 @@ TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding)
     auto partial_concat = arrow::Concatenate(result_partial->chunks()).ValueOrDie();
     ASSERT_TRUE(partial_concat->Equals(expected_struct));
 }
-/// Helper: build a StructArray with a top-level int32 "id" column and a nested struct column
-/// "info" containing two int32 fields: "x" and "y".
-/// id[i] = i, info.x[i] = i * 100, info.y[i] = i * 100 + 1, for i in [0, N).
-///
-/// Arrow schema: { id: int32, info: struct }
-/// Parquet leaf columns: [id (index 0), info.x (index 1), info.y (index 2)]
-static std::shared_ptr MakeNestedStructData(int32_t num_rows) {
-    arrow::Int32Builder id_builder, x_builder, y_builder;
+/// Helper: build an Int32Array with sequential values 0..N-1.
+static std::shared_ptr MakeIdColumn(int32_t num_rows) {
+    arrow::Int32Builder id_builder;
     EXPECT_TRUE(id_builder.Reserve(num_rows).ok());
+    for (int32_t i = 0; i < num_rows; ++i) {
+        id_builder.UnsafeAppend(i);
+    }
+    return id_builder.Finish().ValueOrDie();
+}
+
+/// Helper: build a struct array (without id column).
+/// x[i] = i * 100, y[i] = i * 100 + 1, for i in [0, N).
+static std::shared_ptr MakeNestedStructData(int32_t num_rows) {
+    arrow::Int32Builder x_builder, y_builder;
     EXPECT_TRUE(x_builder.Reserve(num_rows).ok());
     EXPECT_TRUE(y_builder.Reserve(num_rows).ok());
     for (int32_t i = 0; i < num_rows; ++i) {
-        id_builder.UnsafeAppend(i);
         x_builder.UnsafeAppend(i * 100);
         y_builder.UnsafeAppend(i * 100 + 1);
     }
-    auto id_array = id_builder.Finish().ValueOrDie();
     auto x_array = x_builder.Finish().ValueOrDie();
     auto y_array = y_builder.Finish().ValueOrDie();
 
     auto field_x = arrow::field("x", arrow::int32());
     auto field_y = arrow::field("y", arrow::int32());
-    auto inner_struct =
-        arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie();
-
-    auto field_id = arrow::field("id", arrow::int32());
-    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));
-    return arrow::StructArray::Make({id_array, inner_struct}, {field_id, field_info}).ValueOrDie();
+    return arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie();
 }
 
 /// Test: rowgroup-level filtering on a file with nested struct columns.
 ///
-/// This test exposes the bug where BuildPageFilteredSchema fails to correctly map
-/// Parquet leaf column indices to Arrow fields for nested types, and
-/// ReadFilteredRowGroup cannot correctly assemble nested column results.
-///
 /// Schema: { id: int32, info: struct }
 /// Parquet leaf columns: [id=0, info.x=1, info.y=2]
 /// 100 rows, 10 per page, 2 row groups.
-/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected.
+/// Predicate: id >= 70 → page 0-7 skipped, paged 8-9 read → 30 rows expected.
 /// The read schema requests both "id" and "info" columns.
-TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) {
+TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnPageFilter) {
     std::string file_name = dir_->Str() + "/nested_struct_filter.parquet";
-    auto data = MakeNestedStructData(100);
-    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
 
     auto field_x = arrow::field("x", arrow::int32());
     auto field_y = arrow::field("y", arrow::int32());
-    auto read_schema = arrow::schema({arrow::field("id", arrow::int32()),
-                                      arrow::field("info", arrow::struct_({field_x, field_y}))});
+    auto field_id = arrow::field("id", arrow::int32());
+    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));
+
+    auto id_array = MakeIdColumn(100);
+    auto info_array = MakeNestedStructData(100);
+    auto data =
+        arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie();
+    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
+
+    auto read_schema = arrow::schema({field_id, field_info});
 
     auto predicate = PredicateBuilder::GreaterOrEqual(
         /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));
@@ -974,32 +974,36 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnRowGroupFilter) {
     std::shared_ptr result;
     ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
 
-    // Should get rows 50-99 = 50 rows
+    // Should get rows 70-99 = 30 rows
     ASSERT_TRUE(result);
-    ASSERT_EQ(50, result->length());
+    ASSERT_EQ(30, result->length());
 
     // Build expected result: rows 50-99 from the original data
-    auto expected = data->Slice(50, 50);
+    auto expected = data->Slice(70, 30);
     ASSERT_TRUE(expected->Equals(result->chunk(0)));
 }
 
 /// Test: Page-level filtering reading only the predicate column (no nested column in read schema).
 ///
-/// This verifies that when reading only the "id" column (without the nested struct),
-/// page-level filtering works correctly since the read schema contains no nested types.
+/// This verifies that when reading only the "id" column (without the nested struct)
 ///
 /// Schema: { id: int32, info: struct }
 /// Read schema: { id: int32 }
 /// Predicate on "id": id >= 70. Page-level filtering active → rows 70-99 (30 rows).
 TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) {
     std::string file_name = dir_->Str() + "/nested_struct_only_nested.parquet";
-    auto data = MakeNestedStructData(100);
-    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
 
-    auto field_id = arrow::field("id", arrow::int32());
     auto field_x = arrow::field("x", arrow::int32());
     auto field_y = arrow::field("y", arrow::int32());
+    auto field_id = arrow::field("id", arrow::int32());
     auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));
+
+    auto id_array = MakeIdColumn(100);
+    auto info_array = MakeNestedStructData(100);
+    auto data =
+        arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie();
+    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
+
     // Read "id" column only
     auto read_schema = arrow::schema({field_id});
 
@@ -1019,19 +1023,9 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) {
     ASSERT_TRUE(data->field(0)->Slice(70, 30)->Equals(result_struct->field(0)));
 }
 
-/// Helper: build a StructArray with an int32 "id" column and a list "tags" column.
-/// id[i] = i, tags[i] = [i*10, i*10+1], for i in [0, N).
-///
-/// Arrow schema: { id: int32, tags: list }
-/// Parquet leaf columns: [id (index 0), tags.item (index 1)]
-static std::shared_ptr MakeListColumnData(int32_t num_rows) {
-    arrow::Int32Builder id_builder;
-    EXPECT_TRUE(id_builder.Reserve(num_rows).ok());
-    for (int32_t i = 0; i < num_rows; ++i) {
-        id_builder.UnsafeAppend(i);
-    }
-    auto id_array = id_builder.Finish().ValueOrDie();
-
+/// Helper: build a list array (without id column).
+/// tags[i] = [i*10, i*10+1], for i in [0, N).
+static std::shared_ptr MakeListColumnData(int32_t num_rows) {
     auto value_builder = std::make_shared();
     arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder);
     for (int32_t i = 0; i < num_rows; ++i) {
@@ -1039,26 +1033,12 @@ static std::shared_ptr MakeListColumnData(int32_t num_rows)
         EXPECT_TRUE(value_builder->Append(i * 10).ok());
         EXPECT_TRUE(value_builder->Append(i * 10 + 1).ok());
     }
-    auto list_array = list_builder.Finish().ValueOrDie();
-
-    auto field_id = arrow::field("id", arrow::int32());
-    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
-    return arrow::StructArray::Make({id_array, list_array}, {field_id, field_tags}).ValueOrDie();
+    return list_builder.Finish().ValueOrDie();
 }
 
-/// Helper: build a StructArray with an int32 "id" column and a map "props" column.
-/// id[i] = i, props[i] = {"k_i": i * 100}, for i in [0, N).
-///
-/// Arrow schema: { id: int32, props: map }
-/// Parquet leaf columns: [id (index 0), props.key (index 1), props.value (index 2)]
-static std::shared_ptr MakeMapColumnData(int32_t num_rows) {
-    arrow::Int32Builder id_builder;
-    EXPECT_TRUE(id_builder.Reserve(num_rows).ok());
-    for (int32_t i = 0; i < num_rows; ++i) {
-        id_builder.UnsafeAppend(i);
-    }
-    auto id_array = id_builder.Finish().ValueOrDie();
-
+/// Helper: build a map array (without id column).
+/// props[i] = {"k_i": i * 100}, for i in [0, N).
+static std::shared_ptr MakeMapColumnData(int32_t num_rows) {
     auto key_builder = std::make_shared();
     auto value_builder = std::make_shared();
     arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, value_builder);
@@ -1068,26 +1048,27 @@ static std::shared_ptr MakeMapColumnData(int32_t num_rows) {
         EXPECT_TRUE(key_builder->Append(key).ok());
         EXPECT_TRUE(value_builder->Append(i * 100).ok());
     }
-    auto map_array = map_builder.Finish().ValueOrDie();
-
-    auto field_id = arrow::field("id", arrow::int32());
-    auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()));
-    return arrow::StructArray::Make({id_array, map_array}, {field_id, field_props}).ValueOrDie();
+    return map_builder.Finish().ValueOrDie();
 }
 
 /// Test: rowgroup-level filtering on a file with a list column.
 ///
 /// Schema: { id: int32, tags: list }
 /// 100 rows, 10 per page, 2 row groups.
-/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected.
-TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) {
+/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected.
+TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnPageFilter) {
     std::string file_name = dir_->Str() + "/nested_list_filter.parquet";
-    auto data = MakeListColumnData(100);
+
+    auto field_id = arrow::field("id", arrow::int32());
+    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
+
+    auto id_array = MakeIdColumn(100);
+    auto tags_array = MakeListColumnData(100);
+    auto data =
+        arrow::StructArray::Make({id_array, tags_array}, {field_id, field_tags}).ValueOrDie();
     WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
 
-    auto read_schema =
-        arrow::schema({arrow::field("id", arrow::int32()),
-                       arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())))});
+    auto read_schema = arrow::schema({field_id, field_tags});
 
     auto predicate = PredicateBuilder::GreaterOrEqual(
         /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));
@@ -1096,10 +1077,10 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) {
     ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
 
     ASSERT_TRUE(result);
-    ASSERT_EQ(50, result->length());
+    ASSERT_EQ(30, result->length());
 
-    // Build expected result: rows 50-99 from the original data
-    auto expected = data->Slice(50, 50);
+    // Build expected result: rows 70-99 from the original data
+    auto expected = data->Slice(70, 30);
     ASSERT_TRUE(expected->Equals(result->chunk(0)));
 }
 
@@ -1107,116 +1088,32 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnRowGroupFilter) {
 ///
 /// Schema: { id: int32, props: map }
 /// 100 rows, 10 per page, 2 row groups.
-/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected.
-TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnRowGroupFilter) {
+/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected.
+TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnPageFilter) {
     std::string file_name = dir_->Str() + "/nested_map_filter.parquet";
-    auto data = MakeMapColumnData(100);
-    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
-
-    auto read_schema =
-        arrow::schema({arrow::field("id", arrow::int32()),
-                       arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))});
-
-    auto predicate = PredicateBuilder::GreaterOrEqual(
-        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));
-
-    std::shared_ptr result;
-    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
-
-    ASSERT_TRUE(result);
-    ASSERT_EQ(50, result->length());
-
-    // Build expected result: rows 50-99 from the original data
-    auto expected = data->Slice(50, 50);
-    ASSERT_TRUE(expected->Equals(result->chunk(0)));
-}
-
-/// Test: nested map projection falls back to row-group-level filtering when page index filter is
-/// unavailable for nested read schemas.
-///
-/// Schema: { id: int32, props: map }
-/// 100 rows, 10 per page, 2 row group.
-/// Bitmap: {70..99} hits the second row group (50..99).
-/// Because nested schema disables page-level filtering, the entire row group 1 (50..99) is read,
-/// so rows [50, 99] should all be returned.
-TEST_F(PageFilteredRowGroupReaderTest, NestedMapBitmapFallback) {
-    std::string file_name = dir_->Str() + "/nested_map_projection_fallback.parquet";
-    auto data = MakeMapColumnData(100);
-    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
 
+    auto field_id = arrow::field("id", arrow::int32());
     auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()));
-    auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), field_props});
 
-    RoaringBitmap32 bitmap;
-    bitmap.AddRange(70, 100);
-
-    std::shared_ptr result;
-    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result);
-
-    ASSERT_TRUE(result);
-    // Because page-level filtering is skipped for nested schemas, we read full row groups.
-    ASSERT_EQ(50, result->length());
-
-    auto expected = data->Slice(50, 50);
-    ASSERT_TRUE(expected->Equals(result->chunk(0)));
-}
-
-/// Test: nested list projection falls back to row-group-level filtering when page index filter is
-/// unavailable for nested read schemas.
-///
-/// Schema: { id: int32, tags: list }
-/// 100 rows, 10 per page, 2 row group.
-/// Bitmap: {70..99} hits the second row group (50..99).
-/// Because nested schema disables page-level filtering, the entire row group 1 (50..99) is read,
-/// so rows [50, 99] should all be returned.
-TEST_F(PageFilteredRowGroupReaderTest, NestedListBitmapFallback) {
-    std::string file_name = dir_->Str() + "/nested_list_projection_fallback.parquet";
-    auto data = MakeListColumnData(100);
+    auto id_array = MakeIdColumn(100);
+    auto props_array = MakeMapColumnData(100);
+    auto data =
+        arrow::StructArray::Make({id_array, props_array}, {field_id, field_props}).ValueOrDie();
     WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
 
-    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
-    auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), field_tags});
-
-    RoaringBitmap32 bitmap;
-    bitmap.AddRange(70, 100);
+    auto read_schema = arrow::schema({field_id, field_props});
 
-    std::shared_ptr result;
-    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result);
-
-    ASSERT_TRUE(result);
-    ASSERT_EQ(50, result->length());
-
-    auto expected = data->Slice(50, 50);
-    ASSERT_TRUE(expected->Equals(result->chunk(0)));
-}
-
-/// Test: nested struct projection falls back to row-group-level filtering when page index filter is
-/// unavailable for nested read schemas.
-///
-/// Schema: { id: int32, info: struct }
-/// Bitmap: {70..99} hits the second row group (50..99).
-/// Because nested schema disables page-level filtering, the entire second row group (50..99) is
-/// read.
-TEST_F(PageFilteredRowGroupReaderTest, NestedStructBitmapFallback) {
-    std::string file_name = dir_->Str() + "/nested_struct_projection_fallback.parquet";
-    auto data = MakeNestedStructData(100);
-    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
-
-    auto field_x = arrow::field("x", arrow::int32());
-    auto field_y = arrow::field("y", arrow::int32());
-    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));
-    auto read_schema = arrow::schema({arrow::field("id", arrow::int32()), field_info});
-
-    RoaringBitmap32 bitmap;
-    bitmap.AddRange(70, 100);
+    auto predicate = PredicateBuilder::GreaterOrEqual(
+        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));
 
     std::shared_ptr result;
-    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result);
+    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
 
     ASSERT_TRUE(result);
-    ASSERT_EQ(50, result->length());
+    ASSERT_EQ(30, result->length());
 
-    auto expected = data->Slice(50, 50);
+    // Build expected result: rows 70-99 from the original data
+    auto expected = data->Slice(70, 30);
     ASSERT_TRUE(expected->Equals(result->chunk(0)));
 }
 
@@ -1224,39 +1121,20 @@ TEST_F(PageFilteredRowGroupReaderTest, NestedStructBitmapFallback) {
 ///
 /// Schema: { id: int32, info: struct, tags: list }
 /// This tests the boundary handling when two nested fields are adjacent in the schema.
-/// Predicate: id >= 70 → row groups 0 skipped, row groups 1 read → 50 rows expected.
+/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected.
 TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) {
     std::string file_name = dir_->Str() + "/multi_nested.parquet";
 
-    // Build data with id, info (struct), tags (list)
-    arrow::Int32Builder id_builder, x_builder, y_builder;
-    ASSERT_TRUE(id_builder.Reserve(100).ok());
-    ASSERT_TRUE(x_builder.Reserve(100).ok());
-    ASSERT_TRUE(y_builder.Reserve(100).ok());
-    auto value_builder = std::make_shared();
-    arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder);
-
-    for (int32_t i = 0; i < 100; ++i) {
-        id_builder.UnsafeAppend(i);
-        x_builder.UnsafeAppend(i * 100);
-        y_builder.UnsafeAppend(i * 100 + 1);
-        ASSERT_TRUE(list_builder.Append().ok());
-        ASSERT_TRUE(value_builder->Append(i * 10).ok());
-    }
-    auto id_array = id_builder.Finish().ValueOrDie();
-    auto x_array = x_builder.Finish().ValueOrDie();
-    auto y_array = y_builder.Finish().ValueOrDie();
-    auto list_array = list_builder.Finish().ValueOrDie();
-
     auto field_x = arrow::field("x", arrow::int32());
     auto field_y = arrow::field("y", arrow::int32());
-    auto inner_struct =
-        arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie();
-
     auto field_id = arrow::field("id", arrow::int32());
     auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));
     auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
-    auto data = arrow::StructArray::Make({id_array, inner_struct, list_array},
+
+    auto id_array = MakeIdColumn(100);
+    auto info_array = MakeNestedStructData(100);
+    auto tags_array = MakeListColumnData(100);
+    auto data = arrow::StructArray::Make({id_array, info_array, tags_array},
                                          {field_id, field_info, field_tags})
                     .ValueOrDie();
 
@@ -1270,10 +1148,10 @@ TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) {
     ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
 
     ASSERT_TRUE(result);
-    ASSERT_EQ(50, result->length());
+    ASSERT_EQ(30, result->length());
 
-    // Build expected result: rows 50-99 from the original data
-    auto expected = data->Slice(50, 50);
+    // Build expected result: rows 70-99 from the original data
+    auto expected = data->Slice(70, 30);
     ASSERT_TRUE(expected->Equals(result->chunk(0)));
 }
 /// Test: bitmap hits all pages of a subset of row groups (no predicate).
@@ -1722,4 +1600,412 @@ TEST_F(PageFilteredRowGroupReaderTest, BitmapTrimMultiColumnTest) {
     }
 }
 
+/// Test: predicate pushdown with all nested column types (struct, list, map).
+///
+/// Schema: { id: int32, info: struct,
+///           tags: list, props: map }
+/// 100 rows, 10 rows per page, 50 rows per row group → 2 row groups.
+/// Predicate: id in [15, 29] or id in [80, 99] (Between is inclusive).
+/// Read schema: full schema (all columns).
+/// Page-level filtering (10 rows/page):
+///   Between(15, 29) → pages 1-2 (rows 10-29)
+///   Between(80, 99) → pages 8-9 (rows 80-99)
+///   Total: 40 rows.
+TEST_F(PageFilteredRowGroupReaderTest, MultipleNestedColumns) {
+    std::string file_name = dir_->Str() + "/multi_nested_columns.parquet";
+
+    auto field_x = arrow::field("x", arrow::int32());
+    auto field_y = arrow::field("y", arrow::int32());
+    auto field_id = arrow::field("id", arrow::int32());
+    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));
+    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
+    auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()));
+
+    // Build data with all nested column types using shared helpers
+    auto id_array = MakeIdColumn(100);
+    auto info_array = MakeNestedStructData(100);
+    auto tags_array = MakeListColumnData(100);
+    auto props_array = MakeMapColumnData(100);
+    auto data = arrow::StructArray::Make({id_array, info_array, tags_array, props_array},
+                                         {field_id, field_info, field_tags, field_props})
+                    .ValueOrDie();
+
+    // Write: 10 rows per page, 50 rows per row group → 2 row groups
+    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
+
+    // Read full schema
+    auto read_schema = arrow::schema({field_id, field_info, field_tags, field_props});
+
+    // predicate: id in [15, 29] or id in [80, 99]
+    ASSERT_OK_AND_ASSIGN(
+        auto predicate, PredicateBuilder::Or(
+                            {PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id",
+                                                       FieldType::INT, Literal(15), Literal(29)),
+                             PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id",
+                                                       FieldType::INT, Literal(80), Literal(99))}));
+
+    std::shared_ptr result;
+    ReadWithPredicateImpl(file_name, read_schema, predicate, &result,
+                          /*batch_size=*/1024);
+
+    // Page-level filtering (10 rows/page):
+    //   Between(15, 29) → pages 1-2 (rows 10-29)
+    //   Between(80, 99) → pages 8-9 (rows 80-99)
+    //   Total: 40 rows
+    ASSERT_TRUE(result);
+    ASSERT_EQ(40, result->length());
+
+    auto expected =
+        arrow::ChunkedArray::Make({data->Slice(10, 20), data->Slice(80, 20)}).ValueOrDie();
+    ASSERT_TRUE(result->Equals(expected));
+}
+
+/// Test: sub-column projection of a struct type with page-level filtering.
+///
+/// Schema: { id: int32, info: struct }
+/// Read schema: { info: struct } — project only x, not y.
+/// Predicate: id >= 70 → 30 rows expected.
+/// Verifies that reading a sub-column of a nested struct works correctly
+/// with page-level filtering and the ColumnReader tree (GetColumn + filter_leaves).
+TEST_F(PageFilteredRowGroupReaderTest, NestedStructSubColumnProjection) {
+    std::string file_name = dir_->Str() + "/nested_struct_subcol.parquet";
+
+    auto field_x = arrow::field("x", arrow::int32());
+    auto field_y = arrow::field("y", arrow::int32());
+    auto field_id = arrow::field("id", arrow::int32());
+    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));
+
+    auto id_array = MakeIdColumn(100);
+    auto info_array = MakeNestedStructData(100);
+    auto data =
+        arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie();
+    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);
+
+    // Read only info.x (sub-column projection: only x, not y)
+    auto read_schema = arrow::schema({arrow::field("info", arrow::struct_({field_x}))});
+
+    auto predicate = PredicateBuilder::GreaterOrEqual(
+        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));
+
+    std::shared_ptr result;
+    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
+
+    ASSERT_TRUE(result);
+    ASSERT_EQ(30, result->length());
+
+    // Result is struct>
+    auto result_struct = std::dynamic_pointer_cast(result->chunk(0));
+    ASSERT_TRUE(result_struct);
+    ASSERT_EQ(1, result_struct->num_fields());
+
+    auto info_result = std::dynamic_pointer_cast(result_struct->field(0));
+    ASSERT_TRUE(info_result);
+    ASSERT_EQ(1, info_result->num_fields());
+
+    auto x_arr = std::dynamic_pointer_cast(info_result->field(0));
+    ASSERT_TRUE(x_arr);
+    for (int32_t i = 0; i < 30; ++i) {
+        ASSERT_EQ((70 + i) * 100, x_arr->Value(i)) << "Mismatch at index " << i;
+    }
+}
+
+/// Helper: build a struct with a flat key plus several nested columns of different
+/// physical widths / repetition, so that with a byte-based data page size their
+/// leaves flush pages at different row counts (misaligned pages):
+///   key:   int64            (fixed 8B  -> ~5 rows/page)
+///   s:     struct   (x ~10 rows/page, y ~5 rows/page)
+///   tags:  list       (2 values/row -> ~5 rows/page)
+///   props: map   (variable-width utf8 key -> irregular rows/page)
+/// key/s.x/s.y encode the row index (= i); tags/props reuse the shared list/map
+/// helpers. Correctness is verified per row by deep-comparing against this array.
+static std::shared_ptr MakeMisalignedNestedData(int32_t num_rows) {
+    arrow::Int64Builder key_builder;
+    arrow::Int32Builder x_builder;
+    arrow::Int64Builder y_builder;
+    EXPECT_TRUE(key_builder.Reserve(num_rows).ok());
+    EXPECT_TRUE(x_builder.Reserve(num_rows).ok());
+    EXPECT_TRUE(y_builder.Reserve(num_rows).ok());
+    for (int32_t i = 0; i < num_rows; ++i) {
+        key_builder.UnsafeAppend(i);
+        x_builder.UnsafeAppend(i);
+        y_builder.UnsafeAppend(i);
+    }
+    auto key_array = key_builder.Finish().ValueOrDie();
+    auto x_array = x_builder.Finish().ValueOrDie();
+    auto y_array = y_builder.Finish().ValueOrDie();
+
+    auto field_x = arrow::field("x", arrow::int32());
+    auto field_y = arrow::field("y", arrow::int64());
+    auto s_array = arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie();
+
+    // Repeated nested leaves (list and map) paginate on value
+    // bytes, so they misalign with the struct's fixed-width leaves too.
+    auto tags_array = MakeListColumnData(num_rows);
+    auto props_array = MakeMapColumnData(num_rows);
+
+    auto field_key = arrow::field("key", arrow::int64());
+    auto field_s = arrow::field("s", arrow::struct_({field_x, field_y}));
+    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
+    auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()));
+    return arrow::StructArray::Make({key_array, s_array, tags_array, props_array},
+                                    {field_key, field_s, field_tags, field_props})
+        .ValueOrDie();
+}
+
+/// Test: page-level filtering across multiple nested columns whose leaf pages are
+/// MISALIGNED, within a SINGLE row group. The file mixes a flat key, a
+/// struct, a list and a map; with write_batch_size=1
+/// and a byte-based data page size every leaf flushes pages at a different (and, for
+/// the utf8 map key, irregular) row count.
+/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling)
+/// Expected: 25 rows
+TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPagesSingleRowGroup) {
+    std::string file_name = dir_->Str() + "/nested_misaligned_single_rg.parquet";
+    constexpr int32_t kNumRows = 100;
+    auto data = MakeMisalignedNestedData(kNumRows);
+
+    // write_batch_size=1 + a byte-based data page size makes the int32 and int64 leaves
+    // flush pages at different row counts, i.e. deliberately misaligned pages.
+    // max_row_group_length=kNumRows keeps all rows in a single row group.
+    WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/kNumRows,
+                  /*enable_dictionary=*/false, /*data_page_size=*/40);
+
+    auto read_schema =
+        arrow::schema({arrow::field("key", arrow::int64()),
+                       arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()),
+                                                         arrow::field("y", arrow::int64())})),
+                       arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))),
+                       arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))});
+
+    // bitmap: [0,15), [77, 87)
+    RoaringBitmap32 bitmap;
+    bitmap.AddRange(0, 15);
+    bitmap.AddRange(77, 87);
+
+    std::shared_ptr result;
+    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result);
+    ASSERT_TRUE(result);
+
+    int64_t total = 0;
+    auto top = std::dynamic_pointer_cast(result->chunk(0));
+    ASSERT_TRUE(top);
+    auto key_arr = std::dynamic_pointer_cast(top->field(0));
+    for (int64_t i = 0; i < key_arr->length(); ++i) {
+        int64_t k = key_arr->Value(i);
+        // Full-row deep compare across ALL columns (struct + list + map): the returned
+        // row must equal the original row identified by key k. This is what catches a
+        // desync in the repeated list/map leaves, whose reassembly also relies on the
+        // per-leaf skip/read staying row-consistent.
+        ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1)))
+            << "row content mismatch at key " << k;
+        ++total;
+    }
+
+    for (int64_t i = 0; i < 15; ++i) {
+        ASSERT_EQ(i, key_arr->Value(i));
+    }
+    for (int64_t i = 0; i < 10; ++i) {
+        ASSERT_EQ(77 + i, key_arr->Value(15 + i));
+    }
+
+    ASSERT_EQ(total, 25);
+}
+
+/// Test: same misaligned nested layout as the single-row-group case above, but split
+/// across MULTIPLE row groups (max_row_group_length=40 -> row groups of 40/40/20).
+/// The selection bitmap spans row-group boundaries so the reader must keep the
+/// per-leaf skip/read row-consistent both across misaligned pages and across row
+/// groups.
+/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling)
+/// Expected: 25 rows -> keys 0..14 and 77..86
+TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPagesMultiRowGroup) {
+    std::string file_name = dir_->Str() + "/nested_misaligned_multi_rg.parquet";
+    constexpr int32_t kNumRows = 100;
+    auto data = MakeMisalignedNestedData(kNumRows);
+
+    // write_batch_size=1 + byte-based data page size -> misaligned leaf pages.
+    // max_row_group_length=40 -> 3 row groups (40, 40, 20).
+    WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/40,
+                  /*enable_dictionary=*/false, /*data_page_size=*/40);
+
+    auto read_schema =
+        arrow::schema({arrow::field("key", arrow::int64()),
+                       arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()),
+                                                         arrow::field("y", arrow::int64())})),
+                       arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))),
+                       arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))});
+
+    // bitmap: [0,15), [77, 87) -> spans all three row groups.
+    RoaringBitmap32 bitmap;
+    bitmap.AddRange(0, 15);
+    bitmap.AddRange(77, 87);
+
+    std::shared_ptr result;
+    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result);
+    ASSERT_TRUE(result);
+
+    std::vector keys;
+
+    auto concated = arrow::Concatenate(result->chunks()).ValueOrDie();
+    auto top = std::dynamic_pointer_cast(concated);
+    ASSERT_TRUE(top);
+    auto key_arr = std::dynamic_pointer_cast(top->field(0));
+    ASSERT_TRUE(key_arr);
+    for (int64_t i = 0; i < key_arr->length(); ++i) {
+        int64_t k = key_arr->Value(i);
+        ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1)))
+            << "row content mismatch at key " << k;
+        keys.push_back(k);
+    }
+
+    std::vector expected;
+    for (int64_t i = 0; i < 15; ++i) {
+        expected.push_back(i);
+    }
+    for (int64_t i = 77; i < 87; ++i) {
+        expected.push_back(i);
+    }
+    ASSERT_EQ(keys, expected);
+}
+
+/// Helper: like MakeMisalignedNestedData but sprinkles nulls into the nested columns
+/// so that definition levels vary per row (which also perturbs page boundaries):
+///   key:   int64, always non-null (= i, used to identify the row)
+///   s:     struct; whole struct null when i%11==0, otherwise
+///          x null when i%5==0 and y null when i%7==0
+///   tags:  list; null when i%6==0, otherwise [i*10, i*10+1]
+///   props: map; null when i%8==0, otherwise {"k_i": i*100}
+/// Correctness is verified per row by deep-comparing against this array.
+static std::shared_ptr MakeMisalignedNestedDataWithNulls(int32_t num_rows) {
+    arrow::Int64Builder key_builder;
+    EXPECT_TRUE(key_builder.Reserve(num_rows).ok());
+    for (int32_t i = 0; i < num_rows; ++i) {
+        key_builder.UnsafeAppend(i);
+    }
+    auto key_array = key_builder.Finish().ValueOrDie();
+
+    // s: struct with nulls at both leaf and struct level.
+    auto field_x = arrow::field("x", arrow::int32());
+    auto field_y = arrow::field("y", arrow::int64());
+    auto x_builder = std::make_shared();
+    auto y_builder = std::make_shared();
+    arrow::StructBuilder s_builder(arrow::struct_({field_x, field_y}), arrow::default_memory_pool(),
+                                   {x_builder, y_builder});
+    for (int32_t i = 0; i < num_rows; ++i) {
+        if (i % 11 == 0) {
+            // AppendNull() also appends nulls to the child builders, keeping lengths in sync.
+            EXPECT_TRUE(s_builder.AppendNull().ok());
+            continue;
+        }
+        EXPECT_TRUE(s_builder.Append().ok());
+        if (i % 5 == 0) {
+            EXPECT_TRUE(x_builder->AppendNull().ok());
+        } else {
+            EXPECT_TRUE(x_builder->Append(i).ok());
+        }
+        if (i % 7 == 0) {
+            EXPECT_TRUE(y_builder->AppendNull().ok());
+        } else {
+            EXPECT_TRUE(y_builder->Append(i).ok());
+        }
+    }
+    auto s_array = s_builder.Finish().ValueOrDie();
+
+    // tags: list with null lists.
+    auto item_builder = std::make_shared();
+    arrow::ListBuilder tags_builder(arrow::default_memory_pool(), item_builder);
+    for (int32_t i = 0; i < num_rows; ++i) {
+        if (i % 6 == 0) {
+            EXPECT_TRUE(tags_builder.AppendNull().ok());
+        } else {
+            EXPECT_TRUE(tags_builder.Append().ok());
+            EXPECT_TRUE(item_builder->Append(i * 10).ok());
+            EXPECT_TRUE(item_builder->Append(i * 10 + 1).ok());
+        }
+    }
+    auto tags_array = tags_builder.Finish().ValueOrDie();
+
+    // props: map with null maps.
+    auto map_key_builder = std::make_shared();
+    auto map_val_builder = std::make_shared();
+    arrow::MapBuilder props_builder(arrow::default_memory_pool(), map_key_builder, map_val_builder);
+    for (int32_t i = 0; i < num_rows; ++i) {
+        if (i % 8 == 0) {
+            EXPECT_TRUE(props_builder.AppendNull().ok());
+        } else {
+            EXPECT_TRUE(props_builder.Append().ok());
+            EXPECT_TRUE(map_key_builder->Append("k_" + std::to_string(i)).ok());
+            EXPECT_TRUE(map_val_builder->Append(i * 100).ok());
+        }
+    }
+    auto props_array = props_builder.Finish().ValueOrDie();
+
+    auto field_key = arrow::field("key", arrow::int64());
+    auto field_s = arrow::field("s", arrow::struct_({field_x, field_y}));
+    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
+    auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()));
+    return arrow::StructArray::Make({key_array, s_array, tags_array, props_array},
+                                    {field_key, field_s, field_tags, field_props})
+        .ValueOrDie();
+}
+
+/// Test: nested columns containing NULLs, with MISALIGNED leaf pages, split across
+/// MULTIPLE row groups. This combines the three stress dimensions: null-driven
+/// definition levels, byte-based misaligned pages, and row-group boundaries that the
+/// selection bitmap crosses. Every returned row is deep-compared (nulls included)
+/// against the original data identified by its non-null key.
+/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling)
+/// Expected: 25 rows -> keys 0..14 and 77..86
+TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsWithNullsMisalignedPagesMultiRowGroup) {
+    std::string file_name = dir_->Str() + "/nested_nulls_misaligned_multi_rg.parquet";
+    constexpr int32_t kNumRows = 100;
+    auto data = MakeMisalignedNestedDataWithNulls(kNumRows);
+
+    // write_batch_size=1 + byte-based data page size -> misaligned leaf pages.
+    // max_row_group_length=40 -> 3 row groups (40, 40, 20).
+    WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/40,
+                  /*enable_dictionary=*/false, /*data_page_size=*/40);
+
+    auto read_schema =
+        arrow::schema({arrow::field("key", arrow::int64()),
+                       arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()),
+                                                         arrow::field("y", arrow::int64())})),
+                       arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))),
+                       arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))});
+
+    // bitmap: [0,15), [77, 87) -> spans all three row groups.
+    RoaringBitmap32 bitmap;
+    bitmap.AddRange(0, 15);
+    bitmap.AddRange(77, 87);
+
+    std::shared_ptr result;
+    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result);
+    ASSERT_TRUE(result);
+
+    std::vector keys;
+
+    auto concated = arrow::Concatenate(result->chunks()).ValueOrDie();
+    auto top = std::dynamic_pointer_cast(concated);
+    ASSERT_TRUE(top);
+    auto key_arr = std::dynamic_pointer_cast(top->field(0));
+    ASSERT_TRUE(key_arr);
+    for (int64_t i = 0; i < key_arr->length(); ++i) {
+        ASSERT_FALSE(key_arr->IsNull(i)) << "key column must stay non-null";
+        int64_t k = key_arr->Value(i);
+        // Deep compare including nulls across struct/list/map leaves.
+        ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1)))
+            << "row content mismatch at key " << k;
+        keys.push_back(k);
+    }
+
+    std::vector expected;
+    for (int64_t i = 0; i < 15; ++i) {
+        expected.push_back(i);
+    }
+    for (int64_t i = 77; i < 87; ++i) {
+        expected.push_back(i);
+    }
+    ASSERT_EQ(keys, expected);
+}
+
 }  // namespace paimon::parquet::test
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
index 547a3c1e..28dadeea 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
@@ -141,13 +141,6 @@ Status ParquetFileBatchReader::SetReadSchema(
                                           arrow::ImportSchema(schema));
 
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema, reader_->GetSchema());
-        bool has_nested_field = false;
-        for (const auto& field : read_schema->fields()) {
-            if (ArrowSchemaValidator::IsNestedType(field->type())) {
-                has_nested_field = true;
-                break;
-            }
-        }
 
         // Recursively match read_schema against file_schema by field names.
         // STRUCT supports sub-field projection; LIST/MAP require exact type match.
@@ -180,9 +173,7 @@ Status ParquetFileBatchReader::SetReadSchema(
             PAIMON_ASSIGN_OR_RAISE(
                 target_row_groups,
                 FilterRowGroupsByBitmap(selection_bitmap.value(), target_row_groups));
-            // workaround: page index filter does not support nested fields for now, skip page index
-            // bitmap pushdown if there is any nested field in the schema
-            if (!has_nested_field && enable_page_index_filter) {
+            if (enable_page_index_filter) {
                 // To decide which strategy to use, "trim" or "coalesce". "Coalesce" By default.
                 PAIMON_ASSIGN_OR_RAISE(
                     std::string strategy,
@@ -210,9 +201,7 @@ Status ParquetFileBatchReader::SetReadSchema(
         // pages for row groups that the bitmap already excluded.
         // If no predicate is provided, skip page-level filtering
         if (predicate && !target_row_groups.empty()) {
-            // workaround: page index filter does not support nested fields for now, skip page index
-            // filter if there is any nested field in the schema
-            if (enable_page_index_filter && !has_nested_field) {
+            if (enable_page_index_filter) {
                 // Build column name to index map for page-level filtering.
                 // For leaf columns, indices[0] is the correct leaf column index in Parquet.
                 // For nested types (struct/list/map), FlattenSchema produces multiple leaf indices,

From 04bc024e9e43ec67e824512c06710bf8478b69cd Mon Sep 17 00:00:00 2001
From: Yonghao Fang 
Date: Tue, 28 Jul 2026 17:52:24 +0800
Subject: [PATCH 123/138] test: add test for test coverage

---
 src/paimon/CMakeLists.txt                     |   2 +
 .../block_compression_factory_test.cpp        |  31 ++
 .../data/variant/generic_variant_test.cpp     |  73 +++++
 .../infer_variant_shredding_schema_test.cpp   | 158 ++++++++++
 .../variant/variant_access_utils_test.cpp     | 164 +++++++++++
 .../common/data/variant/variant_get_test.cpp  | 130 ++++++++
 .../data/variant/variant_json_utils_test.cpp  |  46 +++
 ...riant_shredding_read_plan_factory_test.cpp | 277 ++++++++++++++++++
 .../data/variant/variant_shredding_test.cpp   | 225 ++++++++++++++
 src/paimon/common/logging/logging_test.cpp    | 142 +++++++++
 .../common/utils/arrow/mem_utils_test.cpp     |  71 +++++
 src/paimon/common/utils/file_type_test.cpp    |  14 +
 src/paimon/common/utils/status_test.cpp       |  37 +++
 13 files changed, 1370 insertions(+)
 create mode 100644 src/paimon/common/data/variant/variant_access_utils_test.cpp
 create mode 100644 src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index d56700b8..4afe3a27 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -475,6 +475,8 @@ if(PAIMON_BUILD_TESTS)
                     common/data/blob_utils_test.cpp
                     common/data/variant/generic_variant_test.cpp
                     common/data/variant/infer_variant_shredding_schema_test.cpp
+                    common/data/variant/variant_access_utils_test.cpp
+                    common/data/variant/variant_shredding_read_plan_factory_test.cpp
                     common/data/variant/variant_shredding_write_plan_factory_test.cpp
                     common/data/variant/variant_get_test.cpp
                     common/data/variant/variant_json_utils_test.cpp
diff --git a/src/paimon/common/compression/block_compression_factory_test.cpp b/src/paimon/common/compression/block_compression_factory_test.cpp
index 107d20a3..90ad4cce 100644
--- a/src/paimon/common/compression/block_compression_factory_test.cpp
+++ b/src/paimon/common/compression/block_compression_factory_test.cpp
@@ -117,4 +117,35 @@ TEST_P(CompressionFactoryTest, TestCompressInsufficientOutputBuffer) {
 INSTANTIATE_TEST_SUITE_P(BlockCompressionTypeGroup, CompressionFactoryTest,
                          ::testing::Values(BlockCompressionType::LZ4, BlockCompressionType::ZSTD));
 
+TEST(CompressionFactoryCreateTest, TestCreateFromCompressOptions) {
+    ASSERT_OK_AND_ASSIGN(auto none_factory,
+                         BlockCompressionFactory::Create(CompressOptions{"none", 0}));
+    ASSERT_EQ(BlockCompressionType::NONE, none_factory->GetCompressionType());
+
+    // Codec name matching is case-insensitive.
+    ASSERT_OK_AND_ASSIGN(auto zstd_factory,
+                         BlockCompressionFactory::Create(CompressOptions{"ZSTD", 1}));
+    ASSERT_EQ(BlockCompressionType::ZSTD, zstd_factory->GetCompressionType());
+
+    ASSERT_OK_AND_ASSIGN(auto lz4_factory,
+                         BlockCompressionFactory::Create(CompressOptions{"Lz4", 0}));
+    ASSERT_EQ(BlockCompressionType::LZ4, lz4_factory->GetCompressionType());
+
+    // Unsupported codec name returns an Invalid status.
+    auto unsupported = BlockCompressionFactory::Create(CompressOptions{"lzo", 0});
+    ASSERT_NOK(unsupported);
+    ASSERT_TRUE(unsupported.status().IsInvalid());
+}
+
+TEST(CompressionFactoryCreateTest, TestCreateFromCompressionType) {
+    ASSERT_OK_AND_ASSIGN(auto none_factory,
+                         BlockCompressionFactory::Create(BlockCompressionType::NONE));
+    ASSERT_EQ(BlockCompressionType::NONE, none_factory->GetCompressionType());
+
+    // LZO is declared but not yet implemented, so it hits the default branch.
+    auto lzo = BlockCompressionFactory::Create(BlockCompressionType::LZO);
+    ASSERT_NOK(lzo);
+    ASSERT_TRUE(lzo.status().IsInvalid());
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/data/variant/generic_variant_test.cpp b/src/paimon/common/data/variant/generic_variant_test.cpp
index d628b3e7..11386a28 100644
--- a/src/paimon/common/data/variant/generic_variant_test.cpp
+++ b/src/paimon/common/data/variant/generic_variant_test.cpp
@@ -19,8 +19,10 @@
 
 #include "paimon/common/data/variant/generic_variant.h"
 
+#include 
 #include 
 #include 
+#include 
 
 #include "gtest/gtest.h"
 #include "paimon/common/data/variant/variant_builder.h"
@@ -348,4 +350,75 @@ TEST_F(GenericVariantTest, NonFiniteDoubleToJson) {
     ASSERT_EQ(json, "\"Infinity\"");
 }
 
+TEST_F(GenericVariantTest, GetTypeInfoReturnsHeaderBits) {
+    // GetTypeInfo exposes the primitive header's type-info bits; 42 is encoded as an int1.
+    auto v = FromJson("42");
+    ASSERT_OK_AND_ASSIGN(int32_t type_info, v->GetTypeInfo());
+    EXPECT_EQ(type_info, VariantDefs::kInt1);
+}
+
+TEST_F(GenericVariantTest, TypedGettersRejectMismatchedTypes) {
+    // Each accessor validates the value header and fails when the stored type differs.
+    auto number = FromJson("42");    // primitive int1
+    auto text = FromJson("\"hi\"");  // short string
+    auto real = FromJson("1.5e0");   // double
+
+    // A primitive long is neither boolean/double/decimal/float/binary/string/uuid, nor a
+    // container.
+    ASSERT_NOK(number->GetBoolean());
+    ASSERT_NOK(number->GetDouble());
+    ASSERT_NOK(number->GetDecimal());
+    ASSERT_NOK(number->GetFloat());
+    ASSERT_NOK(number->GetBinary());
+    ASSERT_NOK(number->GetString());
+    ASSERT_NOK(number->GetUuid());
+    ASSERT_NOK(number->ObjectSize());
+    ASSERT_NOK(number->ArraySize());
+    // A short string is not a primitive, so long/decimal decoding rejects it early.
+    ASSERT_NOK(text->GetLong());
+    ASSERT_NOK(text->GetDecimal());
+    // A double is a primitive but not an integer-like type.
+    ASSERT_NOK(real->GetLong());
+}
+
+TEST_F(GenericVariantTest, ValueSizeCoversAllPrimitiveWidths) {
+    // Builds an array whose elements span the primitive width branches of `ValueSize` (int4,
+    // decimal8, binary, uuid). Copying elements into the array and re-reading each element's
+    // value both exercise `ValueSize`.
+    auto build = [this](const std::function& append) {
+        VariantBuilder builder(/*allow_duplicate_keys=*/false);
+        EXPECT_OK(append(builder));
+        auto result = builder.Build(pool_);
+        EXPECT_TRUE(result.ok()) << result.status().ToString();
+        return result.value();
+    };
+    std::shared_ptr int4 =
+        build([](VariantBuilder& b) { return b.AppendLong(100000); });  // needs 4 bytes
+    std::shared_ptr decimal8 =
+        build([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{1234567890, 2}); });
+    std::shared_ptr binary =
+        build([](VariantBuilder& b) { return b.AppendBinary(std::string_view("abc", 3)); });
+    std::string uuid_bytes(16, '\x07');
+    std::shared_ptr uuid =
+        build([&](VariantBuilder& b) { return b.AppendUuid(uuid_bytes); });
+
+    VariantBuilder array_builder(/*allow_duplicate_keys=*/false);
+    int32_t start = array_builder.GetWritePos();
+    std::vector offsets;
+    for (const std::shared_ptr* element : {&int4, &decimal8, &binary, &uuid}) {
+        offsets.push_back(array_builder.GetWritePos() - start);
+        ASSERT_OK(array_builder.AppendVariant(**element));
+    }
+    ASSERT_OK(array_builder.FinishWritingArray(start, offsets));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr array_variant, array_builder.Build(pool_));
+
+    ASSERT_OK_AND_ASSIGN(int32_t size, array_variant->ArraySize());
+    ASSERT_EQ(size, 4);
+    for (int32_t i = 0; i < size; ++i) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr element,
+                             array_variant->GetElementAtIndex(i));
+        ASSERT_OK(element->Value());
+    }
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp b/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp
index e473782f..9a55b866 100644
--- a/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp
+++ b/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/common/data/variant/infer_variant_shredding_schema.h"
 
+#include 
 #include 
 #include 
 
@@ -55,6 +56,18 @@ class InferVariantShreddingSchemaTest : public ::testing::Test {
         return samples;
     }
 
+    // Builds a single variant using the direct append API, which can encode types (float, binary,
+    // uuid, decimals with a specific scale) that JSON parsing never produces.
+    std::shared_ptr BuildVariant(
+        const std::function& append) {
+        VariantBuilder builder(/*allow_duplicate_keys=*/false);
+        Status st = append(builder);
+        EXPECT_TRUE(st.ok()) << st.ToString();
+        auto result = builder.Build(pool_);
+        EXPECT_TRUE(result.ok()) << result.status().ToString();
+        return result.value();
+    }
+
  protected:
     std::shared_ptr pool_ = GetDefaultPool();
     InferVariantShreddingSchema infer_{/*max_schema_width=*/300, /*max_schema_depth=*/50,
@@ -212,4 +225,149 @@ TEST_F(InferVariantShreddingSchemaTest, TemporalValuesStayUnshredded) {
     ASSERT_EQ(ts_inferred, nullptr);
 }
 
+TEST_F(InferVariantShreddingSchemaTest, MergeObjectsWithDisjointFields) {
+    // Objects whose keys interleave exercise both single-side merge branches (field only in the
+    // first object, and field only in the second).
+    auto samples = Samples({R"({"a": 1, "c": 3})", R"({"b": 2, "c": 4})"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples));
+    ASSERT_NE(inferred, nullptr);
+    auto expected =
+        arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64()),
+                        arrow::field("c", arrow::int64())});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, VariantNullSamplesAndFields) {
+    // A top-level variant null (JSON `null`, not a missing sample) merges away, leaving the other
+    // sample's inferred type.
+    auto scalar_and_null = Samples({"1", "null"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr merged,
+                         InferColumn(infer_, scalar_and_null));
+    ASSERT_NE(merged, nullptr);
+    ASSERT_TRUE(merged->Equals(*arrow::int64())) << merged->ToString();
+
+    // An object field that is always variant-null becomes an untyped variant leaf.
+    auto object_with_null = Samples({R"({"a": null, "b": 1})"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred,
+                         InferColumn(infer_, object_with_null));
+    ASSERT_NE(inferred, nullptr);
+    auto expected =
+        arrow::struct_({arrow::field("a", arrow::null()), arrow::field("b", arrow::int64())});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, ArraysMerge) {
+    // Two arrays merge element-wise into a single typed element schema.
+    auto samples = Samples({"[1, 2]", "[3, 4]"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples));
+    ASSERT_NE(inferred, nullptr);
+    ASSERT_TRUE(inferred->Equals(*arrow::list(arrow::int64()))) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, ArrayBeyondDepthLimitStaysVariant) {
+    InferVariantShreddingSchema shallow_infer{/*max_schema_width=*/300, /*max_schema_depth=*/1,
+                                              /*min_field_cardinality_ratio=*/0.1};
+    auto samples = Samples({R"({"arr": [1, 2]})"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred,
+                         InferColumn(shallow_infer, samples));
+    ASSERT_NE(inferred, nullptr);
+    // Depth 1: the nested array is beyond the limit and stays an untyped variant leaf.
+    auto expected = arrow::struct_({arrow::field("arr", arrow::null())});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, ScalarLeafTypes) {
+    // Float and binary leaves shred to their arrow types.
+    std::shared_ptr float_variant =
+        BuildVariant([](VariantBuilder& b) { return b.AppendFloat(1.5f); });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr float_inferred,
+                         InferColumn(infer_, {float_variant}));
+    ASSERT_NE(float_inferred, nullptr);
+    ASSERT_TRUE(float_inferred->Equals(*arrow::float32())) << float_inferred->ToString();
+
+    std::shared_ptr binary_variant = BuildVariant(
+        [](VariantBuilder& b) { return b.AppendBinary(std::string_view("\x01\x02", 2)); });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr binary_inferred,
+                         InferColumn(infer_, {binary_variant}));
+    ASSERT_NE(binary_inferred, nullptr);
+    ASSERT_TRUE(binary_inferred->Equals(*arrow::binary())) << binary_inferred->ToString();
+
+    // A UUID has no shredding type, so the column stays unshredded.
+    std::string uuid_bytes(16, '\0');
+    std::shared_ptr uuid_variant =
+        BuildVariant([&](VariantBuilder& b) { return b.AppendUuid(uuid_bytes); });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr uuid_inferred,
+                         InferColumn(infer_, {uuid_variant}));
+    ASSERT_EQ(uuid_inferred, nullptr);
+}
+
+TEST_F(InferVariantShreddingSchemaTest, LargeIntegerAndDecimalMerging) {
+    // A 19-digit long exceeds decimal(18) precision, so it stays a genuine int64 leaf.
+    auto big_long = Samples({"1000000000000000000"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr long_inferred,
+                         InferColumn(infer_, big_long));
+    ASSERT_NE(long_inferred, nullptr);
+    ASSERT_TRUE(long_inferred->Equals(*arrow::int64())) << long_inferred->ToString();
+
+    // A long (int64) merged with a fractional decimal widens via MergeDecimalWithLong.
+    auto long_then_decimal = Samples({"1000000000000000000", "1.5"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr ld,
+                         InferColumn(infer_, long_then_decimal));
+    ASSERT_NE(ld, nullptr);
+    ASSERT_TRUE(ld->Equals(*arrow::decimal128(38, 1))) << ld->ToString();
+
+    // The reversed order (decimal first, then long) hits the mirrored branch.
+    auto decimal_then_long = Samples({"1.5", "1000000000000000000"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr dl,
+                         InferColumn(infer_, decimal_then_long));
+    ASSERT_NE(dl, nullptr);
+    ASSERT_TRUE(dl->Equals(*arrow::decimal128(38, 1))) << dl->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, LongMergedWithIntegralDecimalStaysLong) {
+    // A scale-0 decimal that fits in 18 digits merges with a long back to int64.
+    std::shared_ptr integral_decimal =
+        BuildVariant([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{123, 0}); });
+    std::shared_ptr big_long = Samples({"1000000000000000000"})[0];
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred,
+                         InferColumn(infer_, {integral_decimal, big_long}));
+    ASSERT_NE(inferred, nullptr);
+    ASSERT_TRUE(inferred->Equals(*arrow::int64())) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, SmallFractionalDecimalPrecisionAdjusted) {
+    // 0.0015 has more fractional digits than significant digits; precision widens up to the scale.
+    auto samples = Samples({"0.0015"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples));
+    ASSERT_NE(inferred, nullptr);
+    ASSERT_TRUE(inferred->Equals(*arrow::decimal128(18, 4))) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, DecimalMergeOverflowFallsToVariant) {
+    // A 38-digit integral decimal merged with a high-scale decimal would need precision > 38,
+    // which decimal cannot represent, so the column stays unshredded.
+    __int128 wide_unscaled = 0;
+    for (int i = 0; i < 38; ++i) {
+        wide_unscaled = wide_unscaled * 10 + 1;  // 38 ones, no trailing zeros
+    }
+    std::shared_ptr wide_decimal = BuildVariant(
+        [&](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{wide_unscaled, 0}); });
+    std::shared_ptr high_scale_decimal =
+        BuildVariant([](VariantBuilder& b) { return b.AppendDecimal(VariantDecimal{15, 20}); });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred,
+                         InferColumn(infer_, {wide_decimal, high_scale_decimal}));
+    ASSERT_EQ(inferred, nullptr);
+}
+
+TEST_F(InferVariantShreddingSchemaTest, ObjectWithAllRareFieldsStaysUnshredded) {
+    InferVariantShreddingSchema strict_infer{/*max_schema_width=*/300, /*max_schema_depth=*/50,
+                                             /*min_field_cardinality_ratio=*/0.6};
+    // Two objects with disjoint single-occurrence keys: with a 0.6 ratio every field is below the
+    // cardinality threshold, so the object contributes no typed field and the column is dropped.
+    auto samples = Samples({R"({"a": 1})", R"({"b": 2})"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred,
+                         InferColumn(strict_infer, samples));
+    ASSERT_EQ(inferred, nullptr);
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/data/variant/variant_access_utils_test.cpp b/src/paimon/common/data/variant/variant_access_utils_test.cpp
new file mode 100644
index 00000000..6c9803d9
--- /dev/null
+++ b/src/paimon/common/data/variant/variant_access_utils_test.cpp
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/data/variant/variant_access_utils.h"
+
+#include 
+#include 
+#include 
+
+#include "arrow/api.h"
+#include "arrow/util/key_value_metadata.h"
+#include "gtest/gtest.h"
+#include "paimon/common/data/variant/variant_defs.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+namespace {
+
+// An Arrow field carrying a `paimon.description` metadata entry.
+std::shared_ptr DescribedField(const std::string& name,
+                                             const std::shared_ptr& type,
+                                             const std::string& description) {
+    return arrow::field(name, type, /*nullable=*/true,
+                        arrow::key_value_metadata({DataField::DESCRIPTION}, {description}));
+}
+
+std::shared_ptr AccessProjection(
+    const std::vector>& children) {
+    return arrow::field("v", arrow::struct_(children));
+}
+
+// A shredded file field: struct{[metadata], value, typed_value: struct{}}.
+std::shared_ptr ShreddedFile(
+    const std::vector>& typed_children, bool with_metadata = true) {
+    arrow::FieldVector fields;
+    if (with_metadata) {
+        fields.push_back(
+            arrow::field(std::string(VariantDefs::kMetadataFieldName), arrow::binary()));
+    }
+    fields.push_back(arrow::field(std::string(VariantDefs::kValueFieldName), arrow::binary()));
+    fields.push_back(arrow::field(std::string(VariantDefs::kTypedValueFieldName),
+                                  arrow::struct_(typed_children)));
+    return arrow::field("v", arrow::struct_(fields));
+}
+
+std::vector ObjectKeySpecs() {
+    auto proj = AccessProjection({DescribedField(
+        "a", arrow::int32(), VariantAccessUtils::BuildVariantMetadata("$.a", false, "UTC"))});
+    auto result = VariantAccessUtils::ParseAccessSpecs(proj);
+    EXPECT_TRUE(result.ok()) << result.status().ToString();
+    return result.value();
+}
+
+}  // namespace
+
+TEST(VariantAccessUtilsTest, ParseAccessSpecsRejectsNonProjection) {
+    // A plain struct without access descriptions is not a variant-access projection.
+    auto plain = arrow::field("v", arrow::struct_({arrow::field("x", arrow::int32())}));
+    ASSERT_FALSE(VariantAccessUtils::IsVariantAccessType(plain->type()));
+    ASSERT_NOK(VariantAccessUtils::ParseAccessSpecs(plain));
+}
+
+TEST(VariantAccessUtilsTest, ParseAccessSpecsRejectsMalformedDescription) {
+    const std::string key = VariantAccessUtils::kMetadataKey;
+    // A description with no delimiter splits into a single part.
+    auto no_delim = AccessProjection({DescribedField("a", arrow::utf8(), key + "$.a")});
+    ASSERT_TRUE(VariantAccessUtils::IsVariantAccessType(no_delim->type()));
+    ASSERT_NOK(VariantAccessUtils::ParseAccessSpecs(no_delim));
+    // A single delimiter splits into two parts (still not the three that a spec needs).
+    auto one_delim = AccessProjection({DescribedField("a", arrow::utf8(), key + "$.a;true")});
+    ASSERT_TRUE(VariantAccessUtils::IsVariantAccessType(one_delim->type()));
+    ASSERT_NOK(VariantAccessUtils::ParseAccessSpecs(one_delim));
+}
+
+TEST(VariantAccessUtilsTest, ParseAccessSpecsRoundTripsBuildMetadata) {
+    auto proj = AccessProjection({DescribedField(
+        "a", arrow::int32(), VariantAccessUtils::BuildVariantMetadata("$.a", true, "+08:00"))});
+    ASSERT_OK_AND_ASSIGN(std::vector specs,
+                         VariantAccessUtils::ParseAccessSpecs(proj));
+    ASSERT_EQ(specs.size(), 1);
+    EXPECT_EQ(specs[0].path, "$.a");
+    EXPECT_TRUE(specs[0].cast_args.fail_on_error);
+    EXPECT_EQ(specs[0].cast_args.zone_id, "+08:00");
+}
+
+TEST(VariantAccessUtilsTest, ClipRejectsNonStructFileField) {
+    auto non_struct = arrow::field("v", arrow::int32());
+    ASSERT_NOK(VariantAccessUtils::ClipShreddedFileField(ObjectKeySpecs(), non_struct));
+}
+
+TEST(VariantAccessUtilsTest, ClipUnprunablePathsReturnFileUnchanged) {
+    auto file = ShreddedFile({arrow::field("a", arrow::int32())});
+
+    // A root path needs the whole variant, so the file field is returned unchanged.
+    auto root_proj = AccessProjection({DescribedField(
+        "r", arrow::utf8(), VariantAccessUtils::BuildVariantMetadata("$", false, "UTC"))});
+    ASSERT_OK_AND_ASSIGN(std::vector root_specs,
+                         VariantAccessUtils::ParseAccessSpecs(root_proj));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr root_clipped,
+                         VariantAccessUtils::ClipShreddedFileField(root_specs, file));
+    EXPECT_EQ(root_clipped, file);
+
+    // An array-first path likewise cannot be pruned to a top-level key.
+    auto array_proj = AccessProjection({DescribedField(
+        "e", arrow::utf8(), VariantAccessUtils::BuildVariantMetadata("$[0]", false, "UTC"))});
+    ASSERT_OK_AND_ASSIGN(std::vector array_specs,
+                         VariantAccessUtils::ParseAccessSpecs(array_proj));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr array_clipped,
+                         VariantAccessUtils::ClipShreddedFileField(array_specs, file));
+    EXPECT_EQ(array_clipped, file);
+}
+
+TEST(VariantAccessUtilsTest, ClipUnshreddedFileFieldReturnedUnchanged) {
+    // No typed_value column: there is nothing to prune.
+    auto unshredded = arrow::field(
+        "v",
+        arrow::struct_({arrow::field(std::string(VariantDefs::kMetadataFieldName), arrow::binary()),
+                        arrow::field(std::string(VariantDefs::kValueFieldName), arrow::binary())}));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr clipped,
+                         VariantAccessUtils::ClipShreddedFileField(ObjectKeySpecs(), unshredded));
+    EXPECT_EQ(clipped, unshredded);
+}
+
+TEST(VariantAccessUtilsTest, ClipMissingMetadataFails) {
+    auto file_no_metadata =
+        ShreddedFile({arrow::field("a", arrow::int32())}, /*with_metadata=*/false);
+    ASSERT_NOK(VariantAccessUtils::ClipShreddedFileField(ObjectKeySpecs(), file_no_metadata));
+}
+
+TEST(VariantAccessUtilsTest, ClipNarrowsToRequestedKeys) {
+    // "$.a" is requested; the shredded typed_value keeps only "a" and drops the unrelated "b".
+    auto file = ShreddedFile({arrow::field("a", arrow::int32()), arrow::field("b", arrow::utf8())});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr clipped,
+                         VariantAccessUtils::ClipShreddedFileField(ObjectKeySpecs(), file));
+    ASSERT_EQ(clipped->type()->id(), arrow::Type::STRUCT);
+    const auto& clipped_struct = static_cast(*clipped->type());
+    // metadata is always kept; value is dropped because "a" is shredded; typed_value keeps "a".
+    ASSERT_NE(clipped_struct.GetFieldByName(VariantDefs::kMetadataFieldName), nullptr);
+    EXPECT_EQ(clipped_struct.GetFieldByName(VariantDefs::kValueFieldName), nullptr);
+    auto typed = clipped_struct.GetFieldByName(VariantDefs::kTypedValueFieldName);
+    ASSERT_NE(typed, nullptr);
+    ASSERT_EQ(typed->type()->num_fields(), 1);
+    EXPECT_EQ(typed->type()->field(0)->name(), "a");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/data/variant/variant_get_test.cpp b/src/paimon/common/data/variant/variant_get_test.cpp
index ba8d592f..159a573d 100644
--- a/src/paimon/common/data/variant/variant_get_test.cpp
+++ b/src/paimon/common/data/variant/variant_get_test.cpp
@@ -19,6 +19,7 @@
 
 #include "paimon/common/data/variant/variant_get.h"
 
+#include 
 #include 
 
 #include "arrow/api.h"
@@ -79,6 +80,18 @@ class VariantGetTest : public ::testing::Test {
                                               arrow_pool_);
     }
 
+    // Builds a single-scalar variant using the direct append API, which can encode types that
+    // JSON parsing never produces (uuid/date/float/binary/timestamp).
+    std::shared_ptr BuildScalar(
+        const std::function& append) {
+        VariantBuilder builder(/*allow_duplicate_keys=*/false);
+        Status st = append(builder);
+        EXPECT_TRUE(st.ok()) << st.ToString();
+        auto result = builder.Build(pool_);
+        EXPECT_TRUE(result.ok()) << result.status().ToString();
+        return result.value();
+    }
+
  protected:
     std::shared_ptr pool_ = GetDefaultPool();
     std::shared_ptr arrow_pool_ = GetArrowPool(pool_);
@@ -335,4 +348,121 @@ TEST_F(VariantGetTest, NestedTargetNullSemantics) {
     ASSERT_TRUE(array->IsNull(0));
 }
 
+TEST_F(VariantGetTest, UuidSourceCastsToStringOnly) {
+    std::string uuid_bytes(16, '\0');
+    for (int i = 0; i < 16; ++i) {
+        uuid_bytes[i] = static_cast(i);
+    }
+    std::shared_ptr variant =
+        BuildScalar([&](VariantBuilder& b) { return b.AppendUuid(uuid_bytes); });
+    // A UUID has no Paimon type, so it can only be rendered as its canonical string.
+    ASSERT_OK_AND_ASSIGN(std::optional as_string,
+                         VariantGetExecutor::Get(variant, "$", arrow::utf8(), cast_args_));
+    ASSERT_TRUE(as_string.has_value());
+    ASSERT_EQ(as_string->GetValue(), "00010203-0405-0607-0809-0a0b0c0d0e0f");
+    // Any non-string target is an invalid cast, which is SQL NULL when fail_on_error is false.
+    ASSERT_OK_AND_ASSIGN(std::optional as_long,
+                         VariantGetExecutor::Get(variant, "$", arrow::int64(), cast_args_));
+    ASSERT_FALSE(as_long.has_value());
+    cast_args_.fail_on_error = true;
+    ASSERT_NOK(VariantGetExecutor::Get(variant, "$", arrow::int64(), cast_args_));
+}
+
+TEST_F(VariantGetTest, FloatingPointToStringMatchesJava) {
+    // Doubles and floats are stringified via the Java formatting, not the arrow cast.
+    ASSERT_EQ(GetString("$.double", arrow::utf8()), "1.0123456789012346");
+    std::shared_ptr f =
+        BuildScalar([](VariantBuilder& b) { return b.AppendFloat(1.5f); });
+    ASSERT_OK_AND_ASSIGN(std::optional as_string,
+                         VariantGetExecutor::Get(f, "$", arrow::utf8(), cast_args_));
+    ASSERT_TRUE(as_string.has_value());
+    ASSERT_EQ(as_string->GetValue(), "1.5");
+    ASSERT_OK_AND_ASSIGN(std::optional as_float,
+                         VariantGetExecutor::Get(f, "$", arrow::float32(), cast_args_));
+    ASSERT_TRUE(as_float.has_value());
+    ASSERT_EQ(as_float->GetValue(), 1.5f);
+}
+
+TEST_F(VariantGetTest, DateSource) {
+    std::shared_ptr d =
+        BuildScalar([](VariantBuilder& b) { return b.AppendDate(19000); });
+    ASSERT_OK_AND_ASSIGN(std::optional as_date,
+                         VariantGetExecutor::Get(d, "$", arrow::date32(), cast_args_));
+    ASSERT_TRUE(as_date.has_value());
+    ASSERT_EQ(as_date->GetValue(), 19000);
+}
+
+TEST_F(VariantGetTest, MissingCastExecutorYieldsNull) {
+    // No binary->int64 cast executor exists, so the cast is treated as invalid (SQL NULL).
+    std::shared_ptr bin = BuildScalar(
+        [](VariantBuilder& b) { return b.AppendBinary(std::string_view("\x01\x02", 2)); });
+    ASSERT_OK_AND_ASSIGN(std::optional as_long,
+                         VariantGetExecutor::Get(bin, "$", arrow::int64(), cast_args_));
+    ASSERT_FALSE(as_long.has_value());
+}
+
+TEST_F(VariantGetTest, ScalarArrowBuilders) {
+    auto build_array = [&](const std::shared_ptr& variant, const std::string& path,
+                           const std::shared_ptr& type) {
+        auto result = VariantGetExecutor::GetAsArrow(variant, path, arrow::field("x", type),
+                                                     cast_args_, pool_, arrow_pool_);
+        EXPECT_TRUE(result.ok()) << result.status().ToString();
+        return result.value();
+    };
+
+    std::shared_ptr boolean = build_array(variant_, "$.boolean1", arrow::boolean());
+    ASSERT_TRUE(static_cast(*boolean).Value(0));
+
+    // INT8/INT16/INT32 each require a narrowing cast from the int64 source.
+    std::shared_ptr int8 = build_array(variant_, "$.object.age", arrow::int8());
+    ASSERT_EQ(static_cast(*int8).Value(0), 2);
+    std::shared_ptr int16 = build_array(variant_, "$.object.age", arrow::int16());
+    ASSERT_EQ(static_cast(*int16).Value(0), 2);
+    std::shared_ptr int32 = build_array(variant_, "$.object.age", arrow::int32());
+    ASSERT_EQ(static_cast(*int32).Value(0), 2);
+
+    std::shared_ptr float64 = build_array(variant_, "$.double", arrow::float64());
+    ASSERT_DOUBLE_EQ(static_cast(*float64).Value(0), 1.0123456789012346);
+
+    std::shared_ptr f =
+        BuildScalar([](VariantBuilder& b) { return b.AppendFloat(1.5f); });
+    std::shared_ptr float32 = build_array(f, "$", arrow::float32());
+    ASSERT_EQ(static_cast(*float32).Value(0), 1.5f);
+
+    std::shared_ptr decimal =
+        build_array(variant_, "$.decimal", arrow::decimal128(5, 2));
+    ASSERT_EQ(static_cast(*decimal).FormatValue(0), "100.99");
+
+    std::shared_ptr bin = BuildScalar(
+        [](VariantBuilder& b) { return b.AppendBinary(std::string_view("\x01\x02\x03", 3)); });
+    std::shared_ptr binary = build_array(bin, "$", arrow::binary());
+    ASSERT_EQ(static_cast(*binary).GetView(0),
+              std::string_view("\x01\x02\x03", 3));
+
+    std::shared_ptr d =
+        BuildScalar([](VariantBuilder& b) { return b.AppendDate(19000); });
+    std::shared_ptr date = build_array(d, "$", arrow::date32());
+    ASSERT_EQ(static_cast(*date).Value(0), 19000);
+
+    // Only the microsecond unit matches the variant's internal timestamp representation; other
+    // units require a timestamp->timestamp literal cast that is intentionally unsupported.
+    std::shared_ptr ts =
+        BuildScalar([](VariantBuilder& b) { return b.AppendTimestamp(1700000000123456); });
+    std::shared_ptr timestamp =
+        build_array(ts, "$", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"));
+    ASSERT_EQ(static_cast(*timestamp).Value(0), 1700000000123456);
+}
+
+TEST_F(VariantGetTest, ListFromNonArrayIsNull) {
+    auto target = arrow::field("l", arrow::list(arrow::int64()));
+    // An object cannot cast to a list: SQL NULL when fail_on_error is false ...
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr array, GetAsArrow("$.object", target));
+    ASSERT_TRUE(array->IsNull(0));
+    // ... and an error when fail_on_error is true.
+    cast_args_.fail_on_error = true;
+    auto result = VariantGetExecutor::GetAsArrow(variant_, "$.object", target, cast_args_, pool_,
+                                                 arrow_pool_);
+    ASSERT_NOK(result);
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/data/variant/variant_json_utils_test.cpp b/src/paimon/common/data/variant/variant_json_utils_test.cpp
index e54c37d7..d2cf513e 100644
--- a/src/paimon/common/data/variant/variant_json_utils_test.cpp
+++ b/src/paimon/common/data/variant/variant_json_utils_test.cpp
@@ -21,8 +21,10 @@
 
 #include 
 #include 
+#include 
 
 #include "gtest/gtest.h"
+#include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
 
@@ -72,4 +74,48 @@ TEST(VariantJsonUtilsTest, JavaFloatToString) {
     EXPECT_EQ(VariantJsonUtils::JavaFloatToString(2.5F), "2.5");
 }
 
+TEST(VariantJsonUtilsTest, AppendEscapedJsonControlChars) {
+    // Control characters without a named escape are emitted as \uXXXX.
+    std::string out;
+    VariantJsonUtils::AppendEscapedJson(std::string_view("\x01\x1f", 2), &out);
+    EXPECT_EQ(out, "\"\\u0001\\u001f\"");
+}
+
+TEST(VariantJsonUtilsTest, DateToStringNegativeYear) {
+    // A date far before the epoch renders a negative (BCE) year with a leading '-'.
+    std::string result = VariantJsonUtils::DateToString(-800000);
+    ASSERT_FALSE(result.empty());
+    EXPECT_EQ(result.front(), '-');
+}
+
+TEST(VariantJsonUtilsTest, TimestampToStringEdgeCases) {
+    // Sub-second fraction trailing zeros are trimmed.
+    EXPECT_EQ(VariantJsonUtils::TimestampToString(500000, 0, /*with_offset=*/false),
+              "1970-01-01 00:00:00.5");
+    // Negative epoch micros floor to the previous day (FloorDiv keeps a non-negative remainder).
+    EXPECT_EQ(VariantJsonUtils::TimestampToString(-1, 0, /*with_offset=*/false),
+              "1969-12-31 23:59:59.999999");
+    // A positive zone offset is appended as +HH:MM.
+    EXPECT_EQ(VariantJsonUtils::TimestampToString(0, 8 * 3600, /*with_offset=*/true),
+              "1970-01-01 08:00:00+08:00");
+}
+
+TEST(VariantJsonUtilsTest, ZoneOffsetParsing) {
+    // `UTC`/`GMT`/`UT` prefixes are stripped before the fixed offset is parsed.
+    ASSERT_OK_AND_ASSIGN(int32_t utc_prefixed,
+                         VariantJsonUtils::GetZoneOffsetSeconds("UTC+08:00", 0));
+    EXPECT_EQ(utc_prefixed, 8 * 3600);
+    ASSERT_OK_AND_ASSIGN(int32_t ut_prefixed, VariantJsonUtils::GetZoneOffsetSeconds("UT+05", 0));
+    EXPECT_EQ(ut_prefixed, 5 * 3600);
+    // HHMMSS form and a negative offset.
+    ASSERT_OK_AND_ASSIGN(int32_t hms, VariantJsonUtils::GetZoneOffsetSeconds("+18:30:15", 0));
+    EXPECT_EQ(hms, 18 * 3600 + 30 * 60 + 15);
+    ASSERT_OK_AND_ASSIGN(int32_t neg, VariantJsonUtils::GetZoneOffsetSeconds("-06:30", 0));
+    EXPECT_EQ(neg, -(6 * 3600 + 30 * 60));
+    // Invalid forms are rejected: a non-digit char, a wrong digit count, and an out-of-range hour.
+    ASSERT_NOK(VariantJsonUtils::GetZoneOffsetSeconds("+9A", 0));
+    ASSERT_NOK(VariantJsonUtils::GetZoneOffsetSeconds("+123", 0));
+    ASSERT_NOK(VariantJsonUtils::GetZoneOffsetSeconds("+19:00", 0));
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp b/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp
new file mode 100644
index 00000000..6085d1da
--- /dev/null
+++ b/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/data/variant/variant_shredding_read_plan_factory.h"
+
+#include 
+#include 
+#include 
+#include 
+
+#include "arrow/api.h"
+#include "arrow/util/key_value_metadata.h"
+#include "gtest/gtest.h"
+#include "paimon/common/data/variant/generic_variant.h"
+#include "paimon/common/data/variant/variant_access_utils.h"
+#include "paimon/common/data/variant/variant_defs.h"
+#include "paimon/common/data/variant/variant_shredding_utils.h"
+#include "paimon/common/data/variant/variant_shredding_writer.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+class VariantShreddingReadPlanFactoryTest : public ::testing::Test {
+ public:
+    std::shared_ptr Variant(const std::string& json) {
+        auto result = GenericVariant::FromJson(json, pool_);
+        EXPECT_TRUE(result.ok()) << result.status().ToString();
+        return result.value();
+    }
+
+    // The full shredded StructArray (struct{metadata, value, typed_value}) for one variant, using
+    // the production writer.
+    void MakeFullShredded(const std::shared_ptr& logical,
+                          const std::shared_ptr& variant,
+                          std::shared_ptr* physical_type,
+                          std::shared_ptr* array) {
+        ASSERT_OK_AND_ASSIGN(*physical_type,
+                             VariantShreddingUtils::VariantShreddingSchema(logical));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr schema,
+                             VariantShreddingUtils::BuildVariantSchema(*physical_type));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr writer,
+                             VariantShreddedColumnWriter::Create(schema, *physical_type,
+                                                                 arrow::default_memory_pool()));
+        ASSERT_OK(writer->Append(*variant));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr arr, writer->Finish());
+        *array = std::static_pointer_cast(arr);
+    }
+
+    // The unshredded physical StructArray (struct{value, metadata}) for one variant.
+    std::shared_ptr MakeUnshredded(const std::shared_ptr& v) {
+        auto value = v->Value();
+        EXPECT_TRUE(value.ok()) << value.status().ToString();
+        return MakeUnshreddedRaw(value.value(), v->Metadata(), /*metadata_null=*/false);
+    }
+
+    std::shared_ptr MakeUnshreddedRaw(std::string_view value,
+                                                          std::string_view metadata,
+                                                          bool metadata_null) {
+        arrow::BinaryBuilder value_builder;
+        EXPECT_TRUE(value_builder.Append(value).ok());
+        std::shared_ptr value_array;
+        EXPECT_TRUE(value_builder.Finish(&value_array).ok());
+        arrow::BinaryBuilder metadata_builder;
+        if (metadata_null) {
+            EXPECT_TRUE(metadata_builder.AppendNull().ok());
+        } else {
+            EXPECT_TRUE(metadata_builder.Append(metadata).ok());
+        }
+        std::shared_ptr metadata_array;
+        EXPECT_TRUE(metadata_builder.Finish(&metadata_array).ok());
+        auto made = arrow::StructArray::Make({value_array, metadata_array},
+                                             std::vector{"value", "metadata"});
+        EXPECT_TRUE(made.ok()) << made.status().ToString();
+        return made.ValueOrDie();
+    }
+
+    // A variant-access child: an arrow field carrying a `__VARIANT_METADATA` description.
+    static std::shared_ptr AccessChild(const std::string& name,
+                                                     const std::shared_ptr& type,
+                                                     const std::string& path) {
+        return arrow::field(name, type, /*nullable=*/true,
+                            arrow::key_value_metadata({DataField::DESCRIPTION},
+                                                      {VariantAccessUtils::BuildVariantMetadata(
+                                                          path, /*fail_on_error=*/false, "UTC")}));
+    }
+
+    std::shared_ptr CreatePlan(
+        const std::shared_ptr& read_field,
+        const std::shared_ptr& file_field) {
+        auto plans = VariantShreddingReadPlanFactory::CreateReadPlans(
+            arrow::schema({read_field}), arrow::schema({file_field}), pool_);
+        EXPECT_TRUE(plans.ok()) << plans.status().ToString();
+        auto it = plans.value().find(read_field->name());
+        EXPECT_NE(it, plans.value().end());
+        return it->second;
+    }
+
+    static std::shared_ptr Int32Array(int32_t value) {
+        arrow::Int32Builder builder;
+        EXPECT_TRUE(builder.Append(value).ok());
+        std::shared_ptr array;
+        EXPECT_TRUE(builder.Finish(&array).ok());
+        return array;
+    }
+
+ protected:
+    std::shared_ptr pool_ = GetDefaultPool();
+};
+
+TEST_F(VariantShreddingReadPlanFactoryTest, FullVariantReadOfShreddedFile) {
+    std::shared_ptr variant = Variant(R"({"a": 5})");
+    std::shared_ptr physical;
+    std::shared_ptr shredded;
+    MakeFullShredded(arrow::struct_({arrow::field("a", arrow::int64())}), variant, &physical,
+                     &shredded);
+    ASSERT_FALSE(HasFatalFailure());
+
+    auto read_field = VariantTypeUtils::ToArrowField("v");
+    auto file_field = arrow::field("v", physical);
+    std::shared_ptr plan = CreatePlan(read_field, file_field);
+    ASSERT_NE(plan, nullptr);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled,
+                         plan->Assemble(shredded, arrow::default_memory_pool()));
+    auto assembled_struct = std::static_pointer_cast(assembled);
+    auto value_column = std::static_pointer_cast(assembled_struct->field(0));
+    auto metadata_column = std::static_pointer_cast(assembled_struct->field(1));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr rebuilt,
+        GenericVariant::Create(value_column->GetView(0), metadata_column->GetView(0), pool_));
+    ASSERT_OK_AND_ASSIGN(std::string json, rebuilt->ToJson());
+    EXPECT_EQ(json, R"({"a":5})");
+
+    // Assembling a non-struct physical array is an error.
+    auto ints = Int32Array(1);
+    ASSERT_NOK(plan->Assemble(ints, arrow::default_memory_pool()));
+}
+
+TEST_F(VariantShreddingReadPlanFactoryTest, AccessProjectionOnUnshreddedFile) {
+    std::shared_ptr variant = Variant(R"({"a": 5, "b": [10, 20], "c": "hi"})");
+    std::shared_ptr unshredded = MakeUnshredded(variant);
+    ASSERT_FALSE(HasFatalFailure());
+
+    auto read_field =
+        arrow::field("v", arrow::struct_({AccessChild("a", arrow::int64(), "$.a"),
+                                          AccessChild("c", arrow::utf8(), "$.c"),
+                                          AccessChild("b0", arrow::int64(), "$.b[0]"),
+                                          AccessChild("missing", arrow::utf8(), "$.missing")}));
+    // Unshredded file column: struct{value, metadata}.
+    auto file_field = arrow::field("v", VariantTypeUtils::UnshreddedStructType());
+    std::shared_ptr plan = CreatePlan(read_field, file_field);
+    ASSERT_NE(plan, nullptr);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled,
+                         plan->Assemble(unshredded, arrow::default_memory_pool()));
+    auto row = std::static_pointer_cast(assembled);
+    ASSERT_EQ(row->length(), 1);
+    EXPECT_EQ(static_cast(*row->field(0)).Value(0), 5);
+    EXPECT_EQ(static_cast(*row->field(1)).GetString(0), "hi");
+    EXPECT_EQ(static_cast(*row->field(2)).Value(0), 10);
+    EXPECT_TRUE(row->field(3)->IsNull(0));
+
+    // A non-struct physical array is an error.
+    auto ints = Int32Array(1);
+    ASSERT_NOK(plan->Assemble(ints, arrow::default_memory_pool()));
+}
+
+TEST_F(VariantShreddingReadPlanFactoryTest, AccessProjectionRejectsNullMetadata) {
+    std::shared_ptr variant = Variant(R"({"a": 5})");
+    ASSERT_OK_AND_ASSIGN(std::string_view value, variant->Value());
+    std::shared_ptr bad =
+        MakeUnshreddedRaw(value, variant->Metadata(), /*metadata_null=*/true);
+    ASSERT_FALSE(HasFatalFailure());
+
+    auto read_field = arrow::field("v", arrow::struct_({AccessChild("a", arrow::int64(), "$.a")}));
+    auto file_field = arrow::field("v", VariantTypeUtils::UnshreddedStructType());
+    std::shared_ptr plan = CreatePlan(read_field, file_field);
+    ASSERT_NE(plan, nullptr);
+    ASSERT_NOK(plan->Assemble(bad, arrow::default_memory_pool()));
+}
+
+TEST_F(VariantShreddingReadPlanFactoryTest, AccessProjectionOnShreddedFile) {
+    // Extracting shredded object keys reads the typed sub-columns directly (with pruning).
+    std::shared_ptr variant = Variant(R"({"a": 5, "b": "hi"})");
+    std::shared_ptr physical;
+    std::shared_ptr shredded;
+    MakeFullShredded(
+        arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::utf8())}),
+        variant, &physical, &shredded);
+    ASSERT_FALSE(HasFatalFailure());
+
+    auto read_field = arrow::field("v", arrow::struct_({AccessChild("a", arrow::int64(), "$.a"),
+                                                        AccessChild("b", arrow::utf8(), "$.b")}));
+    auto file_field = arrow::field("v", physical);
+    std::shared_ptr plan = CreatePlan(read_field, file_field);
+    ASSERT_NE(plan, nullptr);
+
+    // With both keys shredded and requested, pruning keeps {metadata, typed_value} and drops the
+    // top-level `value` column; project the full array down to match the pushed-down physical.
+    auto made = arrow::StructArray::Make({shredded->field(0), shredded->field(2)},
+                                         {std::string(VariantDefs::kMetadataFieldName),
+                                          std::string(VariantDefs::kTypedValueFieldName)});
+    ASSERT_TRUE(made.ok()) << made.status().ToString();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled,
+                         plan->Assemble(made.ValueOrDie(), arrow::default_memory_pool()));
+    auto row = std::static_pointer_cast(assembled);
+    EXPECT_EQ(static_cast(*row->field(0)).Value(0), 5);
+    EXPECT_EQ(static_cast(*row->field(1)).GetString(0), "hi");
+}
+
+TEST_F(VariantShreddingReadPlanFactoryTest, NestedVariantColumnAndTypeMismatch) {
+    // A struct column holding a variant child: the plan rebuilds the struct around the reassembled
+    // variant.
+    std::shared_ptr variant = Variant(R"({"a": 5})");
+    std::shared_ptr physical;
+    std::shared_ptr shredded;
+    MakeFullShredded(arrow::struct_({arrow::field("a", arrow::int64())}), variant, &physical,
+                     &shredded);
+    ASSERT_FALSE(HasFatalFailure());
+
+    auto read_field = arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()),
+                                                        VariantTypeUtils::ToArrowField("v")}));
+    auto file_field = arrow::field(
+        "s", arrow::struct_({arrow::field("x", arrow::int32()), arrow::field("v", physical)}));
+    // A sibling plain struct with no variant exercises `ContainsNestedVariant` returning false.
+    auto plain_field = arrow::field("plain", arrow::struct_({arrow::field("y", arrow::int32())}));
+    auto plans_result = VariantShreddingReadPlanFactory::CreateReadPlans(
+        arrow::schema({plain_field, read_field}), arrow::schema({plain_field, file_field}), pool_);
+    ASSERT_OK_AND_ASSIGN(auto plans, std::move(plans_result));
+    ASSERT_EQ(plans.count("plain"), 0);
+    ASSERT_EQ(plans.count("s"), 1);
+    std::shared_ptr plan = plans["s"];
+
+    auto x_array = Int32Array(7);
+    auto made = arrow::StructArray::Make({x_array, shredded}, std::vector{"x", "v"});
+    ASSERT_TRUE(made.ok()) << made.status().ToString();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled,
+                         plan->Assemble(made.ValueOrDie(), arrow::default_memory_pool()));
+    auto row = std::static_pointer_cast(assembled);
+    EXPECT_EQ(static_cast(*row->field(0)).Value(0), 7);
+    ASSERT_EQ(row->field(1)->type_id(), arrow::Type::STRUCT);
+
+    // Assembling a physical array whose type differs from the logical struct is an error.
+    auto ints = Int32Array(1);
+    ASSERT_NOK(plan->Assemble(ints, arrow::default_memory_pool()));
+}
+
+TEST_F(VariantShreddingReadPlanFactoryTest, ColumnAbsentInFileSkipped) {
+    // Schema evolution: the variant column is missing from the file, so no plan is produced.
+    auto read_field = VariantTypeUtils::ToArrowField("v");
+    auto other = arrow::field("other", arrow::int32());
+    ASSERT_OK_AND_ASSIGN(auto plans,
+                         VariantShreddingReadPlanFactory::CreateReadPlans(
+                             arrow::schema({read_field}), arrow::schema({other}), pool_));
+    EXPECT_TRUE(plans.empty());
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/data/variant/variant_shredding_test.cpp b/src/paimon/common/data/variant/variant_shredding_test.cpp
index cfbf1571..a2c053ad 100644
--- a/src/paimon/common/data/variant/variant_shredding_test.cpp
+++ b/src/paimon/common/data/variant/variant_shredding_test.cpp
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+#include 
 #include 
 #include 
 #include 
@@ -24,6 +25,7 @@
 #include "arrow/api.h"
 #include "gtest/gtest.h"
 #include "paimon/common/data/variant/generic_variant.h"
+#include "paimon/common/data/variant/variant_builder.h"
 #include "paimon/common/data/variant/variant_defs.h"
 #include "paimon/common/data/variant/variant_reassembler.h"
 #include "paimon/common/data/variant/variant_schema.h"
@@ -90,6 +92,78 @@ class VariantShreddingTest : public ::testing::Test {
     }
 
  protected:
+    // The physical shredded arrow type for a logical shredding type.
+    std::shared_ptr Physical(const std::shared_ptr& logical) {
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr physical,
+                             VariantShreddingUtils::VariantShreddingSchema(logical));
+        return physical;
+    }
+
+    std::shared_ptr Json(const char* json) {
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr variant,
+                             GenericVariant::FromJson(json, pool_));
+        return variant;
+    }
+
+    // Builds a single variant using the direct append API, which can encode types (float, binary,
+    // date, timestamp) that JSON parsing never produces.
+    std::shared_ptr BuildVariant(
+        const std::function& append) {
+        VariantBuilder builder(/*allow_duplicate_keys=*/false);
+        Status st = append(builder);
+        EXPECT_TRUE(st.ok()) << st.ToString();
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr variant, builder.Build(pool_));
+        return variant;
+    }
+
+    // Shreds the given variants (nullptr = null row) against a physical shredded type, reassembles
+    // them, asserts each reassembled variant renders back to the same JSON, and returns the
+    // shredded array. Unlike `RoundTrip`, the physical type is provided directly so that typed
+    // columns unsupported by `VariantShreddingSchema` (date/timestamp) can be exercised.
+    std::shared_ptr ShredAndCheck(
+        const std::shared_ptr& physical,
+        const std::vector>& variants) {
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr schema,
+                             VariantShreddingUtils::BuildVariantSchema(physical));
+        EXPECT_OK_AND_ASSIGN(
+            std::unique_ptr writer,
+            VariantShreddedColumnWriter::Create(schema, physical, arrow::default_memory_pool()));
+        std::vector expected_jsons;
+        for (const auto& variant : variants) {
+            if (variant == nullptr) {
+                EXPECT_OK(writer->AppendNull());
+                expected_jsons.emplace_back();
+                continue;
+            }
+            EXPECT_OK_AND_ASSIGN(std::string expected_json, variant->ToJson());
+            expected_jsons.push_back(std::move(expected_json));
+            EXPECT_OK(writer->Append(*variant));
+        }
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr shredded_array, writer->Finish());
+        auto shredded = std::static_pointer_cast(shredded_array);
+
+        EXPECT_OK_AND_ASSIGN(std::shared_ptr assembled_array,
+                             VariantReassembler::AssembleVariantArray(
+                                 shredded, schema, pool_, arrow::default_memory_pool()));
+        auto assembled = std::static_pointer_cast(assembled_array);
+        auto value_column = std::static_pointer_cast(assembled->field(0));
+        auto metadata_column = std::static_pointer_cast(assembled->field(1));
+        for (size_t i = 0; i < variants.size(); ++i) {
+            SCOPED_TRACE("row " + std::to_string(i));
+            if (variants[i] == nullptr) {
+                EXPECT_TRUE(assembled->IsNull(i));
+                continue;
+            }
+            EXPECT_FALSE(assembled->IsNull(i));
+            EXPECT_OK_AND_ASSIGN(std::shared_ptr variant,
+                                 GenericVariant::Create(value_column->GetView(i),
+                                                        metadata_column->GetView(i), pool_));
+            EXPECT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson());
+            EXPECT_EQ(actual_json, expected_jsons[i]);
+        }
+        return shredded;
+    }
+
     std::shared_ptr pool_ = GetDefaultPool();
 };
 
@@ -242,4 +316,155 @@ TEST_F(VariantShreddingTest, ShredScalarsAndMismatches) {
                            "{\"arr\": [1, 2]}", R"({"arr": {"k": "v"}})"});
 }
 
+TEST_F(VariantShreddingTest, ScalarShreddingIntegerWidths) {
+    // int8/int16 targets: in-range longs shred into the narrow typed column, while out-of-range
+    // values fall back to the residual value column. Decimal-integral inputs take the
+    // decimal->integer shredding path.
+    ShredAndCheck(Physical(arrow::struct_({arrow::field("x", arrow::int8())})),
+                  {Json(R"({"x": 5})"), Json(R"({"x": 200})"), Json(R"({"x": 5.0})")});
+    ShredAndCheck(Physical(arrow::struct_({arrow::field("x", arrow::int16())})),
+                  {Json(R"({"x": 5})"), Json(R"({"x": 40000})"), Json(R"({"x": 5.0})")});
+}
+
+TEST_F(VariantShreddingTest, ScalarShreddingFloatAndBinary) {
+    // Float and binary variants are not producible from JSON, so build them directly. They shred
+    // into their typed columns and round-trip back to the same value.
+    ShredAndCheck(Physical(arrow::float32()),
+                  {BuildVariant([](VariantBuilder& b) { return b.AppendFloat(1.5f); }), nullptr});
+    ShredAndCheck(Physical(arrow::binary()), {BuildVariant([](VariantBuilder& b) {
+                      return b.AppendBinary(std::string_view("\x01\x02\x03", 3));
+                  })});
+}
+
+TEST_F(VariantShreddingTest, ScalarShreddingDate) {
+    // date typed columns are produced by external engines; `VariantShreddingSchema` itself never
+    // emits them, so build the physical shredded type directly.
+    auto physical = arrow::struct_({arrow::field("metadata", arrow::binary(), false),
+                                    arrow::field("value", arrow::binary(), true),
+                                    arrow::field("typed_value", arrow::date32(), true)});
+    ShredAndCheck(physical,
+                  {BuildVariant([](VariantBuilder& b) { return b.AppendDate(19000); }), nullptr});
+}
+
+TEST_F(VariantShreddingTest, TimestampSchemaParsing) {
+    auto make_physical = [](const std::shared_ptr& ts) {
+        return arrow::struct_({arrow::field("metadata", arrow::binary(), false),
+                               arrow::field("value", arrow::binary(), true),
+                               arrow::field("typed_value", ts, true)});
+    };
+    // A microsecond timestamp with a timezone parses as TIMESTAMP_LTZ; without, TIMESTAMP_NTZ.
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr ltz,
+                         VariantShreddingUtils::BuildVariantSchema(
+                             make_physical(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"))));
+    ASSERT_TRUE(ltz->scalar_schema.has_value());
+    ASSERT_EQ(ltz->scalar_schema->kind, VariantSchema::ScalarKind::kTimestampLtz);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr ntz,
+                         VariantShreddingUtils::BuildVariantSchema(
+                             make_physical(arrow::timestamp(arrow::TimeUnit::MICRO))));
+    ASSERT_TRUE(ntz->scalar_schema.has_value());
+    ASSERT_EQ(ntz->scalar_schema->kind, VariantSchema::ScalarKind::kTimestampNtz);
+    // Non-microsecond timestamps cannot represent the variant's microsecond values, so they are
+    // rejected as an invalid shredding schema.
+    ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(
+        make_physical(arrow::timestamp(arrow::TimeUnit::MILLI, "UTC"))));
+}
+
+TEST_F(VariantShreddingTest, TimestampReassembly) {
+    // The writer never produces timestamp typed columns, but the reassembler must handle files
+    // written by engines that do. Build a shredded array with a populated timestamp typed_value
+    // and verify it reassembles into the same variant.
+    auto reassemble_one = [&](const std::shared_ptr& ts_type,
+                              const std::shared_ptr& reference, int64_t micros) {
+        auto physical = arrow::struct_({arrow::field("metadata", arrow::binary(), false),
+                                        arrow::field("value", arrow::binary(), true),
+                                        arrow::field("typed_value", ts_type, true)});
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr schema,
+                             VariantShreddingUtils::BuildVariantSchema(physical));
+        std::string metadata(reference->Metadata());
+        ASSERT_OK_AND_ASSIGN(std::string expected_json, reference->ToJson());
+
+        arrow::BinaryBuilder meta_builder;
+        ASSERT_TRUE(meta_builder.Append(metadata).ok());
+        std::shared_ptr meta_array;
+        ASSERT_TRUE(meta_builder.Finish(&meta_array).ok());
+        arrow::BinaryBuilder value_builder;
+        ASSERT_TRUE(value_builder.AppendNull().ok());
+        std::shared_ptr value_array;
+        ASSERT_TRUE(value_builder.Finish(&value_array).ok());
+        arrow::TimestampBuilder ts_builder(ts_type, arrow::default_memory_pool());
+        ASSERT_TRUE(ts_builder.Append(micros).ok());
+        std::shared_ptr ts_array;
+        ASSERT_TRUE(ts_builder.Finish(&ts_array).ok());
+        auto made = arrow::StructArray::Make({meta_array, value_array, ts_array},
+                                             {"metadata", "value", "typed_value"});
+        ASSERT_TRUE(made.ok()) << made.status().ToString();
+        std::shared_ptr shredded = made.ValueOrDie();
+
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled_array,
+                             VariantReassembler::AssembleVariantArray(
+                                 shredded, schema, pool_, arrow::default_memory_pool()));
+        auto assembled = std::static_pointer_cast(assembled_array);
+        auto value_column = std::static_pointer_cast(assembled->field(0));
+        auto metadata_column = std::static_pointer_cast(assembled->field(1));
+        ASSERT_OK_AND_ASSIGN(
+            std::shared_ptr variant,
+            GenericVariant::Create(value_column->GetView(0), metadata_column->GetView(0), pool_));
+        ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson());
+        ASSERT_EQ(actual_json, expected_json);
+    };
+
+    int64_t micros = 1700000000000000;
+    reassemble_one(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"),
+                   BuildVariant([&](VariantBuilder& b) { return b.AppendTimestamp(micros); }),
+                   micros);
+    reassemble_one(arrow::timestamp(arrow::TimeUnit::MICRO),
+                   BuildVariant([&](VariantBuilder& b) { return b.AppendTimestampNtz(micros); }),
+                   micros);
+}
+
+TEST_F(VariantShreddingTest, ScalarSchemaToArrowType) {
+    using SK = VariantSchema::ScalarKind;
+    auto check = [](VariantSchema::ScalarType scalar,
+                    const std::shared_ptr& expected) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr type,
+                             VariantShreddingUtils::ScalarSchemaToArrowType(scalar));
+        ASSERT_TRUE(type->Equals(*expected)) << type->ToString();
+    };
+    check({SK::kBoolean}, arrow::boolean());
+    check({SK::kByte}, arrow::int8());
+    check({SK::kShort}, arrow::int16());
+    check({SK::kInt}, arrow::int32());
+    check({SK::kLong}, arrow::int64());
+    check({SK::kFloat}, arrow::float32());
+    check({SK::kDouble}, arrow::float64());
+    check({SK::kString}, arrow::utf8());
+    check({SK::kBinary}, arrow::binary());
+    check({SK::kDecimal, 10, 2}, arrow::decimal128(10, 2));
+    check({SK::kDate}, arrow::date32());
+    check({SK::kTimestampLtz}, arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"));
+    check({SK::kTimestampNtz}, arrow::timestamp(arrow::TimeUnit::MICRO));
+    // kUuid has no shredded arrow representation.
+    ASSERT_NOK(VariantShreddingUtils::ScalarSchemaToArrowType({SK::kUuid}));
+}
+
+TEST_F(VariantShreddingTest, InvalidShreddingSchemas) {
+    // Not a struct.
+    ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(arrow::int32()));
+    // Empty struct.
+    ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(arrow::struct_({})));
+    // The "value" column must be binary.
+    ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(
+        arrow::struct_({arrow::field("value", arrow::int32())})));
+    // Unknown field name.
+    ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(
+        arrow::struct_({arrow::field("bogus", arrow::binary())})));
+    // A top-level schema must carry a metadata column.
+    ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(
+        arrow::struct_({arrow::field("value", arrow::binary())})));
+    // Unsupported typed_value type.
+    ASSERT_NOK(VariantShreddingUtils::BuildVariantSchema(arrow::struct_(
+        {arrow::field("metadata", arrow::binary()), arrow::field("value", arrow::binary()),
+         arrow::field("typed_value", arrow::map(arrow::utf8(), arrow::int32()))})));
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/logging/logging_test.cpp b/src/paimon/common/logging/logging_test.cpp
index 88ffc45b..6ffec1f8 100644
--- a/src/paimon/common/logging/logging_test.cpp
+++ b/src/paimon/common/logging/logging_test.cpp
@@ -18,14 +18,59 @@
  */
 #include "paimon/logging.h"
 
+#include 
+#include 
 #include 
+#include 
+#include 
 #include 
+#include 
 #include 
+#include 
 
+#include "glog/log_severity.h"
+#include "glog/logging.h"
+#include "glog/raw_logging.h"
 #include "paimon/common/executor/future.h"
 #include "paimon/executor.h"
+#include "paimon/fs/file_system.h"
 #include "paimon/testing/utils/testharness.h"
 namespace paimon::test {
+namespace {
+
+std::atomic g_creator_calls{0};
+
+// A Logger that forwards to glog exactly like the built-in adaptor. It is used to
+// restore behavior-preserving logging after exercising the custom-creator path,
+// because the registry cannot be reset to "unset" through the public API.
+class GlogForwardingLogger : public Logger {
+ public:
+    void LogV(PaimonLogLevel level, const char* fname, int lineno, const char* /*function*/,
+              const char* fmt, ...) override {
+        va_list args;
+        va_start(args, fmt);
+        google::RawLog__(ToGlog(level), fname, lineno, fmt, args);
+        va_end(args);
+    }
+
+    bool IsLevelEnabled(PaimonLogLevel /*level*/) const override {
+        return true;
+    }
+
+ private:
+    static google::LogSeverity ToGlog(PaimonLogLevel level) {
+        switch (level) {
+            case PAIMON_LOG_LEVEL_WARN:
+                return google::GLOG_WARNING;
+            case PAIMON_LOG_LEVEL_ERROR:
+                return google::GLOG_ERROR;
+            default:
+                return google::GLOG_INFO;
+        }
+    }
+};
+
+}  // namespace
 TEST(LoggerTest, TestMultiThreadGetLogger) {
     ASSERT_OK_AND_ASSIGN(auto executor, CreateDefaultExecutor(/*thread_count=*/4));
     auto get_logger = []() {
@@ -39,4 +84,101 @@ TEST(LoggerTest, TestMultiThreadGetLogger) {
     }
     Wait(futures);
 }
+
+TEST(LoggerTest, TestLogAllSeverities) {
+    // The default logger routes every PaimonLogLevel through the severity mapping.
+    auto logger = Logger::GetLogger("severity_test");
+    ASSERT_TRUE(logger);
+
+    logger->LogV(PAIMON_LOG_LEVEL_DEBUG, __FILE__, __LINE__, __FUNCTION__, "debug severity");
+    logger->LogV(PAIMON_LOG_LEVEL_INFO, __FILE__, __LINE__, __FUNCTION__, "info severity");
+    logger->LogV(PAIMON_LOG_LEVEL_WARN, __FILE__, __LINE__, __FUNCTION__, "warn severity");
+    logger->LogV(PAIMON_LOG_LEVEL_ERROR, __FILE__, __LINE__, __FUNCTION__, "error severity");
+    // NONE and MAX fall into the default branch of the mapping.
+    logger->LogV(PAIMON_LOG_LEVEL_NONE, __FILE__, __LINE__, __FUNCTION__, "none severity");
+    logger->LogV(PAIMON_LOG_LEVEL_MAX, __FILE__, __LINE__, __FUNCTION__, "max severity");
+}
+
+// Demonstrates that glog's normal LOG() path actually writes a log file to disk.
+// Note: Paimon's Logger/GlogAdaptor uses google::RawLog__, which only goes to
+// stderr and never touches disk, so this test drives glog's file sink directly.
+TEST(LoggerTest, TestGlogWritesLogFileToDisk) {
+    // Save the global glog flags we are about to change so other tests are unaffected.
+    const bool prev_logtostderr = FLAGS_logtostderr;
+    const bool prev_timestamp_in_name = FLAGS_timestamp_in_logfile_name;
+    const int32_t prev_minloglevel = FLAGS_minloglevel;
+
+    // A unique, empty directory so the only file inside is the one glog creates for us.
+    // The directory (and everything glog wrote into it) is removed on destruction.
+    auto tmp_dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(tmp_dir);
+    std::shared_ptr fs = tmp_dir->GetFileSystem();
+    const std::string base = tmp_dir->Str() + "/paimon_demo";
+
+    FLAGS_logtostderr = false;                // must be false, otherwise glog skips the file sink
+    FLAGS_timestamp_in_logfile_name = false;  // deterministic file name (no time/pid suffix)
+    FLAGS_minloglevel = google::GLOG_INFO;    // do not filter out INFO
+
+    if (!google::IsGoogleLoggingInitialized()) {
+        google::InitGoogleLogging("paimon-log-disk-test");
+    }
+    // Force INFO logs to a file under our unique directory. Disable the
+    // ".INFO" symlink glog normally creates next to the log file: it would
+    // dangle during recursive deletion and break UniqueTestDirectory cleanup.
+    google::SetLogDestination(google::GLOG_INFO, base.c_str());
+    google::SetLogSymlink(google::GLOG_INFO, "");
+
+    const std::string token = "PAIMON_DISK_LOG_DEMO_" + std::to_string(RandomNumber(0, 1'000'000));
+    LOG(INFO) << "hello from disk logging, token=" << token;
+    google::FlushLogFiles(google::GLOG_INFO);
+
+    // Collect the content of whatever file glog created in our directory.
+    std::string on_disk_path;
+    std::string content;
+    std::vector> entries;
+    ASSERT_OK(fs->ListDir(tmp_dir->Str(), &entries));
+    for (const auto& entry : entries) {
+        if (entry->IsDir()) {
+            continue;
+        }
+        std::string file_content;
+        ASSERT_OK(fs->ReadFile(entry->GetPath(), &file_content));
+        if (file_content.find(token) != std::string::npos) {
+            on_disk_path = entry->GetPath();
+            content = std::move(file_content);
+            break;
+        }
+    }
+
+    // Show the real on-disk file and its raw content so it can be eyeballed in test output.
+    std::cout << "\n===== on-disk glog file: " << on_disk_path << " =====\n"
+              << content << "===== end of file =====\n";
+
+    ASSERT_FALSE(on_disk_path.empty())
+        << "no glog file containing the token was written to " << tmp_dir->Str();
+    ASSERT_NE(content.find(token), std::string::npos);
+
+    // Stop writing INFO logs to the directory that is deleted when tmp_dir goes out of
+    // scope, then restore flags.
+    google::SetLogDestination(google::GLOG_INFO, "");
+    FLAGS_logtostderr = prev_logtostderr;
+    FLAGS_timestamp_in_logfile_name = prev_timestamp_in_name;
+    FLAGS_minloglevel = prev_minloglevel;
+}
+
+// Keep this test last: it installs a process-wide logger creator that cannot be
+// unset, so it must not run before tests that rely on the default logger.
+TEST(LoggerTest, TestRegisterCustomLoggerCreator) {
+    g_creator_calls.store(0);
+    Logger::RegisterLogger([](const std::string& /*path*/) -> std::unique_ptr {
+        g_creator_calls.fetch_add(1);
+        return std::make_unique();
+    });
+
+    auto logger = Logger::GetLogger("custom_path");
+    ASSERT_TRUE(logger);
+    // GetLogger must have gone through the registered creator branch.
+    ASSERT_EQ(1, g_creator_calls.load());
+    ASSERT_TRUE(logger->IsLevelEnabled(PAIMON_LOG_LEVEL_INFO));
+}
 }  // namespace paimon::test
diff --git a/src/paimon/common/utils/arrow/mem_utils_test.cpp b/src/paimon/common/utils/arrow/mem_utils_test.cpp
index d5d28a7a..2e9bda22 100644
--- a/src/paimon/common/utils/arrow/mem_utils_test.cpp
+++ b/src/paimon/common/utils/arrow/mem_utils_test.cpp
@@ -19,10 +19,53 @@
 
 #include "paimon/common/utils/arrow/mem_utils.h"
 
+#include 
+#include 
+#include 
+
+#include "arrow/status.h"
 #include "gtest/gtest.h"
 #include "paimon/memory/memory_pool.h"
 
 namespace paimon::test {
+namespace {
+
+// A MemoryPool whose allocations always fail, either by returning nullptr or by
+// throwing std::bad_alloc, so the adaptor's out-of-memory paths can be exercised.
+class FailingMemoryPool : public MemoryPool {
+ public:
+    enum class Mode { kReturnNull, kThrowBadAlloc };
+
+    explicit FailingMemoryPool(Mode mode) : mode_(mode) {}
+
+    void* Malloc(uint64_t /*size*/, uint64_t /*alignment*/ = 0) override {
+        if (mode_ == Mode::kThrowBadAlloc) {
+            throw std::bad_alloc();
+        }
+        return nullptr;
+    }
+
+    void* Realloc(void* /*p*/, size_t /*old_size*/, size_t /*new_size*/,
+                  uint64_t /*alignment*/ = 0) override {
+        if (mode_ == Mode::kThrowBadAlloc) {
+            throw std::bad_alloc();
+        }
+        return nullptr;
+    }
+
+    void Free(void* /*p*/, uint64_t /*size*/) override {}
+    uint64_t CurrentUsage() const override {
+        return 0;
+    }
+    uint64_t MaxMemoryUsage() const override {
+        return 0;
+    }
+
+ private:
+    Mode mode_;
+};
+
+}  // namespace
 
 TEST(MemUtilsTest, TestSimple) {
     const int64_t alignment = 64;
@@ -74,4 +117,32 @@ TEST(MemUtilsTest, TestSimple) {
     ASSERT_EQ(50, pool->max_memory());
 }
 
+TEST(MemUtilsTest, TestAllocateOutOfMemory) {
+    uint8_t* ptr = nullptr;
+
+    // Underlying pool returns nullptr for a positive size.
+    auto null_pool =
+        GetArrowPool(std::make_shared(FailingMemoryPool::Mode::kReturnNull));
+    ASSERT_TRUE(null_pool->Allocate(16, 64, &ptr).IsOutOfMemory());
+
+    // Underlying pool throws std::bad_alloc.
+    auto throw_pool =
+        GetArrowPool(std::make_shared(FailingMemoryPool::Mode::kThrowBadAlloc));
+    ASSERT_TRUE(throw_pool->Allocate(16, 64, &ptr).IsOutOfMemory());
+}
+
+TEST(MemUtilsTest, TestReallocateOutOfMemory) {
+    uint8_t* ptr = nullptr;
+
+    // Underlying pool returns nullptr for a positive new size.
+    auto null_pool =
+        GetArrowPool(std::make_shared(FailingMemoryPool::Mode::kReturnNull));
+    ASSERT_TRUE(null_pool->Reallocate(/*old_size=*/0, /*new_size=*/16, 64, &ptr).IsOutOfMemory());
+
+    // Underlying pool throws std::bad_alloc.
+    auto throw_pool =
+        GetArrowPool(std::make_shared(FailingMemoryPool::Mode::kThrowBadAlloc));
+    ASSERT_TRUE(throw_pool->Reallocate(/*old_size=*/0, /*new_size=*/16, 64, &ptr).IsOutOfMemory());
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/common/utils/file_type_test.cpp b/src/paimon/common/utils/file_type_test.cpp
index b9698130..9f18a70c 100644
--- a/src/paimon/common/utils/file_type_test.cpp
+++ b/src/paimon/common/utils/file_type_test.cpp
@@ -31,6 +31,14 @@ TEST(FileTypeTest, TestIsIndex) {
     ASSERT_FALSE(FileTypeUtils::IsIndex(FileType::kData));
 }
 
+TEST(FileTypeTest, TestToString) {
+    ASSERT_EQ(FileTypeUtils::ToString(FileType::kMeta), "meta");
+    ASSERT_EQ(FileTypeUtils::ToString(FileType::kData), "data");
+    ASSERT_EQ(FileTypeUtils::ToString(FileType::kBucketIndex), "bucket_index");
+    ASSERT_EQ(FileTypeUtils::ToString(FileType::kGlobalIndex), "global_index");
+    ASSERT_EQ(FileTypeUtils::ToString(FileType::kFileIndex), "file_index");
+}
+
 TEST(FileTypeTest, TestMetaPrefix) {
     ASSERT_EQ(FileTypeUtils::Classify("dfs://cluster/db/snapshot/snapshot-1"), FileType::kMeta);
     ASSERT_EQ(FileTypeUtils::Classify("dfs://cluster/db/schema/schema-2"), FileType::kMeta);
@@ -176,6 +184,12 @@ TEST(FileTypeTest, TestInvalidTempWrapperFallsBackToOriginalName) {
 
     // Too short -> should not unwrap.
     ASSERT_EQ(FileTypeUtils::Classify("dfs://cluster/db/snapshot/.x.tmp"), FileType::kData);
+
+    // Long enough and ends with .tmp, but the char before the trailing 41-char suffix is not a
+    // dot -> should not unwrap.
+    ASSERT_EQ(FileTypeUtils::Classify(
+                  "dfs://cluster/db/snapshot/.snapshot-1234567890123456789012345678901234.tmp"),
+              FileType::kData);
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/common/utils/status_test.cpp b/src/paimon/common/utils/status_test.cpp
index 68fb5c41..cd9b60fe 100644
--- a/src/paimon/common/utils/status_test.cpp
+++ b/src/paimon/common/utils/status_test.cpp
@@ -74,6 +74,43 @@ TEST(StatusTest, TestToStringWithDetail) {
     ASSERT_EQ(status.ToString(), ss.str());
 }
 
+TEST(StatusTest, TestCodeAsString) {
+    ASSERT_EQ("OK", Status::CodeAsString(StatusCode::OK));
+    ASSERT_EQ("Out of memory", Status::CodeAsString(StatusCode::OutOfMemory));
+    ASSERT_EQ("Key error", Status::CodeAsString(StatusCode::KeyError));
+    ASSERT_EQ("Type error", Status::CodeAsString(StatusCode::TypeError));
+    ASSERT_EQ("Invalid", Status::CodeAsString(StatusCode::Invalid));
+    ASSERT_EQ("IOError", Status::CodeAsString(StatusCode::IOError));
+    ASSERT_EQ("Capacity error", Status::CodeAsString(StatusCode::CapacityError));
+    ASSERT_EQ("Index error", Status::CodeAsString(StatusCode::IndexError));
+    ASSERT_EQ("Cancelled", Status::CodeAsString(StatusCode::Cancelled));
+    ASSERT_EQ("Unknown error", Status::CodeAsString(StatusCode::UnknownError));
+    ASSERT_EQ("NotImplemented", Status::CodeAsString(StatusCode::NotImplemented));
+    ASSERT_EQ("Serialization error", Status::CodeAsString(StatusCode::SerializationError));
+    ASSERT_EQ("Not exist", Status::CodeAsString(StatusCode::NotExist));
+    ASSERT_EQ("Exist", Status::CodeAsString(StatusCode::Exist));
+
+    // An out-of-range code falls into the default branch. The cast is intentional to
+    // exercise the defensive default, so the enum-range analyzer check is suppressed.
+    // NOLINTNEXTLINE(clang-analyzer-optin.core.EnumCastOutOfRange)
+    ASSERT_EQ("Unknown", Status::CodeAsString(static_cast(99)));
+
+    // The instance overload returns "OK" for a success status.
+    ASSERT_EQ("OK", Status::OK().CodeAsString());
+}
+
+TEST(StatusDeathTest, TestAbort) {
+    // Death tests fork(); with the default "fast" style, forking in a multi-threaded
+    // process is unsafe and under ThreadSanitizer the child aborts with a sanitizer
+    // message before Abort() runs. The "threadsafe" style re-execs the test binary in
+    // a clean process so Abort()'s own output is produced and can be matched.
+    const std::string prev_style = testing::GTEST_FLAG(death_test_style);
+    testing::GTEST_FLAG(death_test_style) = "threadsafe";
+    ASSERT_DEATH(Status::IOError("boom").Abort(), "Paimon Fatal Error");
+    ASSERT_DEATH(Status::IOError("boom").Abort("custom prefix"), "custom prefix");
+    testing::GTEST_FLAG(death_test_style) = prev_style;
+}
+
 TEST(StatusTest, TestWithDetail) {
     Status status(StatusCode::IOError, "summary");
     auto detail = std::make_shared();

From c4f9f45d05d860115c24a09315a684a641914cd1 Mon Sep 17 00:00:00 2001
From: lszskye <57179283+lszskye@users.noreply.github.com>
Date: Tue, 28 Jul 2026 07:21:01 -0700
Subject: [PATCH 124/138] test: add ut for release-0.3

---
 .../union_global_index_reader.cpp             |  35 ++-
 .../union_global_index_reader_test.cpp        |  63 +++++
 .../bucket/bucket_select_converter_test.cpp   | 185 ++++++------
 .../core/bucket/hive_bucket_function_test.cpp |  13 +
 .../aggregate/field_listagg_agg_test.cpp      |   8 +
 .../merge_tree_compact_manager_test.cpp       |  80 ++++++
 .../operation/commit/commit_scanner_test.cpp  |  73 ++++-
 .../operation/manifest_file_merger_test.cpp   |  41 ++-
 .../core/operation/merge_file_split_read.cpp  |   9 +-
 .../table/system/audit_log_system_table.cpp   | 264 ++++++++++++++----
 .../table/system/audit_log_system_table.h     |  12 +-
 .../core/table/system/binlog_system_table.cpp |  22 +-
 .../core/table/system/binlog_system_table.h   |   2 +
 .../table/system/metadata_system_tables.cpp   |  84 ++++--
 .../core/table/system/system_table_test.cpp   |  92 ++++++
 test/inte/read_inte_test.cpp                  | 148 +++++++++-
 16 files changed, 919 insertions(+), 212 deletions(-)

diff --git a/src/paimon/common/global_index/union_global_index_reader.cpp b/src/paimon/common/global_index/union_global_index_reader.cpp
index a4de3b21..1b52d0a9 100644
--- a/src/paimon/common/global_index/union_global_index_reader.cpp
+++ b/src/paimon/common/global_index/union_global_index_reader.cpp
@@ -42,84 +42,84 @@ Result> UnionGlobalIndexReader::VisitIsNull()
 
 Result> UnionGlobalIndexReader::VisitEqual(
     const Literal& literal) {
-    return Union([&literal](const std::shared_ptr& reader) {
+    return Union([literal](const std::shared_ptr& reader) {
         return reader->VisitEqual(literal);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitNotEqual(
     const Literal& literal) {
-    return Union([&literal](const std::shared_ptr& reader) {
+    return Union([literal](const std::shared_ptr& reader) {
         return reader->VisitNotEqual(literal);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitLessThan(
     const Literal& literal) {
-    return Union([&literal](const std::shared_ptr& reader) {
+    return Union([literal](const std::shared_ptr& reader) {
         return reader->VisitLessThan(literal);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitLessOrEqual(
     const Literal& literal) {
-    return Union([&literal](const std::shared_ptr& reader) {
+    return Union([literal](const std::shared_ptr& reader) {
         return reader->VisitLessOrEqual(literal);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitGreaterThan(
     const Literal& literal) {
-    return Union([&literal](const std::shared_ptr& reader) {
+    return Union([literal](const std::shared_ptr& reader) {
         return reader->VisitGreaterThan(literal);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitGreaterOrEqual(
     const Literal& literal) {
-    return Union([&literal](const std::shared_ptr& reader) {
+    return Union([literal](const std::shared_ptr& reader) {
         return reader->VisitGreaterOrEqual(literal);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitIn(
     const std::vector& literals) {
-    return Union([&literals](const std::shared_ptr& reader) {
+    return Union([literals](const std::shared_ptr& reader) {
         return reader->VisitIn(literals);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitNotIn(
     const std::vector& literals) {
-    return Union([&literals](const std::shared_ptr& reader) {
+    return Union([literals](const std::shared_ptr& reader) {
         return reader->VisitNotIn(literals);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitStartsWith(
     const Literal& prefix) {
-    return Union([&prefix](const std::shared_ptr& reader) {
+    return Union([prefix](const std::shared_ptr& reader) {
         return reader->VisitStartsWith(prefix);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitEndsWith(
     const Literal& suffix) {
-    return Union([&suffix](const std::shared_ptr& reader) {
+    return Union([suffix](const std::shared_ptr& reader) {
         return reader->VisitEndsWith(suffix);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitContains(
     const Literal& literal) {
-    return Union([&literal](const std::shared_ptr& reader) {
+    return Union([literal](const std::shared_ptr& reader) {
         return reader->VisitContains(literal);
     });
 }
 
 Result> UnionGlobalIndexReader::VisitLike(
     const Literal& literal) {
-    return Union([&literal](const std::shared_ptr& reader) {
+    return Union([literal](const std::shared_ptr& reader) {
         return reader->VisitLike(literal);
     });
 }
@@ -127,7 +127,7 @@ Result> UnionGlobalIndexReader::VisitLike(
 Result> UnionGlobalIndexReader::VisitVectorSearch(
     const std::shared_ptr& vector_search) {
     auto results = ExecuteAllReaders>>(
-        [&vector_search](const std::shared_ptr& reader)
+        [vector_search](const std::shared_ptr& reader)
             -> Result> {
             return reader->VisitVectorSearch(vector_search);
         });
@@ -158,15 +158,13 @@ Result> UnionGlobalIndexReader::VisitVe
 
 Result> UnionGlobalIndexReader::VisitFullTextSearch(
     const std::shared_ptr& full_text_search) {
-    return Union([&full_text_search](const std::shared_ptr& reader) {
+    return Union([full_text_search](const std::shared_ptr& reader) {
         return reader->VisitFullTextSearch(full_text_search);
     });
 }
 
 Result> UnionGlobalIndexReader::Union(ReaderAction action) {
-    auto results = ExecuteAllReaders>>(
-        [&action](const std::shared_ptr& reader)
-            -> Result> { return action(reader); });
+    auto results = ExecuteAllReaders>>(action);
 
     std::shared_ptr merged_result = nullptr;
     for (auto& result_or_status : results) {
@@ -210,8 +208,7 @@ std::vector UnionGlobalIndexReader::ExecuteAllReaders(
     std::vector> futures;
     futures.reserve(readers_.size());
     for (const auto& reader : readers_) {
-        futures.push_back(
-            Via(executor_.get(), [&action, reader]() -> R { return action(reader); }));
+        futures.push_back(Via(executor_.get(), [action, reader]() -> R { return action(reader); }));
     }
     return CollectAll(futures);
 }
diff --git a/src/paimon/common/global_index/union_global_index_reader_test.cpp b/src/paimon/common/global_index/union_global_index_reader_test.cpp
index 8d55db42..6c3c6858 100644
--- a/src/paimon/common/global_index/union_global_index_reader_test.cpp
+++ b/src/paimon/common/global_index/union_global_index_reader_test.cpp
@@ -22,6 +22,8 @@
 #include 
 #include 
 #include 
+#include 
+#include 
 #include 
 #include 
 
@@ -57,6 +59,11 @@ class FakeReader : public GlobalIndexReader {
         error_message_ = message;
     }
 
+    void SetThrowException(const std::string& message) {
+        throw_exception_ = true;
+        exception_message_ = message;
+    }
+
     /// Sets a scored result returned by VisitVectorSearch.
     void SetScoredResult(const std::vector& row_ids, const std::vector& scores) {
         scored_row_ids_ = row_ids;
@@ -164,6 +171,9 @@ class FakeReader : public GlobalIndexReader {
  private:
     Result> MakeResult() {
         invocation_count_++;
+        if (throw_exception_) {
+            throw std::runtime_error(exception_message_);
+        }
         if (return_error_) {
             return Status::Invalid(error_message_);
         }
@@ -179,7 +189,9 @@ class FakeReader : public GlobalIndexReader {
     std::vector default_result_;
     bool return_nullptr_ = false;
     bool return_error_ = false;
+    bool throw_exception_ = false;
     std::string error_message_;
+    std::string exception_message_;
     std::vector scored_row_ids_;
     std::vector scored_scores_;
     bool has_scored_result_ = false;
@@ -187,6 +199,40 @@ class FakeReader : public GlobalIndexReader {
     std::atomic invocation_count_{0};
 };
 
+// Runs the first task immediately and defers the rest. This makes it possible to verify that
+// queued tasks own their action even if collecting an earlier future throws.
+class DeferAfterFirstExecutor : public Executor {
+ public:
+    void Add(std::function func) override {
+        if (submission_count_++ == 0) {
+            func();
+        } else {
+            pending_tasks_.push(std::move(func));
+        }
+    }
+
+    void ShutdownNow() override {
+        std::queue> empty;
+        pending_tasks_.swap(empty);
+    }
+
+    uint32_t GetThreadNum() const override {
+        return 1;
+    }
+
+    void RunPendingTasks() {
+        while (!pending_tasks_.empty()) {
+            std::function task = std::move(pending_tasks_.front());
+            pending_tasks_.pop();
+            task();
+        }
+    }
+
+ private:
+    uint32_t submission_count_ = 0;
+    std::queue> pending_tasks_;
+};
+
 class UnionGlobalIndexReaderTest : public ::testing::Test {
  public:
     static void CheckResult(const std::shared_ptr& result,
@@ -344,6 +390,23 @@ TEST_F(UnionGlobalIndexReaderTest, TestErrorPropagationWithExecutor) {
     ASSERT_NOK_WITH_MSG(union_reader.VisitIsNotNull(), "Unknown error for reader2");
 }
 
+TEST_F(UnionGlobalIndexReaderTest, TestDeferredTaskOwnsActionAfterEarlierFutureThrows) {
+    auto throwing_reader = std::make_shared();
+    auto deferred_reader = std::make_shared();
+    throwing_reader->SetThrowException("reader exception");
+    deferred_reader->SetDefaultResult({2});
+
+    std::vector> readers = {throwing_reader, deferred_reader};
+    auto executor = std::make_shared();
+    UnionGlobalIndexReader union_reader(std::move(readers), executor);
+    ASSERT_THROW(
+        { [[maybe_unused]] auto result = union_reader.VisitIsNotNull(); }, std::runtime_error);
+    ASSERT_EQ(deferred_reader->InvocationCount(), 0);
+
+    executor->RunPendingTasks();
+    ASSERT_EQ(deferred_reader->InvocationCount(), 1);
+}
+
 TEST_F(UnionGlobalIndexReaderTest, TestVisitEqualUnion) {
     auto reader1 = std::make_shared();
     auto reader2 = std::make_shared();
diff --git a/src/paimon/core/bucket/bucket_select_converter_test.cpp b/src/paimon/core/bucket/bucket_select_converter_test.cpp
index 94c2f60d..2707eccb 100644
--- a/src/paimon/core/bucket/bucket_select_converter_test.cpp
+++ b/src/paimon/core/bucket/bucket_select_converter_test.cpp
@@ -18,12 +18,14 @@
 
 #include "paimon/core/bucket/bucket_select_converter.h"
 
+#include 
 #include 
 #include 
 
 #include "arrow/api.h"
 #include "gtest/gtest.h"
 #include "paimon/core/bucket/default_bucket_function.h"
+#include "paimon/core/bucket/hive_bucket_function.h"
 #include "paimon/core/bucket/mod_bucket_function.h"
 #include "paimon/data/decimal.h"
 #include "paimon/data/timestamp.h"
@@ -36,41 +38,66 @@
 namespace paimon::test {
 
 class BucketSelectConverterTest : public ::testing::Test {
- protected:
+ public:
+    void AssertDefaultBucket(FieldType field_type, const Literal& literal,
+                             const std::shared_ptr& arrow_type,
+                             const BinaryRowGenerator::ValueType& values,
+                             int32_t num_buckets = 17) const {
+        auto predicate = PredicateBuilder::Equal(0, "key", field_type, literal);
+
+        ASSERT_OK_AND_ASSIGN(
+            std::optional selected_bucket,
+            BucketSelectConverter::Convert(predicate, {"key"}, {arrow_type},
+                                           BucketFunctionType::DEFAULT, num_buckets, pool_.get()));
+        ASSERT_TRUE(selected_bucket.has_value());
+
+        BinaryRow row = BinaryRowGenerator::GenerateRow(values, pool_.get());
+        DefaultBucketFunction function;
+        ASSERT_EQ(function.Bucket(row, num_buckets), selected_bucket.value());
+    }
+
+ private:
     std::shared_ptr pool_ = GetDefaultPool();
 };
 
-TEST_F(BucketSelectConverterTest, SingleIntEqualDefault) {
-    int32_t num_buckets = 10;
-    Literal lit(static_cast(42));
-    auto predicate = PredicateBuilder::Equal(0, "id", FieldType::INT, lit);
-
-    ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert(
-                                          predicate, {"id"}, {arrow::int32()},
-                                          BucketFunctionType::DEFAULT, num_buckets, pool_.get()));
-    ASSERT_TRUE(result.has_value());
+TEST_F(BucketSelectConverterTest, SingleStringEqualDefault) {
+    std::string value = "hello_world";
+    AssertDefaultBucket(FieldType::STRING, Literal(FieldType::STRING, value.c_str(), value.size()),
+                        arrow::utf8(), {value}, 8);
+}
 
-    // Verify by computing the expected bucket manually
-    auto row = BinaryRowGenerator::GenerateRow({static_cast(42)}, pool_.get());
-    DefaultBucketFunction func;
-    ASSERT_EQ(func.Bucket(row, num_buckets), result.value());
+TEST_F(BucketSelectConverterTest, PrimitiveKeyTypes) {
+    AssertDefaultBucket(FieldType::BOOLEAN, Literal(true), arrow::boolean(), {true});
+    AssertDefaultBucket(FieldType::TINYINT, Literal(static_cast(-12)), arrow::int8(),
+                        {static_cast(-12)});
+    AssertDefaultBucket(FieldType::SMALLINT, Literal(static_cast(1234)), arrow::int16(),
+                        {static_cast(1234)});
+    AssertDefaultBucket(FieldType::INT, Literal(static_cast(42)), arrow::int32(),
+                        {static_cast(42)}, 10);
+    AssertDefaultBucket(FieldType::BIGINT, Literal(static_cast(123456789L)),
+                        arrow::int64(), {static_cast(123456789L)}, 16);
+    AssertDefaultBucket(FieldType::FLOAT, Literal(1.25F), arrow::float32(), {1.25F});
+    AssertDefaultBucket(FieldType::DOUBLE, Literal(-123.5), arrow::float64(), {-123.5});
 }
 
-TEST_F(BucketSelectConverterTest, SingleStringEqualDefault) {
-    int32_t num_buckets = 8;
-    std::string val = "hello_world";
-    Literal lit(FieldType::STRING, val.c_str(), val.size());
-    auto predicate = PredicateBuilder::Equal(0, "name", FieldType::STRING, lit);
+TEST_F(BucketSelectConverterTest, TimestampMillisPrecision) {
+    // TIMESTAMP with millisecond precision (compact storage, precision=3)
+    Timestamp ts = Timestamp::FromEpochMillis(1700000000000L);
+    AssertDefaultBucket(FieldType::TIMESTAMP, Literal(ts), arrow::timestamp(arrow::TimeUnit::MILLI),
+                        {TimestampType(ts, 3)}, 10);
+}
 
-    ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert(
-                                          predicate, {"name"}, {arrow::utf8()},
-                                          BucketFunctionType::DEFAULT, num_buckets, pool_.get()));
-    ASSERT_TRUE(result.has_value());
+TEST_F(BucketSelectConverterTest, TimestampMicrosPrecision) {
+    // TIMESTAMP with microsecond precision (non-compact storage, precision=6)
+    Timestamp ts(1700000000000L, 123456);
+    AssertDefaultBucket(FieldType::TIMESTAMP, Literal(ts), arrow::timestamp(arrow::TimeUnit::MICRO),
+                        {TimestampType(ts, 6)}, 10);
+}
 
-    // Verify
-    auto row = BinaryRowGenerator::GenerateRow({val}, pool_.get());
-    DefaultBucketFunction func;
-    ASSERT_EQ(func.Bucket(row, num_buckets), result.value());
+TEST_F(BucketSelectConverterTest, DecimalKey) {
+    Decimal decimal = Decimal::FromUnscaledLong(12345L, 10, 2);
+    AssertDefaultBucket(FieldType::DECIMAL, Literal(decimal), arrow::decimal128(10, 2), {decimal},
+                        10);
 }
 
 TEST_F(BucketSelectConverterTest, MultiKeyAndPredicate) {
@@ -175,22 +202,6 @@ TEST_F(BucketSelectConverterTest, NullPredicateReturnsNullopt) {
     ASSERT_FALSE(result.has_value());
 }
 
-TEST_F(BucketSelectConverterTest, BigintKeyDefault) {
-    int32_t num_buckets = 16;
-    Literal lit(static_cast(123456789L));
-    auto predicate = PredicateBuilder::Equal(0, "user_id", FieldType::BIGINT, lit);
-
-    ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert(
-                                          predicate, {"user_id"}, {arrow::int64()},
-                                          BucketFunctionType::DEFAULT, num_buckets, pool_.get()));
-    ASSERT_TRUE(result.has_value());
-
-    // Verify
-    auto row = BinaryRowGenerator::GenerateRow({static_cast(123456789L)}, pool_.get());
-    DefaultBucketFunction func;
-    ASSERT_EQ(func.Bucket(row, num_buckets), result.value());
-}
-
 TEST_F(BucketSelectConverterTest, AndWithExtraPredicateStillWorks) {
     // AND(EQUAL(id, 42), GREATER_THAN(value, 100))
     // Only id is bucket key, value is not — should still derive bucket from id
@@ -211,60 +222,52 @@ TEST_F(BucketSelectConverterTest, AndWithExtraPredicateStillWorks) {
     ASSERT_EQ(func.Bucket(row, num_buckets), result.value());
 }
 
-TEST_F(BucketSelectConverterTest, TimestampMillisPrecision) {
-    // TIMESTAMP with millisecond precision (compact storage, precision=3)
-    int32_t num_buckets = 10;
-    Timestamp ts = Timestamp::FromEpochMillis(1700000000000L);
-    Literal lit(ts);
-    auto predicate = PredicateBuilder::Equal(0, "ts", FieldType::TIMESTAMP, lit);
+TEST_F(BucketSelectConverterTest, HiveBucketFunctionWithDecimal) {
+    int32_t num_buckets = 11;
+    Decimal decimal = Decimal::FromUnscaledLong(12345L, 10, 2);
+    auto int_predicate =
+        PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(static_cast(7)));
+    auto decimal_predicate =
+        PredicateBuilder::Equal(1, "amount", FieldType::DECIMAL, Literal(decimal));
+    ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({int_predicate, decimal_predicate}));
 
-    auto arrow_type = arrow::timestamp(arrow::TimeUnit::MILLI);
-    ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert(
-                                          predicate, {"ts"}, {arrow_type},
-                                          BucketFunctionType::DEFAULT, num_buckets, pool_.get()));
-    ASSERT_TRUE(result.has_value());
-
-    // Verify: precision=3 uses compact WriteTimestamp
-    auto row = BinaryRowGenerator::GenerateRow({TimestampType(ts, 3)}, pool_.get());
-    DefaultBucketFunction func;
-    ASSERT_EQ(func.Bucket(row, num_buckets), result.value());
+    ASSERT_OK_AND_ASSIGN(
+        std::optional selected_bucket,
+        BucketSelectConverter::Convert(predicate, {"id", "amount"},
+                                       {arrow::int32(), arrow::decimal128(10, 2)},
+                                       BucketFunctionType::HIVE, num_buckets, pool_.get()));
+    ASSERT_TRUE(selected_bucket.has_value());
+
+    BinaryRow row =
+        BinaryRowGenerator::GenerateRow({static_cast(7), decimal}, pool_.get());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr function,
+                         HiveBucketFunction::Create({HiveFieldInfo(FieldType::INT),
+                                                     HiveFieldInfo(FieldType::DECIMAL, 10, 2)}));
+    ASSERT_EQ(function->Bucket(row, num_buckets), selected_bucket.value());
 }
 
-TEST_F(BucketSelectConverterTest, TimestampMicrosPrecision) {
-    // TIMESTAMP with microsecond precision (non-compact storage, precision=6)
-    int32_t num_buckets = 10;
-    Timestamp ts(1700000000000L, 123456);
-    Literal lit(ts);
-    auto predicate = PredicateBuilder::Equal(0, "ts", FieldType::TIMESTAMP, lit);
-
-    auto arrow_type = arrow::timestamp(arrow::TimeUnit::MICRO);
-    ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert(
-                                          predicate, {"ts"}, {arrow_type},
-                                          BucketFunctionType::DEFAULT, num_buckets, pool_.get()));
-    ASSERT_TRUE(result.has_value());
+TEST_F(BucketSelectConverterTest, UnsupportedFieldTypeReturnsError) {
+    auto predicate =
+        PredicateBuilder::Equal(0, "items", FieldType::ARRAY, Literal(static_cast(42)));
 
-    // Verify: precision=6 uses non-compact WriteTimestamp (different layout than precision=3)
-    auto row = BinaryRowGenerator::GenerateRow({TimestampType(ts, 6)}, pool_.get());
-    DefaultBucketFunction func;
-    ASSERT_EQ(func.Bucket(row, num_buckets), result.value());
+    Result> result =
+        BucketSelectConverter::Convert(predicate, {"items"}, {arrow::list(arrow::int32())},
+                                       BucketFunctionType::DEFAULT, 5, pool_.get());
+    ASSERT_NOK_WITH_MSG(result.status(), "unsupported field type");
 }
 
-TEST_F(BucketSelectConverterTest, DecimalKey) {
-    int32_t num_buckets = 10;
-    Decimal dec = Decimal::FromUnscaledLong(12345L, 10, 2);
-    Literal lit(dec);
-    auto predicate = PredicateBuilder::Equal(0, "amount", FieldType::DECIMAL, lit);
-
-    auto arrow_type = arrow::decimal128(10, 2);
-    ASSERT_OK_AND_ASSIGN(auto result, BucketSelectConverter::Convert(
-                                          predicate, {"amount"}, {arrow_type},
-                                          BucketFunctionType::DEFAULT, num_buckets, pool_.get()));
-    ASSERT_TRUE(result.has_value());
-
-    // Verify
-    auto row = BinaryRowGenerator::GenerateRow({dec}, pool_.get());
-    DefaultBucketFunction func;
-    ASSERT_EQ(func.Bucket(row, num_buckets), result.value());
+TEST_F(BucketSelectConverterTest, ModBucketFunctionWithMultipleKeysReturnsError) {
+    auto id_predicate =
+        PredicateBuilder::Equal(0, "id", FieldType::INT, Literal(static_cast(42)));
+    auto region_predicate =
+        PredicateBuilder::Equal(1, "region", FieldType::INT, Literal(static_cast(1)));
+    ASSERT_OK_AND_ASSIGN(auto predicate, PredicateBuilder::And({id_predicate, region_predicate}));
+
+    Result> result = BucketSelectConverter::Convert(
+        predicate, {"id", "region"}, {arrow::int32(), arrow::int32()}, BucketFunctionType::MOD, 5,
+        pool_.get());
+    ASSERT_NOK_WITH_MSG(result.status(),
+                        "MOD bucket function requires exactly one bucket key field");
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/bucket/hive_bucket_function_test.cpp b/src/paimon/core/bucket/hive_bucket_function_test.cpp
index c2b7971b..21f2a984 100644
--- a/src/paimon/core/bucket/hive_bucket_function_test.cpp
+++ b/src/paimon/core/bucket/hive_bucket_function_test.cpp
@@ -107,6 +107,11 @@ class HiveBucketFunctionTest : public ::testing::Test {
         return BinaryRowGenerator::GenerateRow({value}, pool.get());
     }
 
+    BinaryRow CreateShortRow(int16_t value) {
+        auto pool = GetDefaultPool();
+        return BinaryRowGenerator::GenerateRow({value}, pool.get());
+    }
+
     float FloatFromBits(uint32_t bits) {
         float value;
         std::memcpy(&value, &bits, sizeof(value));
@@ -258,6 +263,14 @@ TEST_F(HiveBucketFunctionTest, TestTinyintNegativeValuesCompatibleWithJava) {
     ASSERT_EQ(520, func->Bucket(CreateByteRow(std::numeric_limits::min()), 1000));
 }
 
+TEST_F(HiveBucketFunctionTest, TestSmallintField) {
+    std::vector field_types = {FieldType::SMALLINT};
+    ASSERT_OK_AND_ASSIGN(auto func, HiveBucketFunction::Create(field_types));
+
+    ASSERT_EQ(234, func->Bucket(CreateShortRow(static_cast(1234)), 1000));
+    ASSERT_EQ(647, func->Bucket(CreateShortRow(static_cast(-1)), 1000));
+}
+
 /// Test STRING field
 TEST_F(HiveBucketFunctionTest, TestStringField) {
     std::vector field_types = {FieldType::STRING};
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp
index beb1aeb9..0901bcad 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp
+++ b/src/paimon/core/mergetree/compact/aggregate/field_listagg_agg_test.cpp
@@ -120,6 +120,14 @@ TEST_F(FieldListaggAggTest, TestDistinctNoDuplicates) {
     ASSERT_EQ(DataDefine::GetVariantValue(ret), "a b c d");
 }
 
+TEST_F(FieldListaggAggTest, TestDistinctWithEmptyDelimiterFallsBackToWhitespace) {
+    ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg("", true));
+
+    // Empty delimiter falls back to whitespace, so the repeated "b" is removed.
+    auto ret = agg->Agg(std::string_view("a b"), std::string_view("b c"));
+    ASSERT_EQ(DataDefine::GetVariantValue(ret), "a b c");
+}
+
 TEST_F(FieldListaggAggTest, TestDistinctEmptyInput) {
     ASSERT_OK_AND_ASSIGN(auto agg, MakeAgg(";", true));
 
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp
index f17daf44..19b8bc36 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_manager_test.cpp
@@ -21,6 +21,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -30,12 +31,17 @@
 #include "arrow/api.h"
 #include "gtest/gtest.h"
 #include "paimon/common/utils/fields_comparator.h"
+#include "paimon/core/compact/compact_deletion_file.h"
+#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/manifest/file_source.h"
 #include "paimon/core/mergetree/level_sorted_run.h"
 #include "paimon/core/mergetree/levels.h"
 #include "paimon/core/mergetree/sorted_run.h"
 #include "paimon/core/stats/simple_stats.h"
+#include "paimon/fs/file_system_factory.h"
+#include "paimon/testing/mock/mock_index_path_factory.h"
 #include "paimon/testing/utils/binary_row_generator.h"
 #include "paimon/testing/utils/testharness.h"
 
@@ -411,6 +417,80 @@ TEST_F(MergeTreeCompactManagerTest, TestTriggerFullCompaction) {
     }
 }
 
+TEST_F(MergeTreeCompactManagerTest, TestFullCompactionRewritesEachMaxLevelFile) {
+    std::vector> files = {
+        ToFile(LevelMinMax(2, 1, 3), /*max_sequence=*/3),
+        ToFile(LevelMinMax(2, 4, 6), /*max_sequence=*/6)};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr levels, CreateLevels(files));
+
+    auto manager = std::make_shared(
+        levels, std::make_shared(TestStrategy()), comparator_,
+        /*compaction_file_size=*/2,
+        /*num_sorted_run_stop_trigger=*/std::numeric_limits::max(),
+        std::make_shared(/*expected_drop_delete=*/true),
+        /*metrics_reporter=*/nullptr,
+        /*dv_maintainer=*/nullptr,
+        /*lazy_gen_deletion_file=*/false,
+        /*need_lookup=*/false,
+        /*force_rewrite_all_files=*/true,
+        /*force_keep_delete=*/false, std::make_shared(),
+        std::make_shared());
+
+    ASSERT_OK(manager->TriggerCompaction(/*full_compaction=*/true));
+    ASSERT_OK_AND_ASSIGN(std::optional> compact_result,
+                         manager->GetCompactionResult(/*blocking=*/true));
+    ASSERT_TRUE(compact_result.has_value());
+    ASSERT_EQ(compact_result.value()->Before(), files);
+    ASSERT_EQ(compact_result.value()->After().size(), 2);
+    for (const auto& file : compact_result.value()->After()) {
+        ASSERT_EQ(file->file_name.rfind("rewrite-", /*pos=*/0), 0);
+    }
+}
+
+TEST_F(MergeTreeCompactManagerTest, TestCompactionGeneratesDeletionFileEagerly) {
+    std::vector> files = {
+        ToFile(LevelMinMax(0, 1, 3), /*max_sequence=*/0),
+        ToFile(LevelMinMax(1, 1, 5), /*max_sequence=*/1)};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr levels, CreateLevels(files));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr fs,
+                         FileSystemFactory::Get("local", dir->Str(), {}));
+    auto path_factory = std::make_shared(dir->Str());
+    auto dv_index_file =
+        std::make_shared(fs, path_factory, /*bitmap64=*/false, pool_);
+    auto dv_maintainer = std::make_shared(
+        dv_index_file, std::map>{});
+    ASSERT_OK(dv_maintainer->NotifyNewDeletion("remaining-file", /*position=*/0));
+
+    auto manager = std::make_shared(
+        levels, std::make_shared(TestStrategy()), comparator_,
+        /*compaction_file_size=*/2,
+        /*num_sorted_run_stop_trigger=*/std::numeric_limits::max(),
+        std::make_shared(/*expected_drop_delete=*/true),
+        /*metrics_reporter=*/nullptr, dv_maintainer,
+        /*lazy_gen_deletion_file=*/false,
+        /*need_lookup=*/false,
+        /*force_rewrite_all_files=*/false,
+        /*force_keep_delete=*/false, std::make_shared(),
+        std::make_shared());
+
+    ASSERT_OK(manager->TriggerCompaction(/*full_compaction=*/false));
+    ASSERT_OK_AND_ASSIGN(std::optional> compact_result,
+                         manager->GetCompactionResult(/*blocking=*/true));
+    ASSERT_TRUE(compact_result.has_value());
+    std::shared_ptr deletion_file = compact_result.value()->DeletionFile();
+    ASSERT_NE(deletion_file, nullptr);
+    ASSERT_NE(std::dynamic_pointer_cast(deletion_file), nullptr);
+
+    ASSERT_OK_AND_ASSIGN(std::optional> index_file,
+                         deletion_file->GetOrCompute());
+    ASSERT_TRUE(index_file.has_value());
+    ASSERT_EQ(index_file.value()->IndexType(), DeletionVectorsIndexFile::DELETION_VECTORS_INDEX);
+    ASSERT_OK_AND_ASSIGN(bool exists, dv_index_file->Exists(index_file.value()));
+    ASSERT_TRUE(exists);
+}
+
 TEST_F(MergeTreeCompactManagerTest, TestRejectReentrantFullCompaction) {
     std::vector inputs = {LevelMinMax(0, 1, 3), LevelMinMax(1, 2, 5),
                                        LevelMinMax(1, 6, 7)};
diff --git a/src/paimon/core/operation/commit/commit_scanner_test.cpp b/src/paimon/core/operation/commit/commit_scanner_test.cpp
index 869da737..3d5c6270 100644
--- a/src/paimon/core/operation/commit/commit_scanner_test.cpp
+++ b/src/paimon/core/operation/commit/commit_scanner_test.cpp
@@ -31,9 +31,15 @@
 #include "paimon/common/data/binary_row_writer.h"
 #include "paimon/common/utils/binary_row_partition_computer.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/index/index_file_meta.h"
+#include "paimon/core/manifest/file_kind.h"
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/manifest/index_manifest_file.h"
 #include "paimon/core/manifest/manifest_entry.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/snapshot.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/format/file_format_factory.h"
 #include "paimon/scan_context.h"
 #include "paimon/testing/utils/testharness.h"
 
@@ -41,7 +47,7 @@ namespace paimon::test {
 
 namespace {
 
-Snapshot MakeSnapshot() {
+Snapshot MakeSnapshot(std::optional index_manifest = std::nullopt) {
     return Snapshot(
         /*id=*/1,
         /*schema_id=*/0,
@@ -51,7 +57,7 @@ Snapshot MakeSnapshot() {
         /*delta_manifest_list_size=*/std::nullopt,
         /*changelog_manifest_list=*/std::nullopt,
         /*changelog_manifest_list_size=*/std::nullopt,
-        /*index_manifest=*/std::nullopt,
+        /*index_manifest=*/std::move(index_manifest),
         /*commit_user=*/"test-user",
         /*commit_identifier=*/1, Snapshot::CommitKind::Append(),
         /*time_millis=*/0,
@@ -72,11 +78,22 @@ BinaryRow CreateIntPartition(int32_t value) {
     return row;
 }
 
+IndexManifestEntry CreateIndexEntry(const std::string& file_name, int32_t partition_value) {
+    auto index_file = std::make_shared(
+        /*index_type=*/"HASH", file_name, /*file_size=*/10, /*row_count=*/1,
+        /*dv_ranges=*/std::nullopt,
+        /*external_path=*/std::nullopt);
+    return IndexManifestEntry(FileKind::Add(), CreateIntPartition(partition_value),
+                              /*bucket=*/0, index_file);
+}
+
 }  // namespace
 
 class CommitScannerTest : public testing::Test {
  protected:
     void SetUp() override {
+        dir_ = UniqueTestDirectory::Create();
+        ASSERT_TRUE(dir_);
         schema_ = arrow::schema({arrow::field("pt", arrow::int32())});
         ASSERT_OK_AND_ASSIGN(core_options_, CoreOptions::FromMap({}));
         ASSERT_OK_AND_ASSIGN(partition_computer_,
@@ -86,19 +103,41 @@ class CommitScannerTest : public testing::Test {
                                  /*legacy_partition_name_enabled=*/true, GetDefaultPool()));
     }
 
-    CommitScanner CreateScanner(CommitScanner::ScanSupplier scan_supplier) const {
+    CommitScanner CreateScanner(
+        CommitScanner::ScanSupplier scan_supplier,
+        const std::shared_ptr& index_manifest_file = nullptr) const {
         return CommitScanner(
             /*snapshot_manager=*/nullptr,
             /*schema_manager=*/nullptr,
             /*manifest_list=*/nullptr,
             /*manifest_file=*/nullptr,
-            /*index_manifest_file=*/nullptr,
+            /*index_manifest_file=*/index_manifest_file,
             /*table_schema=*/nullptr, schema_, core_options_,
             /*executor=*/nullptr, GetDefaultPool(), partition_computer_.get(),
             std::move(scan_supplier));
     }
 
+    Result> CreateIndexManifestFile() const {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_format,
+                               FileFormatFactory::Get("orc", {}));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::shared_ptr path_factory,
+            FileStorePathFactory::Create(
+                dir_->Str(), schema_, /*partition_keys=*/{"pt"},
+                /*default_part_value=*/"__DEFAULT_PARTITION__", file_format->Identifier(),
+                /*data_file_prefix=*/"data-",
+                /*legacy_partition_name_enabled=*/true, /*external_paths=*/{},
+                /*global_index_external_path=*/std::nullopt,
+                /*index_file_in_data_file_dir=*/false, GetDefaultPool()));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr index_manifest_file,
+            IndexManifestFile::Create(dir_->GetFileSystem(), file_format, "zstd", path_factory,
+                                      /*bucket_mode=*/2, GetDefaultPool(), core_options_));
+        return std::shared_ptr(std::move(index_manifest_file));
+    }
+
  protected:
+    std::unique_ptr dir_;
     std::shared_ptr schema_;
     CoreOptions core_options_;
     std::unique_ptr partition_computer_;
@@ -150,4 +189,30 @@ TEST_F(CommitScannerTest, TestReadAllEntriesFromChangedPartitionsBuildsScanFilte
     ASSERT_EQ("42", captured_partition_filters[0]["pt"]);
 }
 
+TEST_F(CommitScannerTest, TestReadAllIndexEntriesFromPartitions) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr index_manifest_file,
+                         CreateIndexManifestFile());
+    std::vector entries = {CreateIndexEntry("index-1", /*partition_value=*/1),
+                                               CreateIndexEntry("index-2", /*partition_value=*/2),
+                                               CreateIndexEntry("index-3", /*partition_value=*/3)};
+    ASSERT_OK_AND_ASSIGN(
+        std::optional index_manifest,
+        index_manifest_file->WriteIndexFiles(/*previous_index_manifest=*/std::nullopt, entries));
+    ASSERT_TRUE(index_manifest);
+
+    CommitScanner scanner = CreateScanner(CommitScanner::ScanSupplier{}, index_manifest_file);
+    Snapshot snapshot = MakeSnapshot(index_manifest);
+
+    ASSERT_OK_AND_ASSIGN(std::vector unfiltered,
+                         scanner.ReadAllIndexEntriesFromPartitions(snapshot, /*partitions=*/{}));
+    ASSERT_EQ(3u, unfiltered.size());
+
+    std::vector> partitions = {
+        {{"pt", "2"}}, {{"unknown_partition_key", "value"}}};
+    ASSERT_OK_AND_ASSIGN(std::vector filtered,
+                         scanner.ReadAllIndexEntriesFromPartitions(snapshot, partitions));
+    ASSERT_EQ(1u, filtered.size());
+    ASSERT_EQ("index-2", filtered[0].index_file->FileName());
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/manifest_file_merger_test.cpp b/src/paimon/core/operation/manifest_file_merger_test.cpp
index 150723f6..1ff1ffd7 100644
--- a/src/paimon/core/operation/manifest_file_merger_test.cpp
+++ b/src/paimon/core/operation/manifest_file_merger_test.cpp
@@ -23,6 +23,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -111,9 +112,22 @@ class ManifestFileMergerTest : public testing::Test {
         return manifest_file_metas[0];
     }
 
+    std::set ListManifestFiles() const {
+        std::vector> file_statuses;
+        EXPECT_OK(file_system_->ListFileStatus(
+            FileStorePathFactory::ManifestPath(path_factory_->RootPath()), &file_statuses));
+        std::set files;
+        for (const auto& status : file_statuses) {
+            if (!status->IsDir()) {
+                files.insert(status->GetPath());
+            }
+        }
+        return files;
+    }
+
  private:
     void CreateManifestFile(const std::string& path_str) {
-        auto file_system = std::make_shared();
+        file_system_ = std::make_shared();
         ASSERT_OK_AND_ASSIGN(
             std::shared_ptr file_format,
             FileFormatFactory::Get("parquet", std::map()));
@@ -131,10 +145,11 @@ class ManifestFileMergerTest : public testing::Test {
                 options.GetFileFormat()->Identifier(), options.DataFilePrefix(),
                 options.LegacyPartitionNameEnabled(), external_paths, global_index_external_path,
                 options.IndexFileInDataFileDir(), pool_));
+        path_factory_ = path_factory;
         ASSERT_OK_AND_ASSIGN(std::shared_ptr partition_schema,
                              FieldMapping::GetPartitionSchema(schema, {"f0"}));
         ASSERT_OK_AND_ASSIGN(manifest_file_,
-                             ManifestFile::Create(file_system, file_format, "zstd", path_factory,
+                             ManifestFile::Create(file_system_, file_format, "zstd", path_factory,
                                                   /*target_file_size=*/1024 * 1024, pool_, options,
                                                   partition_schema));
     }
@@ -168,6 +183,8 @@ class ManifestFileMergerTest : public testing::Test {
     std::string test_root_;
     std::shared_ptr pool_;
     std::shared_ptr manifest_file_;
+    std::shared_ptr file_system_;
+    std::shared_ptr path_factory_;
     std::shared_ptr partition_type_;
 };
 
@@ -372,4 +389,24 @@ TEST_F(ManifestFileMergerTest, TestTriggerFullCompaction) {
     ContainSameEntryFile(merged.value(), entry_file_expected);
 }
 
+TEST_F(ManifestFileMergerTest, TestDeleteNewManifestFilesWhenMinorCompactionFails) {
+    std::vector input;
+    for (int32_t i = 0; i < 4; i++) {
+        input.push_back(MakeManifest({MakeEntry(FileKind::Add(), std::to_string(i))}));
+    }
+
+    // The first pair is merged successfully. Removing a source file from the second pair makes
+    // the following merge fail after a new manifest file has already been created.
+    std::string missing_manifest_path = path_factory_->ToManifestFilePath(input[2].FileName());
+    ASSERT_OK(file_system_->Delete(missing_manifest_path));
+    std::set files_before_merge = ListManifestFiles();
+
+    ASSERT_NOK_WITH_MSG(ManifestFileMerger::Merge(
+                            input, /*manifest_target_file_size=*/5000, /*merge_min_count=*/2,
+                            /*full_compaction_file_size=*/MAX_LONG_VALUE, manifest_file_.get()),
+                        "not exists");
+
+    ASSERT_EQ(files_before_merge, ListManifestFiles());
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 09c2e0d5..34567fce 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -265,10 +265,11 @@ Result> MergeFileSplitRead::CreateNoMergeReader(
         pool_);
 
     // create read schema without extra fields (e.g., completed key, sequence fields)
-    auto row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind());
-
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr read_schema,
-                                      raw_read_schema_->AddField(0, row_kind_field));
+    std::shared_ptr read_schema = raw_read_schema_;
+    if (read_schema->GetFieldIndex(SpecialFields::ValueKind().Name()) < 0) {
+        auto row_kind_field = DataField::ConvertDataFieldToArrowField(SpecialFields::ValueKind());
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(read_schema, read_schema->AddField(0, row_kind_field));
+    }
     PAIMON_ASSIGN_OR_RAISE(
         std::vector> raw_file_readers,
         CreateRawFileReaders(data_split->Partition(), data_split->DataFiles(), read_schema,
diff --git a/src/paimon/core/table/system/audit_log_system_table.cpp b/src/paimon/core/table/system/audit_log_system_table.cpp
index 1f89d5b7..e77821fe 100644
--- a/src/paimon/core/table/system/audit_log_system_table.cpp
+++ b/src/paimon/core/table/system/audit_log_system_table.cpp
@@ -34,6 +34,7 @@
 #include "arrow/c/bridge.h"
 #include "arrow/util/checked_cast.h"
 #include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/concat_batch_reader.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/types/row_kind.h"
@@ -41,6 +42,7 @@
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/core/table/source/key_value_table_read.h"
 #include "paimon/defs.h"
 #include "paimon/read_context.h"
@@ -55,7 +57,9 @@ namespace {
 class AuditLogBatchConverter : public ChangelogBatchConverter {
  public:
     Result> ConvertDataColumn(
-        const std::shared_ptr& array, arrow::MemoryPool* /*pool*/) const override {
+        const std::shared_ptr& array,
+        const std::vector& /*row_group_lengths*/,
+        arrow::MemoryPool* /*pool*/) const override {
         return array;
     }
 };
@@ -65,64 +69,76 @@ class ChangelogBatchReader : public BatchReader {
     ChangelogBatchReader(std::unique_ptr reader,
                          std::shared_ptr output_schema, bool include_sequence_number,
                          std::shared_ptr converter,
-                         const std::shared_ptr& pool)
+                         bool pack_update_before_after, const std::shared_ptr& pool)
         : reader_(std::move(reader)),
           output_schema_(std::move(output_schema)),
           include_sequence_number_(include_sequence_number),
           converter_(std::move(converter)),
+          pack_update_before_after_(pack_update_before_after),
           arrow_pool_holder_(GetArrowPool(pool)),
           arrow_pool_(arrow_pool_holder_.get()) {}
 
     Result NextBatch() override {
-        PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch());
-        if (BatchReader::IsEofBatch(batch)) {
-            return batch;
-        }
-        auto& [c_array, c_schema] = batch;
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_array,
-                                          arrow::ImportArray(c_array.get(), c_schema.get()));
-        std::shared_ptr struct_array =
-            std::dynamic_pointer_cast(arrow_array);
-        if (!struct_array) {
-            return Status::Invalid("audit_log system table expects struct batches");
-        }
+        while (true) {
+            PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, reader_->NextBatch());
+            std::shared_ptr struct_array;
+            std::vector row_group_lengths;
+            if (BatchReader::IsEofBatch(batch)) {
+                if (!pending_update_before_) {
+                    return batch;
+                }
+                struct_array = std::move(pending_update_before_);
+                row_group_lengths.push_back(1);
+            } else {
+                auto& [c_array, c_schema] = batch;
+                PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                    std::shared_ptr arrow_array,
+                    arrow::ImportArray(c_array.get(), c_schema.get()));
+                struct_array = std::dynamic_pointer_cast(arrow_array);
+                if (!struct_array) {
+                    return Status::Invalid("audit_log system table expects struct batches");
+                }
+                PAIMON_ASSIGN_OR_RAISE(struct_array, PrependPendingUpdateBefore(struct_array));
+                PAIMON_ASSIGN_OR_RAISE(row_group_lengths, BuildRowGroupLengths(struct_array));
+            }
+            if (row_group_lengths.empty()) {
+                continue;
+            }
 
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr rowkind_array,
-                               BuildRowKindArray(struct_array));
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_kind_array,
+                                   BuildRowKindArray(struct_array, row_group_lengths));
 
-        arrow::ArrayVector output_arrays = {rowkind_array};
-        if (include_sequence_number_) {
-            std::shared_ptr sequence_array =
-                struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name());
-            if (!sequence_array) {
-                return Status::Invalid("cannot find _SEQUENCE_NUMBER in audit_log batch");
+            arrow::ArrayVector output_arrays = {row_kind_array};
+            if (include_sequence_number_) {
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr sequence_array,
+                                       BuildSequenceNumberArray(struct_array, row_group_lengths));
+                output_arrays.push_back(sequence_array);
             }
-            PAIMON_ASSIGN_OR_RAISE(sequence_array, CopyToStablePool(sequence_array));
-            output_arrays.push_back(sequence_array);
-        }
 
-        for (const auto& field : output_schema_->fields()) {
-            if (field->name() == SpecialFields::RowKind().Name() ||
-                field->name() == SpecialFields::SequenceNumber().Name()) {
-                continue;
-            }
-            std::shared_ptr array = struct_array->GetFieldByName(field->name());
-            if (!array) {
-                return Status::Invalid("cannot find ", field->name(), " in changelog batch");
+            for (const auto& field : output_schema_->fields()) {
+                if (field->name() == SpecialFields::RowKind().Name() ||
+                    field->name() == SpecialFields::SequenceNumber().Name()) {
+                    continue;
+                }
+                std::shared_ptr array = struct_array->GetFieldByName(field->name());
+                if (!array) {
+                    return Status::Invalid("cannot find ", field->name(), " in changelog batch");
+                }
+                PAIMON_ASSIGN_OR_RAISE(
+                    array, converter_->ConvertDataColumn(array, row_group_lengths, arrow_pool_));
+                PAIMON_ASSIGN_OR_RAISE(array, CopyToStablePool(array));
+                output_arrays.push_back(array);
             }
-            PAIMON_ASSIGN_OR_RAISE(array, converter_->ConvertDataColumn(array, arrow_pool_));
-            PAIMON_ASSIGN_OR_RAISE(array, CopyToStablePool(array));
-            output_arrays.push_back(array);
-        }
 
-        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
-            std::shared_ptr output_array,
-            arrow::StructArray::Make(output_arrays, output_schema_->field_names()));
-        auto output_c_array = std::make_unique();
-        auto output_c_schema = std::make_unique();
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(
-            arrow::ExportArray(*output_array, output_c_array.get(), output_c_schema.get()));
-        return std::make_pair(std::move(output_c_array), std::move(output_c_schema));
+            PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                std::shared_ptr output_array,
+                arrow::StructArray::Make(output_arrays, output_schema_->field_names()));
+            auto output_c_array = std::make_unique();
+            auto output_c_schema = std::make_unique();
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(
+                arrow::ExportArray(*output_array, output_c_array.get(), output_c_schema.get()));
+            return std::make_pair(std::move(output_c_array), std::move(output_c_schema));
+        }
     }
 
     std::shared_ptr GetReaderMetrics() const override {
@@ -130,10 +146,68 @@ class ChangelogBatchReader : public BatchReader {
     }
 
     void Close() override {
+        pending_update_before_.reset();
         reader_->Close();
     }
 
  private:
+    Result> PrependPendingUpdateBefore(
+        const std::shared_ptr& struct_array) {
+        if (!pending_update_before_) {
+            return struct_array;
+        }
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr combined,
+            arrow::Concatenate({pending_update_before_, struct_array}, arrow_pool_));
+        pending_update_before_.reset();
+        std::shared_ptr result =
+            std::dynamic_pointer_cast(combined);
+        if (!result) {
+            return Status::Invalid("failed to concatenate binlog struct batches");
+        }
+        return result;
+    }
+
+    Result> BuildRowGroupLengths(
+        const std::shared_ptr& struct_array) {
+        std::shared_ptr value_kind_array =
+            std::dynamic_pointer_cast(
+                struct_array->GetFieldByName(SpecialFields::ValueKind().Name()));
+        if (!value_kind_array) {
+            return Status::Invalid("cannot find _VALUE_KIND in audit_log batch");
+        }
+
+        std::vector row_group_lengths;
+        row_group_lengths.reserve(struct_array->length());
+        for (int64_t i = 0; i < value_kind_array->length();) {
+            bool is_update_before =
+                !value_kind_array->IsNull(i) &&
+                value_kind_array->Value(i) == RowKind::UpdateBefore()->ToByteValue();
+            if (!pack_update_before_after_ || !is_update_before) {
+                row_group_lengths.push_back(1);
+                ++i;
+                continue;
+            }
+            if (i + 1 == value_kind_array->length()) {
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr pending,
+                                       CopyToStablePool(struct_array->Slice(i, /*length=*/1)));
+                pending_update_before_ = std::dynamic_pointer_cast(pending);
+                if (!pending_update_before_) {
+                    return Status::Invalid("failed to cache UPDATE_BEFORE in binlog reader");
+                }
+                break;
+            }
+            if (value_kind_array->IsNull(i + 1) ||
+                value_kind_array->Value(i + 1) != RowKind::UpdateAfter()->ToByteValue()) {
+                return Status::Invalid(
+                    "UPDATE_BEFORE is not followed by UPDATE_AFTER in binlog reader");
+            }
+            row_group_lengths.push_back(2);
+            i += 2;
+        }
+        return row_group_lengths;
+    }
+
     Result> CopyToStablePool(
         const std::shared_ptr& array) const {
         /// The imported data batch may release its C Arrow buffers after this wrapper returns.
@@ -144,7 +218,8 @@ class ChangelogBatchReader : public BatchReader {
     }
 
     Result> BuildRowKindArray(
-        const std::shared_ptr& struct_array) const {
+        const std::shared_ptr& struct_array,
+        const std::vector& row_group_lengths) const {
         std::shared_ptr value_kind_array =
             std::dynamic_pointer_cast(
                 struct_array->GetFieldByName(SpecialFields::ValueKind().Name()));
@@ -152,15 +227,46 @@ class ChangelogBatchReader : public BatchReader {
             return Status::Invalid("cannot find _VALUE_KIND in audit_log batch");
         }
         arrow::StringBuilder builder(arrow_pool_);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(value_kind_array->length()));
-        for (int64_t i = 0; i < value_kind_array->length(); ++i) {
-            if (value_kind_array->IsNull(i)) {
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(row_group_lengths.size()));
+        int64_t offset = 0;
+        for (int32_t row_group_length : row_group_lengths) {
+            int64_t row_kind_index = offset + row_group_length - 1;
+            if (value_kind_array->IsNull(row_kind_index)) {
+                return Status::Invalid(
+                    fmt::format("exists null value in value kind array in pos {}", row_kind_index));
                 PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull());
-                continue;
+            } else {
+                PAIMON_ASSIGN_OR_RAISE(
+                    const RowKind* row_kind,
+                    RowKind::FromByteValue(value_kind_array->Value(row_kind_index)));
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(row_kind->ShortString()));
+            }
+            offset += row_group_length;
+        }
+        std::shared_ptr result;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&result));
+        return result;
+    }
+
+    Result> BuildSequenceNumberArray(
+        const std::shared_ptr& struct_array,
+        const std::vector& row_group_lengths) const {
+        std::shared_ptr sequence_array =
+            std::dynamic_pointer_cast(
+                struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name()));
+        if (!sequence_array) {
+            return Status::Invalid("cannot find _SEQUENCE_NUMBER in audit_log batch");
+        }
+        arrow::Int64Builder builder(arrow_pool_);
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Reserve(row_group_lengths.size()));
+        int64_t offset = 0;
+        for (int32_t row_group_length : row_group_lengths) {
+            if (sequence_array->IsNull(offset)) {
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull());
+            } else {
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(sequence_array->Value(offset)));
             }
-            PAIMON_ASSIGN_OR_RAISE(const RowKind* row_kind,
-                                   RowKind::FromByteValue(value_kind_array->Value(i)));
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(row_kind->ShortString()));
+            offset += row_group_length;
         }
         std::shared_ptr result;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&result));
@@ -171,8 +277,10 @@ class ChangelogBatchReader : public BatchReader {
     std::shared_ptr output_schema_;
     bool include_sequence_number_;
     std::shared_ptr converter_;
+    bool pack_update_before_after_;
     std::unique_ptr arrow_pool_holder_;
     arrow::MemoryPool* arrow_pool_;
+    std::shared_ptr pending_update_before_;
 };
 
 class ChangelogTableRead : public TableRead {
@@ -189,17 +297,42 @@ class ChangelogTableRead : public TableRead {
 
     Result> CreateReader(
         const std::vector>& splits) override {
+        // Records across different splits should not be packed, because for streaming reads on a
+        // primary-key table, all data belonging to the same partition and bucket is placed in a
+        // single split. Therefore, an UPDATE_BEFORE/UPDATE_AFTER pair will not be truncated at a
+        // split boundary.
+        if (converter_->PackUpdateBeforeAfter()) {
+            std::vector> readers;
+            readers.reserve(splits.size());
+            for (const auto& split : splits) {
+                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, CreateReader(split));
+                readers.push_back(std::move(reader));
+            }
+            return std::make_unique(std::move(readers), GetMemoryPool());
+        }
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader,
                                data_read_->CreateReader(splits));
-        return std::make_unique(std::move(reader), output_schema_,
-                                                      include_sequence_number_, converter_,
-                                                      GetMemoryPool());
+        return CreateChangelogBatchReader(std::move(reader), output_schema_,
+                                          include_sequence_number_, converter_,
+                                          /*pack_update_before_after=*/false, GetMemoryPool());
     }
 
     Result> CreateReader(
         const std::shared_ptr& split) override {
-        std::vector> splits = {split};
-        return CreateReader(splits);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader,
+                               data_read_->CreateReader(split));
+        bool pack_update_before_after = false;
+        if (converter_->PackUpdateBeforeAfter()) {
+            std::shared_ptr data_split =
+                std::dynamic_pointer_cast(split);
+            if (!data_split) {
+                return Status::Invalid("binlog system table expects data split");
+            }
+            pack_update_before_after = data_split->IsStreaming();
+        }
+        return CreateChangelogBatchReader(std::move(reader), output_schema_,
+                                          include_sequence_number_, converter_,
+                                          pack_update_before_after, GetMemoryPool());
     }
 
  private:
@@ -211,6 +344,15 @@ class ChangelogTableRead : public TableRead {
 
 }  // namespace
 
+std::unique_ptr CreateChangelogBatchReader(
+    std::unique_ptr reader, std::shared_ptr output_schema,
+    bool include_sequence_number, std::shared_ptr converter,
+    bool pack_update_before_after, const std::shared_ptr& pool) {
+    return std::make_unique(std::move(reader), std::move(output_schema),
+                                                  include_sequence_number, std::move(converter),
+                                                  pack_update_before_after, pool);
+}
+
 AuditLogSystemTable::AuditLogSystemTable(std::shared_ptr fs, std::string table_path,
                                          std::shared_ptr table_schema,
                                          std::map options)
@@ -224,10 +366,10 @@ std::string AuditLogSystemTable::Name() const {
 }
 
 Result> AuditLogSystemTable::ArrowSchema() const {
-    std::shared_ptr rowkind_field =
+    std::shared_ptr row_kind_field =
         DataField::ConvertDataFieldToArrowField(SpecialFields::RowKind());
-    rowkind_field = rowkind_field->WithNullable(false);
-    arrow::FieldVector fields = {rowkind_field};
+    row_kind_field = row_kind_field->WithNullable(false);
+    arrow::FieldVector fields = {row_kind_field};
     PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options_));
     if (core_options.TableReadSequenceNumberEnabled()) {
         fields.push_back(DataField::ConvertDataFieldToArrowField(SpecialFields::SequenceNumber()));
diff --git a/src/paimon/core/table/system/audit_log_system_table.h b/src/paimon/core/table/system/audit_log_system_table.h
index 8592ea7b..e21e287c 100644
--- a/src/paimon/core/table/system/audit_log_system_table.h
+++ b/src/paimon/core/table/system/audit_log_system_table.h
@@ -36,9 +36,19 @@ class ChangelogBatchConverter {
     virtual ~ChangelogBatchConverter() = default;
 
     virtual Result> ConvertDataColumn(
-        const std::shared_ptr& array, arrow::MemoryPool* pool) const = 0;
+        const std::shared_ptr& array, const std::vector& row_group_lengths,
+        arrow::MemoryPool* pool) const = 0;
+
+    virtual bool PackUpdateBeforeAfter() const {
+        return false;
+    }
 };
 
+std::unique_ptr CreateChangelogBatchReader(
+    std::unique_ptr reader, std::shared_ptr output_schema,
+    bool include_sequence_number, std::shared_ptr converter,
+    bool pack_update_before_after, const std::shared_ptr& pool);
+
 /// System table for `T$audit_log`, exposing row-level changelog records with rowkind.
 class AuditLogSystemTable : public SystemTable {
  public:
diff --git a/src/paimon/core/table/system/binlog_system_table.cpp b/src/paimon/core/table/system/binlog_system_table.cpp
index 57f05e2b..88541194 100644
--- a/src/paimon/core/table/system/binlog_system_table.cpp
+++ b/src/paimon/core/table/system/binlog_system_table.cpp
@@ -41,11 +41,15 @@ namespace {
 class BinlogBatchConverter : public ChangelogBatchConverter {
  public:
     Result> ConvertDataColumn(
-        const std::shared_ptr& array, arrow::MemoryPool* pool) const override {
+        const std::shared_ptr& array, const std::vector& row_group_lengths,
+        arrow::MemoryPool* pool) const override {
         arrow::Int32Builder offsets_builder(pool);
-        PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Reserve(array->length() + 1));
-        for (int64_t i = 0; i <= array->length(); ++i) {
-            PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(static_cast(i)));
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Reserve(row_group_lengths.size() + 1));
+        int32_t offset = 0;
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(offset));
+        for (int32_t row_group_length : row_group_lengths) {
+            offset += row_group_length;
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Append(offset));
         }
         std::shared_ptr offsets_array;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(offsets_builder.Finish(&offsets_array));
@@ -54,10 +58,18 @@ class BinlogBatchConverter : public ChangelogBatchConverter {
             arrow::ListArray::FromArrays(*offsets_array, *array, pool));
         return list_array;
     }
+
+    bool PackUpdateBeforeAfter() const override {
+        return true;
+    }
 };
 
 }  // namespace
 
+std::shared_ptr CreateBinlogBatchConverter() {
+    return std::make_shared();
+}
+
 BinlogSystemTable::BinlogSystemTable(std::shared_ptr fs, std::string table_path,
                                      std::shared_ptr table_schema,
                                      std::map options)
@@ -87,7 +99,7 @@ Result> BinlogSystemTable::ArrowSchema() const {
 
 Result> BinlogSystemTable::NewRead(
     const std::shared_ptr& context) const {
-    return NewChangelogRead(context, std::make_shared());
+    return NewChangelogRead(context, CreateBinlogBatchConverter());
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/table/system/binlog_system_table.h b/src/paimon/core/table/system/binlog_system_table.h
index 0685fcd8..3b4f45bd 100644
--- a/src/paimon/core/table/system/binlog_system_table.h
+++ b/src/paimon/core/table/system/binlog_system_table.h
@@ -29,6 +29,8 @@ namespace paimon {
 class FileSystem;
 class TableSchema;
 
+std::shared_ptr CreateBinlogBatchConverter();
+
 /// System table for `T$binlog`, exposing changelog records with list-wrapped data columns.
 class BinlogSystemTable : public AuditLogSystemTable {
  public:
diff --git a/src/paimon/core/table/system/metadata_system_tables.cpp b/src/paimon/core/table/system/metadata_system_tables.cpp
index 1563a3f7..4467b7b9 100644
--- a/src/paimon/core/table/system/metadata_system_tables.cpp
+++ b/src/paimon/core/table/system/metadata_system_tables.cpp
@@ -44,7 +44,6 @@
 #include "paimon/common/utils/date_time_utils.h"
 #include "paimon/common/utils/field_type_utils.h"
 #include "paimon/common/utils/internal_row_utils.h"
-#include "paimon/common/utils/object_utils.h"
 #include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/rapidjson_util.h"
 #include "paimon/core/casting/cast_executor_factory.h"
@@ -136,7 +135,7 @@ Result> OptionalLocalDateTimePartsToTimestampMillis(
     return std::optional(timestamp_millis);
 }
 
-std::optional OptionalDoubleToString(const std::optional& value) {
+std::optional OptionalDoubleToString(const std::optional& value) {
     if (!value) {
         return std::optional();
     }
@@ -351,6 +350,45 @@ Result> RowValueStrings(const std::vector& f
     return values;
 }
 
+struct StatsStringOverrides {
+    std::map values;
+    std::map null_counts;
+};
+
+Result OmittedPartitionStats(const std::shared_ptr& table_schema,
+                                                   const DataFileMeta& file,
+                                                   const BinaryRow& partition, int64_t row_count) {
+    StatsStringOverrides overrides;
+    if (!file.write_cols) {
+        return overrides;
+    }
+
+    std::vector partition_fields;
+    partition_fields.reserve(table_schema->PartitionKeys().size());
+    for (const auto& partition_key : table_schema->PartitionKeys()) {
+        PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(partition_key));
+        partition_fields.push_back(std::move(field));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector partition_values,
+                           RowValueStrings(partition_fields, partition));
+    if (partition_fields.size() != partition_values.size()) {
+        return Status::Invalid(
+            fmt::format("partition field count {} does not match partition value count {}",
+                        partition_fields.size(), partition_values.size()));
+    }
+    for (size_t i = 0; i < partition_fields.size(); ++i) {
+        const std::string& field_name = partition_fields[i].Name();
+        if (std::find(file.write_cols->begin(), file.write_cols->end(), field_name) !=
+            file.write_cols->end()) {
+            continue;
+        }
+        overrides.values.emplace(field_name, std::move(partition_values[i]));
+        overrides.null_counts.emplace(field_name,
+                                      partition.IsNullAt(i) ? std::to_string(row_count) : "0");
+    }
+    return overrides;
+}
+
 Result RowValuesString(const std::vector& fields, const InternalRow& row,
                                     std::string_view left, std::string_view right) {
     PAIMON_ASSIGN_OR_RAISE(std::vector values, RowValueStrings(fields, row));
@@ -369,13 +407,16 @@ Result> OptionalRowValuesString(const std::vector FieldsValueMapString(const std::vector& fields,
-                                         const InternalRow& row) {
+                                         const InternalRow& row,
+                                         const std::map& overrides) {
     PAIMON_ASSIGN_OR_RAISE(std::vector values, RowValueStrings(fields, row));
     std::vector> field_values;
     size_t length = std::min(fields.size(), values.size());
     field_values.reserve(length);
     for (size_t i = 0; i < length; ++i) {
-        field_values.emplace_back(fields[i].Name(), std::move(values[i]));
+        auto iter = overrides.find(fields[i].Name());
+        field_values.emplace_back(fields[i].Name(),
+                                  iter == overrides.end() ? std::move(values[i]) : iter->second);
     }
     std::sort(field_values.begin(), field_values.end(),
               [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; });
@@ -389,13 +430,17 @@ Result FieldsValueMapString(const std::vector& fields,
 }
 
 Result NullValueCountsString(const std::vector& fields,
-                                          const InternalArray& null_counts) {
+                                          const InternalArray& null_counts,
+                                          const std::map& overrides) {
     std::vector> field_values;
     int32_t length = std::min(static_cast(fields.size()), null_counts.Size());
     field_values.reserve(length);
     for (int32_t i = 0; i < length; ++i) {
+        auto iter = overrides.find(fields[i].Name());
         std::string value =
-            null_counts.IsNullAt(i) ? "null" : std::to_string(null_counts.GetLong(i));
+            iter == overrides.end()
+                ? (null_counts.IsNullAt(i) ? "null" : std::to_string(null_counts.GetLong(i)))
+                : iter->second;
         field_values.emplace_back(fields[i].Name(), std::move(value));
     }
     std::sort(field_values.begin(), field_values.end(),
@@ -425,7 +470,7 @@ Result> ProjectWriteFields(const std::shared_ptr fields;
-    fields.reserve(file.write_cols->size() + data_schema->PartitionKeys().size());
+    fields.reserve(file.write_cols->size());
     for (const auto& write_col : file.write_cols.value()) {
         if (SpecialFields::IsSystemField(write_col)) {
             continue;
@@ -433,15 +478,6 @@ Result> ProjectWriteFields(const std::shared_ptrGetField(write_col));
         fields.push_back(std::move(field));
     }
-
-    // Partial writes may omit partition columns from write_cols. Keep them in the stats source
-    // fields so SimpleStatsEvolution can map partition stats consistently.
-    for (const auto& partition_key : data_schema->PartitionKeys()) {
-        if (!ObjectUtils::Contains(file.write_cols.value(), partition_key)) {
-            PAIMON_ASSIGN_OR_RAISE(DataField field, data_schema->GetField(partition_key));
-            fields.push_back(std::move(field));
-        }
-    }
     return fields;
 }
 
@@ -876,6 +912,9 @@ Result> FilesSystemTable::BuildRows() const {
         PAIMON_ASSIGN_OR_RAISE(
             SimpleStatsEvolution::EvolutionStats stats,
             stats_evolution->Evolution(file->value_stats, file->row_count, file->value_stats_cols));
+        PAIMON_ASSIGN_OR_RAISE(StatsStringOverrides stats_overrides,
+                               OmittedPartitionStats(context_.table_schema, *file,
+                                                     entry.Partition(), file->row_count));
 
         GenericRow row(schema->num_fields());
         if (context_.table_schema->PartitionKeys().empty()) {
@@ -901,13 +940,16 @@ Result> FilesSystemTable::BuildRows() const {
         row.SetField(8, OptionalStringValue(min_key));
         row.SetField(9, OptionalStringValue(max_key));
         PAIMON_ASSIGN_OR_RAISE(std::string null_value_counts,
-                               NullValueCountsString(value_stats_fields, *stats.null_counts));
+                               NullValueCountsString(value_stats_fields, *stats.null_counts,
+                                                     stats_overrides.null_counts));
         row.SetField(10, StringValue(null_value_counts));
-        PAIMON_ASSIGN_OR_RAISE(std::string min_value_stats,
-                               FieldsValueMapString(value_stats_fields, *stats.min_values));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::string min_value_stats,
+            FieldsValueMapString(value_stats_fields, *stats.min_values, stats_overrides.values));
         row.SetField(11, StringValue(min_value_stats));
-        PAIMON_ASSIGN_OR_RAISE(std::string max_value_stats,
-                               FieldsValueMapString(value_stats_fields, *stats.max_values));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::string max_value_stats,
+            FieldsValueMapString(value_stats_fields, *stats.max_values, stats_overrides.values));
         row.SetField(12, StringValue(max_value_stats));
         row.SetField(13, file->min_sequence_number);
         row.SetField(14, file->max_sequence_number);
diff --git a/src/paimon/core/table/system/system_table_test.cpp b/src/paimon/core/table/system/system_table_test.cpp
index 3e9233ba..0c67ae3a 100644
--- a/src/paimon/core/table/system/system_table_test.cpp
+++ b/src/paimon/core/table/system/system_table_test.cpp
@@ -23,9 +23,11 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include "arrow/api.h"
+#include "arrow/ipc/json_simple.h"
 #include "gtest/gtest.h"
 #include "paimon/core/schema/table_schema.h"
 #include "paimon/core/table/system/audit_log_system_table.h"
@@ -34,8 +36,12 @@
 #include "paimon/defs.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/file_system_factory.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/reader/batch_reader.h"
 #include "paimon/result.h"
 #include "paimon/status.h"
+#include "paimon/testing/mock/mock_file_batch_reader.h"
+#include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -71,6 +77,92 @@ TEST(SystemTableTest, TestChangelogArrowSchemaReturnsInvalidOptions) {
                         "Invalid Config [table-read.sequence-number.enabled: invalid]");
 }
 
+TEST(SystemTableTest, TestBinlogArrowSchemaWithSequenceNumber) {
+    std::map options = {
+        {Options::TABLE_READ_SEQUENCE_NUMBER_ENABLED, "true"}};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                         CreateTableSchemaForTest(options));
+
+    BinlogSystemTable binlog(/*fs=*/nullptr, "/tmp/table", table_schema, options);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, binlog.ArrowSchema());
+
+    ASSERT_EQ(schema->field_names(),
+              (std::vector{"rowkind", "_SEQUENCE_NUMBER", "pk", "v"}));
+    ASSERT_EQ(schema->field(0)->type()->id(), arrow::Type::STRING);
+    ASSERT_FALSE(schema->field(0)->nullable());
+    ASSERT_EQ(schema->field(1)->type()->id(), arrow::Type::INT64);
+    ASSERT_EQ(schema->field(2)->type()->id(), arrow::Type::LIST);
+    ASSERT_EQ(schema->field(3)->type()->id(), arrow::Type::LIST);
+}
+
+TEST(SystemTableTest, TestStreamingBinlogPacksUpdateAcrossBatches) {
+    std::shared_ptr input_type = arrow::struct_({
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        arrow::field("_SEQUENCE_NUMBER", arrow::int64()),
+        arrow::field("pk", arrow::utf8()),
+        arrow::field("v", arrow::int32()),
+    });
+    std::shared_ptr input =
+        arrow::ipc::internal::json::ArrayFromJSON(
+            input_type, R"([[0, 10, "a", 1], [1, 11, "b", 2], [2, 12, "b", 3], [3, 13, "d", 4]])")
+            .ValueOrDie();
+    std::shared_ptr output_schema = arrow::schema({
+        arrow::field("rowkind", arrow::utf8(), /*nullable=*/false),
+        arrow::field("_SEQUENCE_NUMBER", arrow::int64()),
+        arrow::field("pk", arrow::list(arrow::utf8())),
+        arrow::field("v", arrow::list(arrow::int32())),
+    });
+    std::unique_ptr reader = CreateChangelogBatchReader(
+        std::make_unique(input, input_type, /*read_batch_size=*/2),
+        output_schema,
+        /*include_sequence_number=*/true, CreateBinlogBatchConverter(),
+        /*pack_update_before_after=*/true, GetDefaultPool());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr actual,
+                         ReadResultCollector::CollectResult(reader.get()));
+    std::shared_ptr expected_array =
+        arrow::ipc::internal::json::ArrayFromJSON(actual->type(), R"([
+                ["+I", 10, ["a"], [1]],
+                ["+U", 11, ["b", "b"], [2, 3]],
+                ["-D", 13, ["d"], [4]]
+            ])")
+            .ValueOrDie();
+    auto expected = std::make_shared(expected_array);
+    ASSERT_TRUE(actual->Equals(*expected))
+        << "expected: " << expected->ToString() << "\nactual: " << actual->ToString();
+}
+
+TEST(SystemTableTest, TestStreamingBinlogEmitsUnmatchedUpdateBefore) {
+    std::shared_ptr input_type = arrow::struct_({
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        arrow::field("pk", arrow::utf8()),
+        arrow::field("v", arrow::int32()),
+    });
+    std::shared_ptr input =
+        arrow::ipc::internal::json::ArrayFromJSON(input_type, R"([[1, "b", 2]])").ValueOrDie();
+    std::shared_ptr output_schema = arrow::schema({
+        arrow::field("rowkind", arrow::utf8(), /*nullable=*/false),
+        arrow::field("pk", arrow::list(arrow::utf8())),
+        arrow::field("v", arrow::list(arrow::int32())),
+    });
+    std::unique_ptr reader = CreateChangelogBatchReader(
+        std::make_unique(input, input_type, /*read_batch_size=*/1),
+        output_schema,
+        /*include_sequence_number=*/false, CreateBinlogBatchConverter(),
+        /*pack_update_before_after=*/true, GetDefaultPool());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr actual,
+                         ReadResultCollector::CollectResult(reader.get()));
+    std::shared_ptr expected_array =
+        arrow::ipc::internal::json::ArrayFromJSON(actual->type(), R"([
+                ["-U", ["b"], [2]]
+            ])")
+            .ValueOrDie();
+    auto expected = std::make_shared(expected_array);
+    ASSERT_TRUE(actual->Equals(*expected))
+        << "expected: " << expected->ToString() << "\nactual: " << actual->ToString();
+}
+
 TEST(SystemTableTest, TestReadOptimizedSystemTableRegistration) {
     ASSERT_TRUE(SystemTableLoader::IsSupported(ReadOptimizedSystemTable::kName));
 
diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp
index 9b9f8bde..9ebf2d60 100644
--- a/test/inte/read_inte_test.cpp
+++ b/test/inte/read_inte_test.cpp
@@ -38,6 +38,7 @@
 #include "gtest/gtest.h"
 #include "paimon/catalog/catalog.h"
 #include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/factories/io_hook.h"
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
@@ -60,6 +61,8 @@
 #include "paimon/data/decimal.h"
 #include "paimon/data/timestamp.h"
 #include "paimon/defs.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
 #include "paimon/fs/file_system.h"
 #include "paimon/fs/local/local_file_system.h"
 #include "paimon/memory/memory_pool.h"
@@ -80,6 +83,7 @@
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/test_helper.h"
 #include "paimon/testing/utils/testharness.h"
+#include "paimon/write_context.h"
 
 namespace paimon::test {
 
@@ -275,10 +279,12 @@ std::vector StructFieldNames(const std::shared_ptrtype()->fields())->field_names();
 }
 
-Result ReadSystemTable(
-    const std::string& system_table_path, const std::map& options,
-    bool streaming_mode = false, const std::shared_ptr& predicate = nullptr,
-    const std::vector& read_field_names = {}) {
+Result ReadSystemTable(const std::string& system_table_path,
+                                              const std::map& options,
+                                              bool streaming_mode = false,
+                                              const std::shared_ptr& predicate = nullptr,
+                                              const std::vector& read_field_names = {},
+                                              bool read_next_plan = false) {
     ScanContextBuilder scan_context_builder(system_table_path);
     scan_context_builder.SetOptions(options).WithStreamingMode(streaming_mode);
     if (predicate) {
@@ -289,6 +295,9 @@ Result ReadSystemTable(
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan,
                            TableScan::Create(std::move(scan_context)));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan());
+    if (read_next_plan) {
+        PAIMON_ASSIGN_OR_RAISE(plan, table_scan->CreatePlan());
+    }
 
     ReadContextBuilder read_context_builder(system_table_path);
     read_context_builder.SetOptions(options);
@@ -1209,6 +1218,76 @@ TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedTable) {
     ASSERT_EQ(max_value_stats_array->GetString(0), "{dt=20260527, pk=a, v=1}");
 }
 
+TEST(SystemTableReadInteTest, TestReadFilesSystemTableForPartitionedPartialWrite) {
+    arrow::FieldVector fields = {
+        arrow::field("dt", arrow::utf8()),
+        arrow::field("id", arrow::int32()),
+        arrow::field("score", arrow::int32()),
+    };
+    auto schema = arrow::schema(fields);
+    std::map options = {
+        {Options::FILE_SYSTEM, "local"},         {Options::FILE_FORMAT, "parquet"},
+        {Options::MANIFEST_FORMAT, "avro"},      {Options::BUCKET, "-1"},
+        {Options::ROW_TRACKING_ENABLED, "true"}, {Options::DATA_EVOLUTION_ENABLED, "true"},
+    };
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema,
+                                                         /*partition_keys=*/{"dt"},
+                                                         /*primary_keys=*/{}, options,
+                                                         /*is_streaming_mode=*/true));
+    std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+
+    WriteContextBuilder write_context_builder(table_path, "partial-write");
+    write_context_builder.WithWriteSchema({"score"});
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context,
+                         write_context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr write,
+                         FileStoreWrite::Create(std::move(write_context)));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr batch,
+        TestHelper::MakeRecordBatch(arrow::struct_({fields[2]}), R"([[10], [20]])",
+                                    /*partition_map=*/{{"dt", "20260724"}}, /*bucket=*/0, {}));
+    ASSERT_OK(write->Write(std::move(batch)));
+    ASSERT_OK_AND_ASSIGN(std::vector> commit_messages,
+                         write->PrepareCommit());
+    ASSERT_OK(write->Close());
+
+    CommitContextBuilder commit_context_builder(table_path, "partial-write");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit_context,
+                         commit_context_builder.Finish());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr commit,
+                         FileStoreCommit::Create(std::move(commit_context)));
+    ASSERT_OK(commit->Commit(commit_messages));
+
+    ASSERT_OK_AND_ASSIGN(auto files_result, ReadSystemTable(table_path + "$files", options));
+    auto files_array = SingleStructChunk(files_result);
+    ASSERT_EQ(files_array->length(), 1);
+    auto partition_array = std::dynamic_pointer_cast(files_array->field(0));
+    auto null_value_counts_array =
+        std::dynamic_pointer_cast(files_array->field(10));
+    auto min_value_stats_array =
+        std::dynamic_pointer_cast(files_array->field(11));
+    auto max_value_stats_array =
+        std::dynamic_pointer_cast(files_array->field(12));
+    auto write_cols_array = std::dynamic_pointer_cast(files_array->field(19));
+    ASSERT_TRUE(partition_array);
+    ASSERT_TRUE(null_value_counts_array);
+    ASSERT_TRUE(min_value_stats_array);
+    ASSERT_TRUE(max_value_stats_array);
+    ASSERT_TRUE(write_cols_array);
+
+    ASSERT_EQ(partition_array->GetString(0), "{20260724}");
+    ASSERT_EQ(null_value_counts_array->GetString(0), "{dt=0, id=2, score=0}");
+    ASSERT_EQ(min_value_stats_array->GetString(0), "{dt=20260724, id=null, score=10}");
+    ASSERT_EQ(max_value_stats_array->GetString(0), "{dt=20260724, id=null, score=20}");
+    auto write_cols_values =
+        std::dynamic_pointer_cast(write_cols_array->values());
+    ASSERT_TRUE(write_cols_values);
+    ASSERT_EQ(write_cols_values->length(), 1);
+    ASSERT_EQ(write_cols_values->GetString(0), "score");
+}
+
 TEST(SystemTableReadInteTest, TestReadFilesSystemTableForDatePartition) {
     arrow::FieldVector fields = {
         arrow::field("dt", arrow::date32()),
@@ -1612,6 +1691,67 @@ TEST(SystemTableReadInteTest, TestReadAuditLogAndBinlogSystemTableWithChangelogR
     ])");
 }
 
+TEST(SystemTableReadInteTest, TestStreamingBinlogPacksUpdateBeforeAndAfter) {
+    arrow::FieldVector fields = {
+        arrow::field("pk", arrow::utf8()),
+        arrow::field("v", arrow::int32()),
+    };
+    auto schema = arrow::schema(fields);
+    std::map options = {
+        {Options::FILE_SYSTEM, "local"},    {Options::FILE_FORMAT, "parquet"},
+        {Options::MANIFEST_FORMAT, "avro"}, {Options::BUCKET, "1"},
+        {Options::WRITE_BUFFER_SIZE, "1"},  {Options::WRITE_BUFFER_SPILLABLE, "false"},
+    };
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema,
+                                                         /*partition_keys=*/{},
+                                                         /*primary_keys=*/{"pk"}, options,
+                                                         /*is_streaming_mode=*/true));
+
+    std::vector row_kinds_1 = {
+        RecordBatch::RowKind::INSERT,
+        RecordBatch::RowKind::UPDATE_BEFORE,
+    };
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr batch_1,
+        TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["a", 1], ["b", 2]])",
+                                    /*partition_map=*/{}, /*bucket=*/0, row_kinds_1));
+    std::vector row_kinds_2 = {
+        RecordBatch::RowKind::UPDATE_AFTER,
+        RecordBatch::RowKind::DELETE,
+    };
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr batch_2,
+        TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([["b", 3], ["c", 4]])",
+                                    /*partition_map=*/{}, /*bucket=*/0, row_kinds_2));
+    std::vector> batches;
+    batches.push_back(std::move(batch_1));
+    batches.push_back(std::move(batch_2));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    std::map streaming_options = options;
+    streaming_options[Options::SCAN_MODE] = "from-snapshot";
+    streaming_options[Options::SCAN_SNAPSHOT_ID] = "1";
+    ASSERT_OK_AND_ASSIGN(
+        auto result,
+        ReadSystemTable(PathUtil::JoinPath(dir->Str(), "foo.db/bar$binlog"), streaming_options,
+                        /*streaming_mode=*/true, /*predicate=*/nullptr,
+                        /*read_field_names=*/{}, /*read_next_plan=*/true));
+    ASSERT_TRUE(result.array);
+    ASSERT_EQ(result.array->num_chunks(), 2);
+    auto array = std::dynamic_pointer_cast(
+        arrow::Concatenate(result.array->chunks()).ValueOrDie());
+    ASSERT_TRUE(array);
+    ASSERT_EQ(StructFieldNames(array), (std::vector{"rowkind", "pk", "v"}));
+    AssertStructArrayEqualsJson(array, R"([
+        ["+I", ["a"], [1]],
+        ["+U", ["b", "b"], [2, 3]],
+        ["-D", ["c"], [4]]
+    ])");
+}
+
 TEST(SystemTableReadInteTest, TestReadBinlogSystemTableWithNullValue) {
     arrow::FieldVector fields = {
         arrow::field("pk", arrow::utf8(), /*nullable=*/false),

From f15596bde88d30f277a7dc6875aee7345378c397 Mon Sep 17 00:00:00 2001
From: Zhang Jiawei <30893610+zjw1111@users.noreply.github.com>
Date: Wed, 29 Jul 2026 09:58:30 +0800
Subject: [PATCH 125/138] chore: remove Git LFS and refresh documentation

---
 .devcontainer/centos7/Dockerfile    |  4 +---
 .devcontainer/centos7/run.sh        |  2 --
 LICENSE                             | 23 +++++++++++++++++++++++
 NOTICE                              |  5 +++++
 docs/source/user_guide/prefetch.rst | 10 +++++-----
 docs/source/user_guide/write.rst    | 28 +++++++++++++++-------------
 include/paimon/commit_message.h     |  4 ++--
 include/paimon/file_store_commit.h  |  8 ++++----
 8 files changed, 55 insertions(+), 29 deletions(-)

diff --git a/.devcontainer/centos7/Dockerfile b/.devcontainer/centos7/Dockerfile
index 01940d02..9a25b5e8 100644
--- a/.devcontainer/centos7/Dockerfile
+++ b/.devcontainer/centos7/Dockerfile
@@ -35,7 +35,6 @@
 #   scl enable devtoolset-11 rh-python38 -- bash        # activate modern gcc + python
 #   source /opt/paimon-env.sh                           # PATH for rust, cmake
 #   cd /workspaces/paimon-cpp
-#   git lfs install --local && git lfs pull             # critical: boost & friends are LFS
 #
 # Run ./.devcontainer/centos7/run.sh smoke from the host for the full check.
 
@@ -93,7 +92,7 @@ enabled=0\n' > /etc/yum.repos.d/CentOS-Base.repo \
     && yum makecache
 
 # ---------- Base toolchain ----------
-# EPEL provides git-lfs, ninja-build, a newer python3 than the base 3.6.
+# EPEL provides ninja-build and a newer python3 than the base 3.6.
 # SCL (Software Collections) provides devtoolset-11 (gcc 11) and rh-python38
 # without overriding the system gcc/python. CentOS 7's default gcc 4.8 is
 # too old for C++17/20 used by lucene++ and our tantivy wrapper.
@@ -131,7 +130,6 @@ enabled=1\n' > /etc/yum.repos.d/CentOS-SCLo-scl.repo \
         rh-python38 \
         rh-python38-python-pip \
         git \
-        git-lfs \
         ninja-build \
         make \
         patch \
diff --git a/.devcontainer/centos7/run.sh b/.devcontainer/centos7/run.sh
index d9ae5d21..2f494f27 100755
--- a/.devcontainer/centos7/run.sh
+++ b/.devcontainer/centos7/run.sh
@@ -99,8 +99,6 @@ case "${cmd}" in
             "${CONTAINER}" bash -lc '
             set -eux
             cd /workspaces/paimon-cpp
-            git lfs install --local
-            git lfs pull
             cmake -S . -B build-centos7 \
                 -G Ninja \
                 -DCMAKE_BUILD_TYPE=Release \
diff --git a/LICENSE b/LICENSE
index fc64292e..6e39411a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -219,6 +219,7 @@ This product includes code from Apache Iceberg C++.
 * Dev Container utilities:
   * .devcontainer/Dockerfile.template
   * .devcontainer/devcontainer.json.template
+  * .devcontainer/x86_64/devcontainer.json.template
 * CI utilities:
   * .pre-commit-config.yaml
 * Avro direct decoder/encoder:
@@ -234,6 +235,28 @@ License: https://www.apache.org/licenses/LICENSE-2.0
 
 --------------------------------------------------------------------------------
 
+This product includes code based on Apache Spark.
+
+* Variant utilities:
+  * src/paimon/common/data/variant/generic_variant.cpp
+  * src/paimon/common/data/variant/generic_variant.h
+  * src/paimon/common/data/variant/variant_binary_util.cpp
+  * src/paimon/common/data/variant/variant_binary_util.h
+  * src/paimon/common/data/variant/variant_builder.cpp
+  * src/paimon/common/data/variant/variant_builder.h
+  * src/paimon/common/data/variant/variant_reassembler.cpp
+  * src/paimon/common/data/variant/variant_reassembler.h
+  * src/paimon/common/data/variant/variant_schema.h
+  * src/paimon/common/data/variant/variant_shredding_utils.cpp
+  * src/paimon/common/data/variant/variant_shredding_writer.cpp
+  * src/paimon/common/data/variant/variant_shredding_writer.h
+
+Copyright: 2014 and onwards The Apache Software Foundation.
+Home page: https://spark.apache.org/
+License: https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
 This product includes code based on Google Guava.
 
 * Preconditions utility in src/paimon/common/utils/preconditions.h
diff --git a/NOTICE b/NOTICE
index a0284350..a12e432f 100644
--- a/NOTICE
+++ b/NOTICE
@@ -24,6 +24,11 @@ Copyright 2024-2025 The Apache Software Foundation
 
 --------------------------------------------------------------------------------
 
+Apache Spark
+Copyright 2014 and onwards The Apache Software Foundation.
+
+--------------------------------------------------------------------------------
+
 Apache ORC
 Copyright 2013 and onwards The Apache Software Foundation.
 
diff --git a/docs/source/user_guide/prefetch.rst b/docs/source/user_guide/prefetch.rst
index 9f62da20..4e601219 100644
--- a/docs/source/user_guide/prefetch.rst
+++ b/docs/source/user_guide/prefetch.rst
@@ -23,13 +23,13 @@ Prefetch
    :align: center
    :width: 100%
 
-In C++ Paimon, we use a multi-producer, single-consumer model to optimize file
-reading. The core idea is to split a file into line-based ReadRanges and assign
+In Paimon C++, we use a multi-producer, single-consumer model to optimize file
+reading. The core idea is to split a file into row-based read ranges and assign
 them to multiple reader threads (producers). Each reader thread owns an
 independent result queue that holds its processed RecordBatches. In the main
-reader thread (the consumer), we sort the heads of all queues by the ReadRange
-start offset in ascending order and select the RecordBatch with the smallest
-start offset to ensure globally ordered results.
+reader thread (the consumer), we sort the heads of all queues by the read
+range's starting row and select the RecordBatch with the smallest starting row
+to ensure globally ordered results.
 
 Read Range Splitting Strategy
 =============================
diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst
index 6fd5c41b..a9d81906 100644
--- a/docs/source/user_guide/write.rst
+++ b/docs/source/user_guide/write.rst
@@ -128,11 +128,11 @@ produce a correct ``Snapshot``, which commonly includes (but is not limited to):
 .. note::
 
    The C++ writer supports Append and PK tables and can produce
-   ``CommitMessage`` objects for both. ``FileStoreCommit`` currently executes
-   local commits only for append-only tables on non-object-store file systems.
-   PK and object-store commit messages must be sent to an external control
-   plane. Changelog is out of scope and should not be emitted in
-   ``CommitMessage`` until explicitly supported.
+   ``CommitMessage`` objects for both. ``FileStoreCommit`` supports direct
+   file-system commits for both table types on non-object-store paths.
+   Object-store paths require REST catalog commit mode. Changelog is out of
+   scope and should not be emitted in ``CommitMessage`` until explicitly
+   supported.
 
 Serialization and Deserialization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -155,21 +155,23 @@ Operational Flow
 1. Writer nodes perform data ingestion and produce Arrow ``RecordBatch``
    organized by partition and bucket.
 
-2. Writers flush batches into ORC/Parquet files via registered ``file.format``
-   and ``file-system`` backends, producing file-level metadata and per-batch
-   commit state.
+2. Writers flush batches into ORC, Parquet, or Avro files via registered
+   ``file.format`` and ``file-system`` backends, producing file-level metadata
+   and per-batch commit state.
 
 3. Each writer invokes ``PrepareCommit``, which:
-   - Aggregates per-writer state into a ``CommitMessage``.
+   - Aggregates per-writer state into ``CommitMessage`` objects.
    - Returns ``CommitMessage`` objects; it does not serialize them.
 
 4. The compute engine gathers ``CommitMessage`` objects from all writers. For
    cross-process transport, it explicitly calls ``Serialize`` or
    ``SerializeList`` and carries ``CurrentVersion()`` alongside the payload.
 
-5. For a supported local append-table commit, the engine passes the objects to
-   ``FileStoreCommit``. For PK tables or object-store paths, it sends the
-   serialized payload and version to an external control plane.
+5. For a direct file-system commit on a non-object-store path, the engine
+   passes the objects to ``FileStoreCommit`` for either an Append or PK table.
+   For an object-store path, it enables REST catalog commit mode, calls
+   ``Commit``, obtains the JSON request from ``GetLastCommitTableRequest``, and
+   sends that request to the REST catalog.
 
-6. The local committer or external coordinator validates the messages, updates
+6. The local committer or REST catalog validates the messages, updates
    manifests/metadata, and finalizes the snapshot atomically.
diff --git a/include/paimon/commit_message.h b/include/paimon/commit_message.h
index 45218927..d7cba283 100644
--- a/include/paimon/commit_message.h
+++ b/include/paimon/commit_message.h
@@ -33,7 +33,7 @@ class CommitMessageSerializer;
 class MemoryPool;
 
 /// Commit message for partition and bucket. Supports serialization and deserialization compatible
-/// with the Java version.
+/// with Java Paimon.
 ///
 /// @note Serialized payloads do not embed their serialization version. Transport
 /// `CurrentVersion()` alongside the payload and pass it explicitly to `Deserialize()` or
@@ -47,7 +47,7 @@ class PAIMON_EXPORT CommitMessage {
     virtual ~CommitMessage();
 
     /// Serializes a single commit message to a binary string format.
-    /// The serialized format is compatible with the Java version of Paimon.
+    /// The serialized format is compatible with Java Paimon.
     /// The serialization version is not included in the returned payload.
     /// @param commit_message The commit message to serialize.
     /// @param pool Memory pool for memory allocation during serialization.
diff --git a/include/paimon/file_store_commit.h b/include/paimon/file_store_commit.h
index cb4c2f26..82bbc003 100644
--- a/include/paimon/file_store_commit.h
+++ b/include/paimon/file_store_commit.h
@@ -44,10 +44,10 @@ class CommitMessage;
 /// The `FileStoreCommit` class provides interfaces for committing changes, expiring old snapshots,
 /// dropping partitions, and retrieving commit metrics.
 ///
-/// @note Local commit execution currently supports append-only tables on non-object-store file
-/// systems. `Create()` returns `NotImplemented` for primary-key tables and object-store paths.
-/// Primary-key writers can still produce `CommitMessage` objects; those messages must be committed
-/// by an external control plane.
+/// @note Direct file-system commits support append-only and primary-key tables on non-object-store
+/// paths. Object-store paths require REST catalog commit mode: enable it with
+/// `CommitContextBuilder::UseRESTCatalogCommit()`, call `Commit()` or `FilterAndCommit()`, and then
+/// retrieve the request with `GetLastCommitTableRequest()`.
 class PAIMON_EXPORT FileStoreCommit {
  public:
     /// Create an instance of `FileStoreCommit`.

From c6f2b91b5d2eb7343e9ba6e252f6a9c05e250cc9 Mon Sep 17 00:00:00 2001
From: lxy <38709059+lxy-9602@users.noreply.github.com>
Date: Wed, 29 Jul 2026 20:20:45 +0800
Subject: [PATCH 126/138] fix(blob): allow blob files across schema IDs

---
 .../operation/data_evolution_split_read.cpp   |  4 +-
 test/inte/blob_table_inte_test.cpp            | 56 +++++++++++++++++++
 2 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp
index a75acad0..5ebf6f56 100644
--- a/src/paimon/core/operation/data_evolution_split_read.cpp
+++ b/src/paimon/core/operation/data_evolution_split_read.cpp
@@ -96,9 +96,7 @@ Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptrschema_id != files_[0]->schema_id) {
-                return Status::Invalid("All files in a blob bunch should have the same schema id.");
-            }
+            // Blob files for the same field may span schema ids.
             if (file->write_cols != files_[0]->write_cols) {
                 return Status::Invalid(
                     "All files in a blob bunch should have the same write columns.");
diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp
index b4c9c281..6ff31288 100644
--- a/test/inte/blob_table_inte_test.cpp
+++ b/test/inte/blob_table_inte_test.cpp
@@ -876,6 +876,62 @@ TEST_P(BlobTableInteTest, TestBasic) {
     }
 }
 
+TEST_P(BlobTableInteTest, TestBlobFilesAcrossSchemaIds) {
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(/*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+
+    // Simulate the post-compaction layout: one normal file bridges blob files across schema ids.
+    std::vector write_cols0 = {"f0", "f2"};
+    auto src_array0 = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[0], fields_[2]}), R"([
+        [1, "a"],
+        [2, "b"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, write_cols0, {src_array0}));
+    ASSERT_OK(Commit(table_path, commit_msgs0));
+
+    std::vector blob_write_cols = {"f1"};
+    auto src_array1 = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([
+        ["c"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1,
+                         WriteArray(table_path, {}, blob_write_cols, {src_array1}));
+    SetFirstRowId(0, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    auto f3 = arrow::field("f3", arrow::int64());
+    ASSERT_OK(WriteNextSchema(table_path,
+                              {DataField(0, fields_[0]), DataField(1, fields_[1]),
+                               DataField(2, fields_[2]), DataField(3, f3)},
+                              /*highest_field_id=*/3, options));
+
+    auto src_array2 = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([
+        ["d"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs2,
+                         WriteArray(table_path, {}, blob_write_cols, {src_array2}));
+    SetFirstRowId(1, commit_msgs2);
+    ASSERT_OK(Commit(table_path, commit_msgs2));
+
+    auto expected_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields_), R"([
+        [1, "c", "a"],
+        [2, "d", "b"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, arrow::schema(fields_)->field_names(), expected_array));
+}
+
 TEST_P(BlobTableInteTest, TestMultipleAppends) {
     CreateTable();
     std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");

From 27f6d89ab54d1becfeb2477b815271dbba5d080c Mon Sep 17 00:00:00 2001
From: Nicholas Jiang 
Date: Thu, 30 Jul 2026 08:42:12 +0800
Subject: [PATCH 127/138] fix(core): support fallback keys for ignore-delete
 option

---
 include/paimon/defs.h                         |  9 +++
 src/paimon/common/defs.cpp                    |  3 +
 src/paimon/core/core_options.cpp              | 17 +++++-
 src/paimon/core/core_options_test.cpp         | 44 +++++++++++++++
 .../utils/primary_key_table_utils_test.cpp    | 56 +++++++++++++++++++
 5 files changed, 127 insertions(+), 2 deletions(-)

diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index 2f137539..6fd57434 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -323,6 +323,15 @@ struct PAIMON_EXPORT Options {
     /// "ignore-delete" - Whether to ignore delete records. Default value is "false".
     static const char IGNORE_DELETE[];
 
+    /// "first-row.ignore-delete" deprecated as a fallback for `IGNORE_DELETE`.
+    static const char FALLBACK_FIRST_ROW_IGNORE_DELETE[];
+
+    /// "deduplicate.ignore-delete" deprecated as a fallback for `IGNORE_DELETE`.
+    static const char FALLBACK_DEDUPLICATE_IGNORE_DELETE[];
+
+    /// "partial-update.ignore-delete" deprecated as a fallback for `IGNORE_DELETE`.
+    static const char FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE[];
+
     /// "fields.default-aggregate-function" - Default aggregate function of all fields for
     /// partial-update and aggregate merge function.
     static const char FIELDS_DEFAULT_AGG_FUNC[];
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index 605b0c0b..11823d5b 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -77,6 +77,9 @@ const char Options::SEQUENCE_FIELD_SORT_ORDER[] = "sequence.field.sort-order";
 const char Options::MERGE_ENGINE[] = "merge-engine";
 const char Options::SORT_ENGINE[] = "sort-engine";
 const char Options::IGNORE_DELETE[] = "ignore-delete";
+const char Options::FALLBACK_FIRST_ROW_IGNORE_DELETE[] = "first-row.ignore-delete";
+const char Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE[] = "deduplicate.ignore-delete";
+const char Options::FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE[] = "partial-update.ignore-delete";
 const char Options::FIELDS_DEFAULT_AGG_FUNC[] = "fields.default-aggregate-function";
 const char Options::DELETION_VECTORS_ENABLED[] = "deletion-vectors.enabled";
 const char Options::DELETION_VECTOR_INDEX_FILE_TARGET_SIZE[] =
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index f2b93d92..e38766fa 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -710,8 +710,21 @@ struct CoreOptions::Impl {
         PAIMON_RETURN_NOT_OK(parser.ParseSortEngine(&sort_engine));
         // Parse merge-engine - merge engine for primary key table, default "deduplicate"
         PAIMON_RETURN_NOT_OK(parser.ParseMergeEngine(&merge_engine));
-        // Parse ignore-delete - whether to ignore delete records, default false
-        PAIMON_RETURN_NOT_OK(parser.Parse(Options::IGNORE_DELETE, &ignore_delete));
+        // Parse ignore-delete - whether to ignore delete records, default false.
+        // Java CoreOptions declares first-row.ignore-delete, deduplicate.ignore-delete
+        // and partial-update.ignore-delete as fallback keys, checked in that order only
+        // when ignore-delete itself is absent.
+        std::optional ignore_delete_value;
+        PAIMON_RETURN_NOT_OK(parser.Parse(Options::IGNORE_DELETE, &ignore_delete_value));
+        for (const char* fallback_key : {Options::FALLBACK_FIRST_ROW_IGNORE_DELETE,
+                                         Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE,
+                                         Options::FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE}) {
+            if (ignore_delete_value.has_value()) {
+                break;
+            }
+            PAIMON_RETURN_NOT_OK(parser.Parse(fallback_key, &ignore_delete_value));
+        }
+        ignore_delete = ignore_delete_value.value_or(false);
         // Parse fields.default-aggregate-function - default agg function for partial-update
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::FIELDS_DEFAULT_AGG_FUNC, &field_default_func));
         // Parse changelog-producer - whether to double write to a changelog file, default "none"
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index e22989c4..90e905de 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -983,6 +983,50 @@ TEST(CoreOptionsTest, TestFallback) {
     }
 }
 
+TEST(CoreOptionsTest, TestIgnoreDeleteFallbackKeys) {
+    {
+        // Tables written by Java may carry first-row.ignore-delete instead of ignore-delete.
+        ASSERT_OK_AND_ASSIGN(
+            CoreOptions options,
+            CoreOptions::FromMap({{Options::MERGE_ENGINE, "first-row"},
+                                  {Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "true"}}));
+        ASSERT_TRUE(options.IgnoreDelete());
+    }
+    {
+        ASSERT_OK_AND_ASSIGN(
+            CoreOptions options,
+            CoreOptions::FromMap({{Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE, "true"}}));
+        ASSERT_TRUE(options.IgnoreDelete());
+    }
+    {
+        ASSERT_OK_AND_ASSIGN(
+            CoreOptions options,
+            CoreOptions::FromMap({{Options::FALLBACK_PARTIAL_UPDATE_IGNORE_DELETE, "true"}}));
+        ASSERT_TRUE(options.IgnoreDelete());
+    }
+    {
+        // The primary key takes precedence over fallback keys, matching Java CoreOptions.
+        ASSERT_OK_AND_ASSIGN(
+            CoreOptions options,
+            CoreOptions::FromMap({{Options::IGNORE_DELETE, "false"},
+                                  {Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "true"}}));
+        ASSERT_FALSE(options.IgnoreDelete());
+    }
+    {
+        // Fallback keys are checked in declaration order.
+        ASSERT_OK_AND_ASSIGN(
+            CoreOptions options,
+            CoreOptions::FromMap({{Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "false"},
+                                  {Options::FALLBACK_DEDUPLICATE_IGNORE_DELETE, "true"}}));
+        ASSERT_FALSE(options.IgnoreDelete());
+    }
+    {
+        ASSERT_NOK_WITH_MSG(
+            CoreOptions::FromMap({{Options::FALLBACK_FIRST_ROW_IGNORE_DELETE, "invalid"}}),
+            "Invalid Config [first-row.ignore-delete: invalid]");
+    }
+}
+
 TEST(CoreOptionsTest, TestMapStorageLayout) {
     // Test shared-shredding layout configured for a specific column
     {
diff --git a/src/paimon/core/utils/primary_key_table_utils_test.cpp b/src/paimon/core/utils/primary_key_table_utils_test.cpp
index 448a495f..a205f1e2 100644
--- a/src/paimon/core/utils/primary_key_table_utils_test.cpp
+++ b/src/paimon/core/utils/primary_key_table_utils_test.cpp
@@ -20,15 +20,23 @@
 
 #include 
 #include 
+#include 
+#include 
 #include 
 
 #include "arrow/type.h"
 #include "gtest/gtest.h"
 #include "paimon/common/types/data_field.h"
+#include "paimon/common/types/row_kind.h"
 #include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/core_options.h"
+#include "paimon/core/key_value.h"
+#include "paimon/core/mergetree/compact/merge_function.h"
 #include "paimon/defs.h"
+#include "paimon/memory/memory_pool.h"
 #include "paimon/status.h"
+#include "paimon/testing/utils/binary_row_generator.h"
+#include "paimon/testing/utils/key_value_checker.h"
 #include "paimon/testing/utils/testharness.h"
 
 namespace paimon::test {
@@ -56,4 +64,52 @@ TEST(PrimaryKeyTableUtilsTest, TestCreateSequenceFieldsComparator) {
     }
 }
 
+TEST(PrimaryKeyTableUtilsTest, TestCreateFirstRowMergeFunctionWithIgnoreDelete) {
+    auto pool = GetDefaultPool();
+    auto value_schema = arrow::schema({arrow::field("v0", arrow::int32())});
+
+    // ignore-delete can also be configured through the merge-engine-specific
+    // first-row.ignore-delete key, which must reach FirstRowMergeFunction the same way.
+    for (const char* ignore_delete_key :
+         {Options::IGNORE_DELETE, Options::FALLBACK_FIRST_ROW_IGNORE_DELETE}) {
+        ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                             CoreOptions::FromMap({{Options::MERGE_ENGINE, "first-row"},
+                                                   {ignore_delete_key, "true"}}));
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr merge_function,
+            PrimaryKeyTableUtils::CreateMergeFunction(value_schema, {"k0"}, core_options));
+        merge_function->Reset();
+
+        KeyValue insert_kv(RowKind::Insert(), /*sequence_number=*/0, /*level=*/0, /*key=*/
+                           BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
+                           /*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get()));
+        KeyValue delete_kv(RowKind::Delete(), /*sequence_number=*/1, /*level=*/0, /*key=*/
+                           BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
+                           /*value=*/BinaryRowGenerator::GenerateRowPtr({200}, pool.get()));
+        ASSERT_OK(merge_function->Add(std::move(insert_kv)));
+        ASSERT_OK(merge_function->Add(std::move(delete_kv)));
+
+        ASSERT_OK_AND_ASSIGN(std::optional result_kv, merge_function->GetResult());
+        ASSERT_TRUE(result_kv.has_value());
+        KeyValue expected(RowKind::Insert(), /*sequence_number=*/0, /*level=*/0, /*key=*/
+                          BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
+                          /*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get()));
+        KeyValueChecker::CheckResult(expected, result_kv.value(), /*key_arity=*/1,
+                                     /*value_arity=*/1);
+    }
+
+    // Without the option, the first-row merge engine still rejects retract records.
+    ASSERT_OK_AND_ASSIGN(CoreOptions core_options,
+                         CoreOptions::FromMap({{Options::MERGE_ENGINE, "first-row"}}));
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr merge_function,
+        PrimaryKeyTableUtils::CreateMergeFunction(value_schema, {"k0"}, core_options));
+    merge_function->Reset();
+    KeyValue delete_kv(RowKind::Delete(), /*sequence_number=*/0, /*level=*/0, /*key=*/
+                       BinaryRowGenerator::GenerateRowPtr({10}, pool.get()),
+                       /*value=*/BinaryRowGenerator::GenerateRowPtr({100}, pool.get()));
+    ASSERT_NOK_WITH_MSG(merge_function->Add(std::move(delete_kv)),
+                        "First row merge engine can not accept DELETE/UPDATE_BEFORE records");
+}
+
 }  // namespace paimon::test

From 662bccdf13102e3ff007329b15e6c26cd4bd246c Mon Sep 17 00:00:00 2001
From: lxy <38709059+lxy-9602@users.noreply.github.com>
Date: Thu, 30 Jul 2026 11:00:04 +0800
Subject: [PATCH 128/138] feat(shredding): support adaptive schemas across
 rolling files

---
 .../map_shared_shredding_schema_utils.h       |   4 +-
 include/paimon/defs.h                         |  14 +
 src/paimon/CMakeLists.txt                     |   2 +-
 .../map_shared_shredding_batch_converter.cpp  |  11 +
 .../map_shared_shredding_batch_converter.h    |   3 +
 ..._shared_shredding_batch_converter_test.cpp | 130 +++++
 .../shredding/map_shared_shredding_context.h  |   2 +-
 .../map_shared_shredding_file_reader_test.cpp |   9 +-
 .../map_shared_shredding_schema_utils.cpp     |   5 +-
 ...map_shared_shredding_schema_utils_test.cpp |  20 +-
 .../shredding/map_shared_shredding_utils.cpp  |  34 +-
 .../shredding/map_shared_shredding_utils.h    |  48 +-
 .../map_shared_shredding_utils_test.cpp       |  39 +-
 ...ap_shared_shredding_write_plan_factory.cpp |  54 +-
 .../map_shared_shredding_write_plan_factory.h |  23 +-
 .../data/shredding/map_shredding_defs.h       |   3 +
 .../shredding_write_plan_factories.cpp        |  29 +-
 .../shredding_write_plan_factories.h          |   8 +-
 .../shredding/shredding_write_plan_factory.h  |   9 +-
 .../infer_variant_shredding_schema.cpp        | 461 +++++++++++++++++-
 .../variant/infer_variant_shredding_schema.h  |  89 +++-
 .../infer_variant_shredding_schema_test.cpp   | 174 ++++++-
 src/paimon/common/data/variant/variant_defs.h |   5 +
 .../variant_shredding_inference_session.cpp   |  61 +++
 .../variant_shredding_inference_session.h     |  62 +++
 .../variant_shredding_read_plan_factory.cpp   |  37 ++
 ...riant_shredding_read_plan_factory_test.cpp |  33 ++
 .../data/variant/variant_shredding_test.cpp   |  12 +-
 .../data/variant/variant_shredding_utils.cpp  |  17 +
 .../data/variant/variant_shredding_utils.h    |   4 +
 .../variant/variant_shredding_write_plan.cpp  |  70 ++-
 .../variant/variant_shredding_write_plan.h    |  13 +-
 .../variant_shredding_write_plan_factory.cpp  | 159 +++---
 .../variant_shredding_write_plan_factory.h    |  24 +-
 ...iant_shredding_write_plan_factory_test.cpp | 330 ++++++++++++-
 src/paimon/common/defs.cpp                    |   6 +
 src/paimon/core/append/append_only_writer.cpp |  47 +-
 src/paimon/core/append/append_only_writer.h   |   9 +-
 .../core/append/append_only_writer_test.cpp   | 245 ++++------
 src/paimon/core/core_options.cpp              |  70 +++
 src/paimon/core/core_options.h                |   5 +
 src/paimon/core/core_options_test.cpp         |  40 +-
 .../core/io/infer_shredding_file_writer.h     |   5 +-
 .../io/infer_shredding_file_writer_test.cpp   |  12 +-
 .../io/map_shared_shredding_core_utils.cpp    | 141 ------
 .../core/io/map_shared_shredding_core_utils.h |  51 --
 .../core/io/rolling_blob_file_writer.cpp      |  21 +-
 src/paimon/core/io/rolling_blob_file_writer.h |   2 +-
 src/paimon/core/io/rolling_file_writer.h      |  15 +-
 ...edding_append_data_file_writer_factory.cpp |  10 +-
 ...ing_key_value_data_file_writer_factory.cpp |  10 +-
 src/paimon/core/io/single_file_writer.h       |  19 +-
 .../core/io/single_file_writer_test.cpp       |  38 ++
 src/paimon/core/manifest/manifest_file.cpp    |   4 +-
 .../compact/changelog_merge_tree_rewriter.cpp |  11 +-
 .../compact/changelog_merge_tree_rewriter.h   |   1 -
 .../lookup_merge_tree_compact_rewriter.cpp    |  10 +-
 .../lookup_merge_tree_compact_rewriter.h      |   1 -
 ...ookup_merge_tree_compact_rewriter_test.cpp |  20 +-
 .../compact/merge_tree_compact_rewriter.cpp   |  33 +-
 .../compact/merge_tree_compact_rewriter.h     |   8 -
 .../merge_tree_compact_rewriter_test.cpp      |   9 +
 .../remote_lookup_file_manager_test.cpp       |   3 +-
 .../core/mergetree/lookup_levels_test.cpp     |   3 +-
 .../core/mergetree/merge_tree_writer.cpp      |  31 +-
 src/paimon/core/mergetree/merge_tree_writer.h |   8 +-
 .../core/mergetree/merge_tree_writer_test.cpp | 116 ++---
 .../core/operation/abstract_split_read.cpp    |   3 +-
 .../append_only_file_store_write.cpp          |  27 +-
 .../operation/append_only_file_store_write.h  |   6 +-
 .../append_only_file_store_write_test.cpp     | 187 ++++---
 .../operation/key_value_file_store_write.cpp  |  16 +-
 .../key_value_file_store_write_test.cpp       |  48 +-
 .../postpone_bucket_file_store_write.h        |  10 +-
 .../core/postpone/postpone_bucket_writer.cpp  |  37 +-
 .../core/postpone/postpone_bucket_writer.h    |   8 +-
 .../postpone/postpone_bucket_writer_test.cpp  |  97 ++--
 .../core/schema/arrow_schema_validator.cpp    |   4 +
 src/paimon/core/schema/schema_validation.cpp  |  97 +++-
 .../core/schema/schema_validation_test.cpp    | 161 +++++-
 src/paimon/core/schema/table_schema.cpp       |   7 +-
 src/paimon/core/schema/table_schema_test.cpp  |  28 ++
 .../format/parquet/variant_parquet_test.cpp   | 126 ++++-
 test/inte/append_compaction_inte_test.cpp     |  14 +-
 test/inte/blob_table_inte_test.cpp            |  14 +-
 test/inte/data_evolution_table_test.cpp       |   2 +-
 test/inte/nested_column_pruning_inte_test.cpp |   2 +-
 test/inte/pk_compaction_inte_test.cpp         |   4 +-
 test/inte/variant_table_inte_test.cpp         | 395 +++++++++++++++
 test/inte/write_and_read_inte_test.cpp        | 339 ++++++++++++-
 test/inte/write_inte_test.cpp                 |  71 +++
 91 files changed, 3619 insertions(+), 1086 deletions(-)
 create mode 100644 src/paimon/common/data/variant/variant_shredding_inference_session.cpp
 create mode 100644 src/paimon/common/data/variant/variant_shredding_inference_session.h
 delete mode 100644 src/paimon/core/io/map_shared_shredding_core_utils.cpp
 delete mode 100644 src/paimon/core/io/map_shared_shredding_core_utils.h

diff --git a/include/paimon/data/shredding/map_shared_shredding_schema_utils.h b/include/paimon/data/shredding/map_shared_shredding_schema_utils.h
index 393b625e..b8470c4d 100644
--- a/include/paimon/data/shredding/map_shared_shredding_schema_utils.h
+++ b/include/paimon/data/shredding/map_shared_shredding_schema_utils.h
@@ -88,11 +88,9 @@ class PAIMON_EXPORT MapSharedShreddingSchemaUtils {
     /// @param physical_schema The Arrow C physical schema that contains the target field.
     ///        Ownership of schema resources is transferred to this method.
     /// @param field_name The physical field name whose metadata should be extracted.
-    /// @param compression Compression codec name for field_dict deserialization.
     /// @return Parsed shared-shredding metadata for the field.
     static Result ExtractMetadataFromField(
-        std::unique_ptr<::ArrowSchema> physical_schema, const std::string& field_name,
-        const std::string& compression);
+        std::unique_ptr<::ArrowSchema> physical_schema, const std::string& field_name);
 };
 
 }  // namespace paimon
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index 6fd57434..b938dd38 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -120,6 +120,12 @@ struct PAIMON_EXPORT Options {
     /// append table: the default value is 256 MB.
     static const char TARGET_FILE_SIZE[];
 
+    /// "target-file-row-num" - Target number of rows per newly written data file. Disabled by
+    /// default. A file rolls when this or target-file-size is reached, whichever comes first.
+    /// This limit is enforced at write-batch granularity, so a file may exceed the target by up
+    /// to one batch.
+    static const char TARGET_FILE_ROW_NUM[];
+
     /// "blob.target-file-size" - Target size of a blob file. Default is TARGET_FILE_SIZE.
     static const char BLOB_TARGET_FILE_SIZE[];
 
@@ -436,6 +442,8 @@ struct PAIMON_EXPORT Options {
     /// "variant.inferShreddingSchema" - Whether to automatically infer the shredding schema when
     /// writing Variant columns. Default value is "false".
     static const char VARIANT_INFER_SHREDDING_SCHEMA[];
+    /// "variant.shredding.inferenceMode" - "per-file" or "adaptive". Default is "per-file".
+    static const char VARIANT_SHREDDING_INFERENCE_MODE[];
     /// "variant.shredding.maxSchemaWidth" - Maximum number of shredded fields allowed in an
     /// inferred schema. Default value is 300.
     static const char VARIANT_SHREDDING_MAX_SCHEMA_WIDTH[];
@@ -449,6 +457,12 @@ struct PAIMON_EXPORT Options {
     /// "variant.shredding.maxInferBufferRow" - Maximum number of rows to buffer for schema
     /// inference. Default value is 4096.
     static const char VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW[];
+    /// "variant.shredding.adaptive.maxInferBufferRow" - Maximum prefix rows sampled after the
+    /// first file in an adaptive session. Default value is 256.
+    static const char VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW[];
+    /// "variant.shredding.adaptive.retentionRatio" - Minimum combined ratio for retaining a
+    /// previously selected path. Default value is 0.05.
+    static const char VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO[];
 
     /// "blob-as-descriptor" - Read blob field using blob descriptor rather than blob
     /// bytes. Default value is "false".
diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 4afe3a27..1dfc2a5d 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -42,6 +42,7 @@ set(PAIMON_COMMON_SRCS
     common/data/timestamp.cpp
     common/data/variant/generic_variant.cpp
     common/data/variant/infer_variant_shredding_schema.cpp
+    common/data/variant/variant_shredding_inference_session.cpp
     common/data/variant/variant.cpp
     common/data/variant/variant_binary_util.cpp
     common/data/variant/variant_builder.cpp
@@ -266,7 +267,6 @@ set(PAIMON_CORE_SRCS
     core/io/key_value_meta_projection_consumer.cpp
     core/io/key_value_projection_consumer.cpp
     core/io/key_value_projection_reader.cpp
-    core/io/map_shared_shredding_core_utils.cpp
     core/io/shredding_append_data_file_writer_factory.cpp
     core/io/shredding_key_value_data_file_writer_factory.cpp
     core/io/multiple_blob_file_writer.cpp
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp
index 6869a5b2..2a9b69dc 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.cpp
@@ -340,6 +340,17 @@ Result MapSharedShreddingBatchConverter::BuildField
         "cannot find field_name '{}' in MapSharedShreddingBatchConverter contexts", field_name));
 }
 
+Result MapSharedShreddingBatchConverter::GetMaxRowWidth(
+    const std::string& field_name) const {
+    for (const auto& context : contexts_) {
+        if (context.field_name == field_name) {
+            return context.allocator->GetMaxRowWidth();
+        }
+    }
+    return Status::Invalid(fmt::format(
+        "cannot find field_name '{}' in MapSharedShreddingBatchConverter contexts", field_name));
+}
+
 const std::vector& MapSharedShreddingBatchConverter::GetShreddingColumnNames() const {
     return shredding_field_names_;
 }
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h
index 8bea8120..b808692c 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h
+++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter.h
@@ -74,6 +74,9 @@ class MapSharedShreddingBatchConverter : public ShreddingBatchConverter {
     /// Called at file close to serialize metadata.
     Result BuildFieldMeta(const std::string& field_name) const;
 
+    /// Returns the maximum number of entries observed in one row for a shredding column.
+    Result GetMaxRowWidth(const std::string& field_name) const;
+
     /// Returns all shredding column field names.
     const std::vector& GetShreddingColumnNames() const;
 
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp
index db63fd7c..df99c021 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_batch_converter_test.cpp
@@ -31,6 +31,7 @@
 #include "gtest/gtest.h"
 #include "paimon/common/data/shredding/map_shared_shredding_context.h"
 #include "paimon/common/data/shredding/map_shared_shredding_utils.h"
+#include "paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h"
 #include "paimon/common/data/shredding/map_shredding_defs.h"
 #include "paimon/core/core_options.h"
 #include "paimon/memory/memory_pool.h"
@@ -121,6 +122,77 @@ TEST_F(MapSharedShreddingBatchConverterTest, BasicConversion) {
     ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value());
 }
 
+TEST_F(MapSharedShreddingBatchConverterTest, MapWithNullValue) {
+    auto logical_schema =
+        arrow::schema({arrow::field("metrics", arrow::map(arrow::utf8(), arrow::int64()))});
+    auto context =
+        std::make_shared(std::map{{"metrics", 2}});
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"metrics", "plain"}}));
+    ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create(
+                                             logical_schema, context, options, pool_));
+    auto logical_type = arrow::struct_(logical_schema->fields());
+    auto physical_type = arrow::struct_(converter->GetPhysicalSchema()->fields());
+
+    auto actual =
+        RunConvert(logical_type, R"([[[["a", null], ["b", 20]]]])", physical_type, converter.get());
+    auto expected = ArrayFromJSON(physical_type, R"([[[[0, 1], null, 20, null]]])").ValueOrDie();
+    AssertArrayEquals(expected, actual);
+
+    MapSharedShreddingFieldMeta expected_meta;
+    expected_meta.name_to_id = {{"a", 0}, {"b", 1}};
+    expected_meta.field_to_columns = {{0, {0}}, {1, {1}}};
+    expected_meta.num_columns = 2;
+    expected_meta.max_row_width = 2;
+    ASSERT_EQ(expected_meta, converter->BuildFieldMeta("metrics").value());
+}
+
+TEST_F(MapSharedShreddingBatchConverterTest, OverflowWhenExceedK) {
+    auto logical_schema =
+        arrow::schema({arrow::field("metrics", arrow::map(arrow::utf8(), arrow::int64()))});
+    auto context =
+        std::make_shared(std::map{{"metrics", 2}});
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"metrics", "plain"}}));
+    ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create(
+                                             logical_schema, context, options, pool_));
+    auto logical_type = arrow::struct_(logical_schema->fields());
+    auto physical_type = arrow::struct_(converter->GetPhysicalSchema()->fields());
+
+    auto actual = RunConvert(logical_type, R"([[[["a", 10], ["b", 20], ["c", 30]]]])",
+                             physical_type, converter.get());
+    auto expected = ArrayFromJSON(physical_type, R"([[[[0, 1], 10, 20, [[2, 30]]]]])").ValueOrDie();
+    AssertArrayEquals(expected, actual);
+
+    MapSharedShreddingFieldMeta expected_meta;
+    expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}};
+    expected_meta.field_to_columns = {{0, {0}}, {1, {1}}};
+    expected_meta.overflow_field_set = {2};
+    expected_meta.num_columns = 2;
+    expected_meta.max_row_width = 3;
+    ASSERT_EQ(expected_meta, converter->BuildFieldMeta("metrics").value());
+}
+
+TEST_F(MapSharedShreddingBatchConverterTest, EmptyAndNullMaps) {
+    auto logical_schema =
+        arrow::schema({arrow::field("metrics", arrow::map(arrow::utf8(), arrow::int64()))});
+    auto context =
+        std::make_shared(std::map{{"metrics", 2}});
+    ASSERT_OK_AND_ASSIGN(CoreOptions options, MakeCoreOptions({{"metrics", "plain"}}));
+    ASSERT_OK_AND_ASSIGN(auto converter, MapSharedShreddingBatchConverter::Create(
+                                             logical_schema, context, options, pool_));
+    auto logical_type = arrow::struct_(logical_schema->fields());
+    auto physical_type = arrow::struct_(converter->GetPhysicalSchema()->fields());
+
+    auto actual = RunConvert(logical_type, R"([[null], [[]]])", physical_type, converter.get());
+    auto expected =
+        ArrayFromJSON(physical_type, R"([[null], [[[-1, -1], null, null, null]]])").ValueOrDie();
+    AssertArrayEquals(expected, actual);
+
+    MapSharedShreddingFieldMeta expected_meta;
+    expected_meta.num_columns = 2;
+    expected_meta.max_row_width = 0;
+    ASSERT_EQ(expected_meta, converter->BuildFieldMeta("metrics").value());
+}
+
 TEST_F(MapSharedShreddingBatchConverterTest, NestedValueStruct) {
     // MAP>, K=2
     auto value_type = arrow::struct_({
@@ -411,13 +483,18 @@ TEST_F(MapSharedShreddingBatchConverterTest, BuildFieldMetaInvalidFieldName) {
 
     // Valid case: "tags" exists
     ASSERT_OK_AND_ASSIGN([[maybe_unused]] auto meta, converter->BuildFieldMeta("tags"));
+    ASSERT_OK_AND_ASSIGN(int32_t max_row_width, converter->GetMaxRowWidth("tags"));
+    ASSERT_EQ(0, max_row_width);
 
     // Invalid case: "id" is not a shredding field
     ASSERT_NOK_WITH_MSG(converter->BuildFieldMeta("id"), "cannot find field_name 'id'");
+    ASSERT_NOK_WITH_MSG(converter->GetMaxRowWidth("id"), "cannot find field_name 'id'");
 
     // Invalid case: nonexistent field name
     ASSERT_NOK_WITH_MSG(converter->BuildFieldMeta("nonexistent"),
                         "cannot find field_name 'nonexistent'");
+    ASSERT_NOK_WITH_MSG(converter->GetMaxRowWidth("nonexistent"),
+                        "cannot find field_name 'nonexistent'");
 }
 
 TEST_F(MapSharedShreddingBatchConverterTest, SequentialPlacementUsesSmallestColumn) {
@@ -499,4 +576,57 @@ TEST_F(MapSharedShreddingBatchConverterTest, LruPlacementPreservesResidentColumn
     ASSERT_EQ(expected_meta, converter->BuildFieldMeta("tags").value());
 }
 
+TEST_F(MapSharedShreddingBatchConverterTest, FactoryUsesMaxColumnCountForFirstFile) {
+    auto logical_schema =
+        arrow::schema({arrow::field("tags", arrow::map(arrow::utf8(), arrow::int32()))});
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions options,
+        CoreOptions::FromMap({{"fields.tags.map.storage-layout", "shared-shredding"},
+                              {"fields.tags.map.shared-shredding.max-columns", "4"}}));
+    ASSERT_OK_AND_ASSIGN(
+        auto factory, MapSharedShreddingWritePlanFactory::Create(options, logical_schema, pool_));
+
+    ASSERT_TRUE(factory->ShouldCreateWritePlan());
+    ASSERT_FALSE(factory->ShouldInferWritePlan());
+    ASSERT_EQ(0, factory->InferBufferRowCount());
+    ASSERT_OK_AND_ASSIGN(auto converter, factory->CreateConverter("parquet", {}));
+    ASSERT_OK_AND_ASSIGN(auto expected, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                            logical_schema, {{"tags", 4}}));
+    ASSERT_TRUE(converter->GetPhysicalSchema()->Equals(*expected));
+}
+
+TEST_F(MapSharedShreddingBatchConverterTest, FactoryUsesCompletedFileStatsForNextFile) {
+    auto logical_schema =
+        arrow::schema({arrow::field("tags", arrow::map(arrow::utf8(), arrow::int32()))});
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions options,
+        CoreOptions::FromMap({{"fields.tags.map.storage-layout", "shared-shredding"},
+                              {"fields.tags.map.shared-shredding.max-columns", "8"}}));
+    ASSERT_OK_AND_ASSIGN(
+        auto factory, MapSharedShreddingWritePlanFactory::Create(options, logical_schema, pool_));
+    ASSERT_OK_AND_ASSIGN(auto first, factory->CreateConverter("parquet", {}));
+
+    auto logical_type = arrow::struct_(logical_schema->fields());
+    auto first_physical_type = arrow::struct_(first->GetPhysicalSchema()->fields());
+    auto input = ArrayFromJSON(logical_type, R"([
+        [[["a", 1], ["b", 2]]],
+        [[["c", 3], ["d", 4], ["e", 5]]]
+    ])")
+                     .ValueOrDie();
+    ArrowArray c_input;
+    ASSERT_TRUE(arrow::ExportArray(*input, &c_input).ok());
+    ASSERT_OK_AND_ASSIGN(auto c_output, first->Convert(&c_input));
+    auto first_output = arrow::ImportArray(c_output.get(), first_physical_type).ValueOrDie();
+    ASSERT_EQ(2, first_output->length());
+    ASSERT_OK(factory->OnFileCompleted(first));
+
+    ASSERT_OK_AND_ASSIGN(auto second, factory->CreateConverter("parquet", {}));
+    ASSERT_OK_AND_ASSIGN(auto expected_first, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                  logical_schema, {{"tags", 8}}));
+    ASSERT_OK_AND_ASSIGN(auto expected_second, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                   logical_schema, {{"tags", 3}}));
+    ASSERT_TRUE(first->GetPhysicalSchema()->Equals(*expected_first));
+    ASSERT_TRUE(second->GetPhysicalSchema()->Equals(*expected_second));
+}
+
 }  // namespace paimon
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_context.h b/src/paimon/common/data/shredding/map_shared_shredding_context.h
index c3497556..1f530a36 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_context.h
+++ b/src/paimon/common/data/shredding/map_shared_shredding_context.h
@@ -28,7 +28,7 @@ namespace paimon {
 
 /// Cross-file shared context for shared-shredding MAP columns.
 ///
-/// Lifetime: same as the owning writer (e.g. AppendOnlyWriter).
+/// Lifetime: same as one rolling writer / write-plan factory.
 /// Holds per-column K_max and a sliding window of recent max_row_width
 /// values to support adaptive K sizing across files.
 ///
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp
index 9fbdd9ba..c8ab3d83 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_file_reader_test.cpp
@@ -111,9 +111,7 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test {
             if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata)) {
                 continue;
             }
-            EXPECT_OK_AND_ASSIGN(auto meta,
-                                 MapSharedShreddingUtils::DeserializeMetadata(
-                                     metadata, MapSharedShreddingDefine::kDefaultDictCompression));
+            EXPECT_OK_AND_ASSIGN(auto meta, MapSharedShreddingUtils::DeserializeMetadata(metadata));
             auto physical_type =
                 arrow::internal::checked_pointer_cast(field->type());
             std::shared_ptr item_field;
@@ -222,12 +220,9 @@ class MapSharedShreddingFileReaderTest : public ::testing::Test {
         const std::optional>& write_cols, int64_t max_sequence_number,
         const std::shared_ptr& path_factory,
         const std::shared_ptr& compact_manager) const {
-        PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr shredding_context,
-            MapSharedShreddingUtils::CreateShreddingContext(logical_schema, core_options));
         return std::make_unique(core_options, schema_id, logical_schema,
                                                   write_cols, max_sequence_number, path_factory,
-                                                  compact_manager, shredding_context, pool_);
+                                                  compact_manager, pool_);
     }
 
  private:
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp
index 11a8f2f0..c8dc5412 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils.cpp
@@ -78,8 +78,7 @@ Result> MapSharedShreddingSchemaUtils::AttachMeta
 }
 
 Result MapSharedShreddingSchemaUtils::ExtractMetadataFromField(
-    std::unique_ptr<::ArrowSchema> physical_schema, const std::string& field_name,
-    const std::string& compression) {
+    std::unique_ptr<::ArrowSchema> physical_schema, const std::string& field_name) {
     if (!physical_schema) {
         return Status::Invalid("physical schema is null");
     }
@@ -93,7 +92,7 @@ Result MapSharedShreddingSchemaUtils::ExtractMetada
 
     auto metadata =
         field->metadata() ? field->metadata()->Copy() : std::shared_ptr();
-    return MapSharedShreddingUtils::DeserializeMetadata(metadata, compression);
+    return MapSharedShreddingUtils::DeserializeMetadata(metadata);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp
index e6e5cf63..e421bc3f 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_schema_utils_test.cpp
@@ -65,7 +65,7 @@ TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaBasic) {
     auto tags_metadata = updated_schema->GetFieldByName("tags")->metadata()->Copy();
     ASSERT_TRUE(MapSharedShreddingUtils::HasShreddingMetadata(tags_metadata));
     ASSERT_OK_AND_ASSIGN(auto deserialized,
-                         MapSharedShreddingUtils::DeserializeMetadata(tags_metadata, "none"));
+                         MapSharedShreddingUtils::DeserializeMetadata(tags_metadata));
     ASSERT_EQ(deserialized, tags_meta);
 }
 
@@ -162,7 +162,7 @@ TEST(MapSharedShreddingSchemaUtilsTest, AttachMetadataToSchemaOverwritesExisting
     }
     ASSERT_EQ(storage_layout_key_count, 1);
     ASSERT_OK_AND_ASSIGN(auto deserialized,
-                         MapSharedShreddingUtils::DeserializeMetadata(updated_metadata, "none"));
+                         MapSharedShreddingUtils::DeserializeMetadata(updated_metadata));
     ASSERT_EQ(deserialized, tags_meta);
 }
 
@@ -186,7 +186,7 @@ TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromField) {
     auto c_schema = std::make_unique<::ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok());
     ASSERT_OK_AND_ASSIGN(auto parsed_meta, MapSharedShreddingSchemaUtils::ExtractMetadataFromField(
-                                               std::move(c_schema), "tags", "none"));
+                                               std::move(c_schema), "tags"));
     ASSERT_EQ(parsed_meta, tags_meta);
 }
 
@@ -197,14 +197,14 @@ TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromFieldNoShreddingMetad
     auto c_schema = std::make_unique<::ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok());
 
-    ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::ExtractMetadataFromField(std::move(c_schema),
-                                                                                "tags", "none"),
-                        "metadata is null or storage layout is not shared-shredding");
+    ASSERT_NOK_WITH_MSG(
+        MapSharedShreddingSchemaUtils::ExtractMetadataFromField(std::move(c_schema), "tags"),
+        "metadata is null or storage layout is not shared-shredding");
 }
 
 TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromFieldInvalidInput) {
     ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::ExtractMetadataFromField(
-                            std::unique_ptr<::ArrowSchema>(), "tags", "none"),
+                            std::unique_ptr<::ArrowSchema>(), "tags"),
                         "physical schema is null");
 
     auto schema = arrow::schema({arrow::field("id", arrow::int32())});
@@ -212,9 +212,9 @@ TEST(MapSharedShreddingSchemaUtilsTest, ExtractMetadataFromFieldInvalidInput) {
     auto c_schema = std::make_unique<::ArrowSchema>();
     ASSERT_TRUE(arrow::ExportSchema(*schema, c_schema.get()).ok());
 
-    ASSERT_NOK_WITH_MSG(MapSharedShreddingSchemaUtils::ExtractMetadataFromField(std::move(c_schema),
-                                                                                "tags", "none"),
-                        "Shared-shredding field 'tags' not found in physical schema.");
+    ASSERT_NOK_WITH_MSG(
+        MapSharedShreddingSchemaUtils::ExtractMetadataFromField(std::move(c_schema), "tags"),
+        "Shared-shredding field 'tags' not found in physical schema.");
 }
 
 TEST(MapSharedShreddingSchemaUtilsTest, LogicalToPhysicalSchemaInvalidInput) {
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp
index 5c6f421a..6ba2b43c 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.cpp
@@ -29,7 +29,6 @@
 #include "paimon/common/compression/block_compressor.h"
 #include "paimon/common/compression/block_decompressor.h"
 #include "paimon/common/data/shredding/map_shared_shredding_batch_converter.h"
-#include "paimon/common/data/shredding/map_shared_shredding_context.h"
 #include "paimon/common/utils/arrow/status_utils.h"
 #include "paimon/common/utils/string_utils.h"
 #include "paimon/core/core_options.h"
@@ -81,18 +80,6 @@ Result> MapSharedShreddingUtils::DetectShreddingColumns
     return field_names;
 }
 
-Result> MapSharedShreddingUtils::CreateShreddingContext(
-    const std::shared_ptr& schema, const CoreOptions& options) {
-    PAIMON_ASSIGN_OR_RAISE(std::vector shredding_field_names,
-                           DetectShreddingColumns(schema, options));
-    if (shredding_field_names.empty()) {
-        return std::shared_ptr();
-    }
-    std::map field_to_k_max;
-    PAIMON_ASSIGN_OR_RAISE(field_to_k_max, BuildColumnToNumColumns(shredding_field_names, options));
-    return std::make_shared(field_to_k_max);
-}
-
 // ---- Schema conversion ----
 std::shared_ptr MapSharedShreddingUtils::BuildSpecificPhysicalStructType(
     const std::shared_ptr& value_type, const std::set& physical_col_ids,
@@ -361,6 +348,7 @@ Result> DeserializeOverflowSet(const std::string& json_str) {
 Status MapSharedShreddingUtils::SerializeMetadata(const MapSharedShreddingFieldMeta& field_meta,
                                                   const std::string& compression,
                                                   arrow::KeyValueMetadata* metadata) {
+    const std::string normalized_compression = StringUtils::ToLowerCase(compression);
     PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set(
         MapShreddingDefine::kStorageLayout, MapShreddingDefine::kStorageLayoutSharedShredding));
     PAIMON_RETURN_NOT_OK_FROM_ARROW(
@@ -370,8 +358,10 @@ Status MapSharedShreddingUtils::SerializeMetadata(const MapSharedShreddingFieldM
     std::string field_dict_json = SerializeFieldDict(field_meta);
     PAIMON_RETURN_NOT_OK_FROM_ARROW(metadata->Set(MapSharedShreddingDefine::kFieldDictOriginalSize,
                                                   std::to_string(field_dict_json.size())));
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        metadata->Set(MapSharedShreddingDefine::kFieldDictCompression, normalized_compression));
     PAIMON_ASSIGN_OR_RAISE(std::string compressed_dict,
-                           CompressString(field_dict_json, compression));
+                           CompressString(field_dict_json, normalized_compression));
     PAIMON_RETURN_NOT_OK_FROM_ARROW(
         metadata->Set(MapSharedShreddingDefine::kFieldDict, std::move(compressed_dict)));
 
@@ -388,7 +378,7 @@ Status MapSharedShreddingUtils::SerializeMetadata(const MapSharedShreddingFieldM
 }
 
 Result MapSharedShreddingUtils::DeserializeMetadata(
-    const std::shared_ptr& metadata, const std::string& compression) {
+    const std::shared_ptr& metadata) {
     if (!HasShreddingMetadata(metadata)) {
         return Status::Invalid("metadata is null or storage layout is not shared-shredding");
     }
@@ -408,8 +398,13 @@ Result MapSharedShreddingUtils::DeserializeMetadata
         GetRequiredInt32(metadata, MapSharedShreddingDefine::kFieldDictOriginalSize));
     PAIMON_ASSIGN_OR_RAISE(std::string compressed_dict,
                            GetRequiredValue(metadata, MapSharedShreddingDefine::kFieldDict));
+    int32_t compression_index = metadata->FindKey(MapSharedShreddingDefine::kFieldDictCompression);
+    std::string field_dict_compression = compression_index < 0
+                                             ? MapSharedShreddingDefine::kDefaultDictCompression
+                                             : metadata->value(compression_index);
+    field_dict_compression = StringUtils::ToLowerCase(field_dict_compression);
     PAIMON_ASSIGN_OR_RAISE(std::string field_dict_json,
-                           DecompressString(compressed_dict, original_len, compression));
+                           DecompressString(compressed_dict, original_len, field_dict_compression));
     PAIMON_ASSIGN_OR_RAISE(result.name_to_id, DeserializeFieldDict(field_dict_json));
 
     // field_columns
@@ -456,10 +451,8 @@ Result MapSharedShreddingUtils::IsOverflowField(const MapSharedShreddingFi
 std::function>()>
 MapSharedShreddingUtils::BuildMetadataFinalizer(
     const std::shared_ptr& converter,
-    const std::string& compression, const std::shared_ptr& context,
-    const std::shared_ptr& physical_schema) {
-    return [converter, compression, context,
-            physical_schema]() -> Result> {
+    const std::string& compression, const std::shared_ptr& physical_schema) {
+    return [converter, compression, physical_schema]() -> Result> {
         const std::vector& shredding_field_names =
             converter->GetShreddingColumnNames();
         arrow::FieldVector updated_fields = physical_schema->fields();
@@ -477,7 +470,6 @@ MapSharedShreddingUtils::BuildMetadataFinalizer(
             PAIMON_RETURN_NOT_OK(
                 MapSharedShreddingUtils::SerializeMetadata(file_meta, compression, metadata.get()));
             updated_fields[col_index] = field->WithMetadata(metadata);
-            context->ReportFileStats(field_name, file_meta.max_row_width);
         }
         return arrow::schema(std::move(updated_fields));
     };
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils.h b/src/paimon/common/data/shredding/map_shared_shredding_utils.h
index 97d57b80..00351b65 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_utils.h
+++ b/src/paimon/common/data/shredding/map_shared_shredding_utils.h
@@ -42,7 +42,6 @@ namespace paimon {
 
 class CoreOptions;
 class MapSharedShreddingBatchConverter;
-class MapSharedShreddingContext;
 
 /// Utility functions for shared-shredding MAP storage layout.
 class MapSharedShreddingUtils {
@@ -57,13 +56,23 @@ class MapSharedShreddingUtils {
     /// @return true if the type is MAP.
     static bool IsShreddingKeyMap(const std::shared_ptr& arrow_type);
 
-    /// Creates a MapSharedShreddingContext for the given schema and options.
-    /// Returns nullptr if no shredding MAP columns are detected.
+    /// Finds all shredding MAP field names in a schema by checking per-column config
+    /// via CoreOptions.
     /// @param schema The logical Arrow schema.
     /// @param options CoreOptions containing per-column configuration.
-    /// @return Shared context, or nullptr if no shredding columns.
-    static Result> CreateShreddingContext(
+    /// @return Vector of field names whose map.storage-layout is "shared-shredding", or error
+    ///         if validation fails.
+    static Result> DetectShreddingColumns(
         const std::shared_ptr& schema, const CoreOptions& options);
+
+    /// Builds shared-shredding max column counts from DetectShreddingColumns result and
+    /// CoreOptions.
+    /// @param shredding_field_names Field names returned by DetectShreddingColumns.
+    /// @param options CoreOptions containing per-column shared-shredding config.
+    /// @return Map from field name to its configured maximum physical width.
+    static Result> BuildColumnToNumColumns(
+        const std::vector& shredding_field_names, const CoreOptions& options);
+
     // ---- Schema conversion ----
 
     /// Converts a logical schema to a physical schema by replacing shredding MAP columns
@@ -97,10 +106,9 @@ class MapSharedShreddingUtils {
 
     /// Deserializes shredding metadata from file footer KeyValueMetadata (per field).
     /// @param metadata The KeyValueMetadata from file footer.
-    /// @param compression Compression codec name.
     /// @return Parsed MapSharedShreddingFieldMeta, or error if metadata is missing/malformed.
     static Result DeserializeMetadata(
-        const std::shared_ptr& metadata, const std::string& compression);
+        const std::shared_ptr& metadata);
 
     /// Checks whether a KeyValueMetadata contains shredding MAP metadata.
     static bool HasShreddingMetadata(const std::shared_ptr& metadata);
@@ -112,18 +120,15 @@ class MapSharedShreddingUtils {
     // ---- Writer helpers ----
 
     /// Builds a MetadataFinalizer that serializes shredding metadata into per-field
-    /// KeyValueMetadata and reports file stats back to context for K adaptation.
+    /// KeyValueMetadata.
     /// Shared by DataFileWriter (append-only) and KeyValueDataFileWriter (PK table).
     /// @param converter The batch converter that holds field-dict state for BuildFieldMeta.
     /// @param compression Compression codec name for field_dict serialization (e.g. "zstd").
-    /// @param context The cross-file shared context for K adaptation.
     /// @param physical_schema The physical schema used for writing.
-    /// @return A callable that produces the updated schema with shredding metadata
-    ///         and reports file stats to context.
+    /// @return A callable that produces the updated schema with shredding metadata.
     static std::function>()> BuildMetadataFinalizer(
         const std::shared_ptr& converter,
-        const std::string& compression, const std::shared_ptr& context,
-        const std::shared_ptr& physical_schema);
+        const std::string& compression, const std::shared_ptr& physical_schema);
 
  private:
     /// Returns the physical column indices for the given field name from the shredding meta.
@@ -134,23 +139,6 @@ class MapSharedShreddingUtils {
     static Result> GetPhysicalColumnIndices(
         const MapSharedShreddingFieldMeta& meta, const std::string& name);
 
-    /// Finds all shredding MAP field names in a schema by checking per-column config
-    /// via CoreOptions.
-    /// @param schema The logical Arrow schema.
-    /// @param options CoreOptions containing per-column configuration.
-    /// @return Vector of field names whose map.storage-layout is "shared-shredding", or error
-    ///         if validation fails.
-    static Result> DetectShreddingColumns(
-        const std::shared_ptr& schema, const CoreOptions& options);
-
-    /// Builds shared-shredding max column counts from DetectShreddingColumns result and
-    /// CoreOptions.
-    /// @param shredding_field_names Field names returned by DetectShreddingColumns.
-    /// @param options CoreOptions containing per-column shared-shredding config.
-    /// @return Map from field name to its configured maximum physical width.
-    static Result> BuildColumnToNumColumns(
-        const std::vector& shredding_field_names, const CoreOptions& options);
-
     /// Builds the physical Arrow type for one shredding MAP column.
     /// @param value_type The value type of the original MAP.
     /// @param num_columns Number of physical columns K.
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp
index fe85708e..85467e48 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_utils_test.cpp
@@ -279,8 +279,7 @@ TEST(MapSharedShreddingUtilsTest, MetadataRoundtripNoneCompression) {
     ASSERT_EQ(find_value(MapSharedShreddingDefine::kOverflowSet), "[1,5]");
 
     // Roundtrip verify
-    ASSERT_OK_AND_ASSIGN(auto deserialized,
-                         MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"));
+    ASSERT_OK_AND_ASSIGN(auto deserialized, MapSharedShreddingUtils::DeserializeMetadata(metadata));
     ASSERT_EQ(deserialized, original);
 }
 
@@ -297,7 +296,7 @@ TEST(MapSharedShreddingUtilsTest, MetadataRoundtripCompression) {
         ASSERT_OK(
             MapSharedShreddingUtils::SerializeMetadata(original, compression, metadata.get()));
         ASSERT_OK_AND_ASSIGN(auto deserialized,
-                             MapSharedShreddingUtils::DeserializeMetadata(metadata, compression));
+                             MapSharedShreddingUtils::DeserializeMetadata(metadata));
         ASSERT_EQ(deserialized, original);
     };
 
@@ -314,7 +313,7 @@ TEST(MapSharedShreddingUtilsTest, MetadataRoundtripEmptyData) {
         ASSERT_OK(
             MapSharedShreddingUtils::SerializeMetadata(original, compression, metadata.get()));
         ASSERT_OK_AND_ASSIGN(auto deserialized,
-                             MapSharedShreddingUtils::DeserializeMetadata(metadata, compression));
+                             MapSharedShreddingUtils::DeserializeMetadata(metadata));
         ASSERT_EQ(deserialized, original);
     };
 
@@ -323,33 +322,47 @@ TEST(MapSharedShreddingUtilsTest, MetadataRoundtripEmptyData) {
     verify_roundtrip("zstd");
 }
 
+TEST(MapSharedShreddingUtilsTest, DeserializeLegacyMetadataWithoutCompression) {
+    MapSharedShreddingFieldMeta original;
+    original.name_to_id = {{"alpha", 0}, {"beta", 1}};
+    original.field_to_columns = {{0, {0}}, {1, {1}}};
+    original.overflow_field_set = {1};
+    original.num_columns = 2;
+    original.max_row_width = 2;
+
+    auto metadata = std::make_shared();
+    ASSERT_OK(MapSharedShreddingUtils::SerializeMetadata(
+        original, MapSharedShreddingDefine::kDefaultDictCompression, metadata.get()));
+    ASSERT_TRUE(metadata->Delete(MapSharedShreddingDefine::kFieldDictCompression).ok());
+
+    ASSERT_OK_AND_ASSIGN(auto deserialized, MapSharedShreddingUtils::DeserializeMetadata(metadata));
+    ASSERT_EQ(deserialized, original);
+}
+
 // ---- DeserializeMetadata error cases ----
 
 TEST(MapSharedShreddingUtilsTest, DeserializeMetadataErrors) {
     const std::string layout_error = "metadata is null or storage layout is not shared-shredding";
     // nullptr
-    ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(nullptr, "none"),
-                        layout_error);
+    ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(nullptr), layout_error);
     // missing storage layout
     {
         auto metadata = std::make_shared();
         metadata->Append("some_key", "some_value");
-        ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"),
-                            layout_error);
+        ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata), layout_error);
     }
     // wrong storage layout
     {
         auto metadata = std::make_shared();
         metadata->Append(MapShreddingDefine::kStorageLayout, "default");
-        ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"),
-                            layout_error);
+        ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata), layout_error);
     }
     // missing version
     {
         auto metadata = std::make_shared();
         metadata->Append(MapShreddingDefine::kStorageLayout,
                          MapShreddingDefine::kStorageLayoutSharedShredding);
-        ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"),
+        ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata),
                             "missing shredding metadata key: paimon.map.shared-shredding.version");
     }
     // wrong version
@@ -360,7 +373,7 @@ TEST(MapSharedShreddingUtilsTest, DeserializeMetadataErrors) {
         metadata->Append(MapSharedShreddingDefine::kVersion, "999");
         metadata->Append(MapSharedShreddingDefine::kFieldDictOriginalSize, "2");
         metadata->Append(MapSharedShreddingDefine::kFieldDict, "{}");
-        ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"),
+        ASSERT_NOK_WITH_MSG(MapSharedShreddingUtils::DeserializeMetadata(metadata),
                             "unsupported shared-shredding metadata version: 999");
     }
     // missing field_dict
@@ -371,7 +384,7 @@ TEST(MapSharedShreddingUtilsTest, DeserializeMetadataErrors) {
         metadata->Append(MapSharedShreddingDefine::kVersion, "1");
         metadata->Append(MapSharedShreddingDefine::kFieldDictOriginalSize, "2");
         ASSERT_NOK_WITH_MSG(
-            MapSharedShreddingUtils::DeserializeMetadata(metadata, "none"),
+            MapSharedShreddingUtils::DeserializeMetadata(metadata),
             "missing shredding metadata key: paimon.map.shared-shredding.field-dict");
     }
 }
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp
index 3ee94e0f..04c821a7 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp
+++ b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.cpp
@@ -28,14 +28,31 @@
 
 namespace paimon {
 
+Result>
+MapSharedShreddingWritePlanFactory::Create(const CoreOptions& options,
+                                           const std::shared_ptr& write_schema,
+                                           const std::shared_ptr& pool) {
+    PAIMON_ASSIGN_OR_RAISE(std::vector shredding_fields,
+                           MapSharedShreddingUtils::DetectShreddingColumns(write_schema, options));
+    std::map field_to_max_columns;
+    PAIMON_ASSIGN_OR_RAISE(field_to_max_columns, MapSharedShreddingUtils::BuildColumnToNumColumns(
+                                                     shredding_fields, options));
+    return std::shared_ptr(
+        new MapSharedShreddingWritePlanFactory(options, write_schema, field_to_max_columns, pool));
+}
+
 MapSharedShreddingWritePlanFactory::MapSharedShreddingWritePlanFactory(
     const CoreOptions& options, const std::shared_ptr& write_schema,
-    const std::shared_ptr& context,
+    const std::map& field_to_max_columns,
     const std::shared_ptr& pool)
-    : options_(options), write_schema_(write_schema), context_(context), pool_(pool) {}
+    : options_(options),
+      write_schema_(write_schema),
+      field_to_max_columns_(field_to_max_columns),
+      context_(std::make_shared(field_to_max_columns)),
+      pool_(pool) {}
 
 bool MapSharedShreddingWritePlanFactory::ShouldCreateWritePlan() const {
-    return context_ != nullptr;
+    return !field_to_max_columns_.empty();
 }
 
 bool MapSharedShreddingWritePlanFactory::ShouldInferWritePlan() const {
@@ -49,9 +66,9 @@ int32_t MapSharedShreddingWritePlanFactory::InferBufferRowCount() const {
 Result>
 MapSharedShreddingWritePlanFactory::CreateConverter(
     const std::string& file_format_identifier,
-    const std::vector>& sample_batches) const {
-    if (context_ == nullptr) {
-        return Status::Invalid("Shared-shredding write plan requires a shredding context.");
+    const std::vector>& sample_batches) {
+    if (!ShouldCreateWritePlan()) {
+        return Status::Invalid("MAP shared-shredding write plan is not active.");
     }
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr converter,
@@ -61,12 +78,29 @@ MapSharedShreddingWritePlanFactory::CreateConverter(
 
 ShreddingWritePlanFactory::MetadataFinalizer
 MapSharedShreddingWritePlanFactory::CreateMetadataFinalizer(
-    const std::shared_ptr& converter) const {
+    const std::shared_ptr& converter,
+    const std::string& compression) const {
     // The converter is created by CreateConverter above; the concrete type is guaranteed.
     auto map_converter = std::static_pointer_cast(converter);
-    return MapSharedShreddingUtils::BuildMetadataFinalizer(
-        map_converter, MapSharedShreddingDefine::kDefaultDictCompression, context_,
-        map_converter->GetPhysicalSchema());
+    return MapSharedShreddingUtils::BuildMetadataFinalizer(map_converter, compression,
+                                                           map_converter->GetPhysicalSchema());
+}
+
+Status MapSharedShreddingWritePlanFactory::OnFileCompleted(
+    const std::shared_ptr& converter) {
+    auto map_converter = std::dynamic_pointer_cast(converter);
+    if (map_converter == nullptr) {
+        return Status::Invalid("Unexpected converter for MAP shared-shredding.");
+    }
+    std::vector> completed_stats;
+    for (const std::string& field_name : map_converter->GetShreddingColumnNames()) {
+        PAIMON_ASSIGN_OR_RAISE(int32_t max_row_width, map_converter->GetMaxRowWidth(field_name));
+        completed_stats.emplace_back(field_name, max_row_width);
+    }
+    for (const auto& [field_name, max_row_width] : completed_stats) {
+        context_->ReportFileStats(field_name, max_row_width);
+    }
+    return Status::OK();
 }
 
 }  // namespace paimon
diff --git a/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h
index f4911fa8..cb7afb1c 100644
--- a/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h
+++ b/src/paimon/common/data/shredding/map_shared_shredding_write_plan_factory.h
@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include 
 #include 
 #include 
 #include 
@@ -37,15 +38,14 @@ namespace paimon {
 
 class MapSharedShreddingContext;
 
-/// Creates MAP shared-shredding batch converters driven by the cross-file adaptive-K context.
+/// Detects configured MAP shared-shredding fields and owns their cross-file adaptive-K context.
 /// The write plan is never inferred from samples; per-file field metadata is persisted into the
 /// file footer by the metadata finalizer.
 class MapSharedShreddingWritePlanFactory : public ShreddingWritePlanFactory {
  public:
-    MapSharedShreddingWritePlanFactory(const CoreOptions& options,
-                                       const std::shared_ptr& write_schema,
-                                       const std::shared_ptr& context,
-                                       const std::shared_ptr& pool);
+    static Result> Create(
+        const CoreOptions& options, const std::shared_ptr& write_schema,
+        const std::shared_ptr& pool);
 
     bool ShouldCreateWritePlan() const override;
 
@@ -55,14 +55,23 @@ class MapSharedShreddingWritePlanFactory : public ShreddingWritePlanFactory {
 
     Result> CreateConverter(
         const std::string& file_format_identifier,
-        const std::vector>& sample_batches) const override;
+        const std::vector>& sample_batches) override;
 
     MetadataFinalizer CreateMetadataFinalizer(
-        const std::shared_ptr& converter) const override;
+        const std::shared_ptr& converter,
+        const std::string& compression) const override;
+
+    Status OnFileCompleted(const std::shared_ptr& converter) override;
 
  private:
+    MapSharedShreddingWritePlanFactory(const CoreOptions& options,
+                                       const std::shared_ptr& write_schema,
+                                       const std::map& field_to_max_columns,
+                                       const std::shared_ptr& pool);
+
     CoreOptions options_;
     std::shared_ptr write_schema_;
+    std::map field_to_max_columns_;
     std::shared_ptr context_;
     std::shared_ptr pool_;
 };
diff --git a/src/paimon/common/data/shredding/map_shredding_defs.h b/src/paimon/common/data/shredding/map_shredding_defs.h
index 1ada7a7c..44348b8f 100644
--- a/src/paimon/common/data/shredding/map_shredding_defs.h
+++ b/src/paimon/common/data/shredding/map_shredding_defs.h
@@ -45,6 +45,9 @@ struct MapSharedShreddingDefine {
     static constexpr int32_t kCurrentVersion = 1;
     /// JSON-encoded field name <-> field id dictionary (may be compressed).
     static constexpr const char* kFieldDict = "paimon.map.shared-shredding.field-dict";
+    /// Compression codec used by field_dict. Missing in legacy files, which default to zstd.
+    static constexpr const char* kFieldDictCompression =
+        "paimon.map.shared-shredding.field-dict-compression";
     /// Original (uncompressed) size of field_dict value.
     static constexpr const char* kFieldDictOriginalSize =
         "paimon.map.shared-shredding.field-dict-original-size";
diff --git a/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp b/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp
index b7992871..ace79875 100644
--- a/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp
+++ b/src/paimon/common/data/shredding/shredding_write_plan_factories.cpp
@@ -25,24 +25,27 @@
 
 namespace paimon {
 
-std::shared_ptr ShreddingWritePlanFactories::SelectActive(
+Result> ShreddingWritePlanFactories::SelectActive(
     const CoreOptions& options, const std::shared_ptr& write_schema,
-    const std::shared_ptr& shredding_context,
     const std::shared_ptr& pool) {
-    // MAP shared-shredding is active exactly when a context exists; constructing its factory
-    // copies the options, so skip it otherwise.
-    if (shredding_context != nullptr) {
-        auto map_factory = std::make_shared(
-            options, write_schema, shredding_context, pool);
-        if (map_factory->ShouldCreateWritePlan()) {
-            return map_factory;
-        }
-    }
+    std::shared_ptr active_factory;
+
     auto variant_factory = VariantShreddingWritePlanFactory::Create(options, write_schema, pool);
     if (variant_factory->ShouldCreateWritePlan()) {
-        return variant_factory;
+        active_factory = std::move(variant_factory);
     }
-    return std::shared_ptr(nullptr);
+
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr map_factory,
+                           MapSharedShreddingWritePlanFactory::Create(options, write_schema, pool));
+    if (map_factory->ShouldCreateWritePlan()) {
+        if (active_factory != nullptr) {
+            return Status::NotImplemented(
+                "Composing multiple active shredding write plans is not supported.");
+        }
+        active_factory = std::move(map_factory);
+    }
+
+    return active_factory;
 }
 
 }  // namespace paimon
diff --git a/src/paimon/common/data/shredding/shredding_write_plan_factories.h b/src/paimon/common/data/shredding/shredding_write_plan_factories.h
index cfad5f97..c5cfb495 100644
--- a/src/paimon/common/data/shredding/shredding_write_plan_factories.h
+++ b/src/paimon/common/data/shredding/shredding_write_plan_factories.h
@@ -32,18 +32,14 @@ class Schema;
 namespace paimon {
 
 class CoreOptions;
-class MapSharedShreddingContext;
-
 /// Composes the known shredding write-plan factories (MAP shared-shredding and VARIANT
 /// shredding) and selects the one active for a write schema.
 class ShreddingWritePlanFactories {
  public:
     /// Returns the single active write-plan factory for the write, or nullptr when no shredding
-    /// applies. MAP shared-shredding takes precedence over VARIANT shredding, preserving the
-    /// selection order of the writer call sites.
-    static std::shared_ptr SelectActive(
+    /// applies. Each concrete factory detects whether it is active and owns its internal state.
+    static Result> SelectActive(
         const CoreOptions& options, const std::shared_ptr& write_schema,
-        const std::shared_ptr& shredding_context,
         const std::shared_ptr& pool);
 };
 
diff --git a/src/paimon/common/data/shredding/shredding_write_plan_factory.h b/src/paimon/common/data/shredding/shredding_write_plan_factory.h
index 601e5114..ed05a209 100644
--- a/src/paimon/common/data/shredding/shredding_write_plan_factory.h
+++ b/src/paimon/common/data/shredding/shredding_write_plan_factory.h
@@ -59,12 +59,17 @@ class ShreddingWritePlanFactory {
     /// no conversion is useful for this file (the file is written with the logical schema).
     virtual Result> CreateConverter(
         const std::string& file_format_identifier,
-        const std::vector>& sample_batches) const = 0;
+        const std::vector>& sample_batches) = 0;
 
     /// The per-file metadata finalizer persisted into the file footer, or nullptr when the
     /// physical schema is self-describing (as it is for VARIANT shredding).
     virtual MetadataFinalizer CreateMetadataFinalizer(
-        const std::shared_ptr& converter) const = 0;
+        const std::shared_ptr& converter,
+        const std::string& compression) const = 0;
+
+    /// Advances rolling-writer-scoped state after the file and its output stream have been
+    /// closed successfully. Failed files must never affect the next file's write plan.
+    virtual Status OnFileCompleted(const std::shared_ptr& converter) = 0;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp b/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp
index 4f4bc967..c1aee113 100644
--- a/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp
+++ b/src/paimon/common/data/variant/infer_variant_shredding_schema.cpp
@@ -19,28 +19,32 @@
 
 #include "paimon/common/data/variant/infer_variant_shredding_schema.h"
 
+#include 
 #include 
+#include 
 #include 
+#include 
 #include 
 
 #include "arrow/api.h"
+#include "arrow/util/checked_cast.h"
 #include "paimon/common/data/variant/variant_binary_util.h"
 #include "paimon/common/data/variant/variant_defs.h"
+#include "paimon/common/data/variant/variant_shredding_write_plan.h"
+#include "paimon/common/data/variant/variant_type_utils.h"
 
 namespace paimon {
 
-namespace {
-
 constexpr int32_t kMaxRowFieldSize = 1000;
 
 // The inference type lattice. A scalar node holds an arrow type (`arrow::null()` is the untyped
 // VARIANT sentinel); object nodes track per-field occurrence counts so that rare fields can be
 // dropped in the final schema.
-struct SimpleSchema {
+struct InferVariantShreddingSchema::SimpleSchema {
     struct Field {
         std::string name;
         std::shared_ptr schema;
-        int64_t count;
+        double count;
     };
 
     bool is_object = false;
@@ -62,6 +66,10 @@ struct SimpleSchema {
     }
 };
 
+namespace {
+
+using SimpleSchema = InferVariantShreddingSchema::SimpleSchema;
+
 std::shared_ptr MergeSchema(const std::shared_ptr& s1,
                                           const std::shared_ptr& s2);
 
@@ -185,9 +193,6 @@ Result> SchemaOf(const GenericVariant& variant, in
                 }
                 PAIMON_ASSIGN_OR_RAISE(std::shared_ptr field_schema,
                                        SchemaOf(*field->value, max_depth - 1));
-                if (field_schema == nullptr) {
-                    field_schema = SimpleSchema::Variant();
-                }
                 result->fields.push_back(SimpleSchema::Field{field->key, field_schema, 1});
             }
             // According to the variant spec, object fields must be sorted alphabetically.
@@ -275,7 +280,7 @@ Result> SchemaOf(const GenericVariant& variant, in
 // Finalizes the inferred schema: 1) widen integer types to int64, 2) replace empty objects with
 // VARIANT, 3) limit the total number of shredded fields in the schema.
 std::shared_ptr FinalizeSimpleSchema(
-    const std::shared_ptr& schema, int64_t min_cardinality,
+    const std::shared_ptr& schema, double min_cardinality,
     InferVariantShreddingSchema::MaxFields* max_fields) {
     // Every field uses a value column.
     --max_fields->remaining;
@@ -328,32 +333,446 @@ std::shared_ptr FinalizeSimpleSchema(
     }
 }
 
+std::shared_ptr ScaleFieldCounts(const std::shared_ptr& schema,
+                                               double scale) {
+    if (schema == nullptr) {
+        return nullptr;
+    }
+    auto result = std::make_shared(*schema);
+    if (schema->is_object) {
+        result->fields.clear();
+        result->fields.reserve(schema->fields.size());
+        for (const auto& field : schema->fields) {
+            result->fields.push_back(SimpleSchema::Field{
+                field.name, ScaleFieldCounts(field.schema, scale), field.count * scale});
+        }
+    } else if (schema->is_array) {
+        result->element = ScaleFieldCounts(schema->element, scale);
+    }
+    return result;
+}
+
+InferVariantShreddingSchema::ColumnEvidence ScaleToAtMost(
+    const InferVariantShreddingSchema::ColumnEvidence& evidence, int32_t max_root_value_count) {
+    if (evidence.root_value_count <= 0 ||
+        evidence.root_value_count <= static_cast(max_root_value_count)) {
+        return evidence;
+    }
+    double scale = static_cast(max_root_value_count) / evidence.root_value_count;
+    return InferVariantShreddingSchema::ColumnEvidence{
+        static_cast(max_root_value_count),
+        ScaleFieldCounts(evidence.observed_schema, scale)};
+}
+
+const SimpleSchema::Field* FindSimpleField(const std::shared_ptr& schema,
+                                           const std::string& name) {
+    if (schema == nullptr || !schema->is_object) {
+        return nullptr;
+    }
+    auto it = std::lower_bound(schema->fields.begin(), schema->fields.end(), name,
+                               [](const SimpleSchema::Field& field, const std::string& target) {
+                                   return field.name < target;
+                               });
+    return it != schema->fields.end() && it->name == name ? &*it : nullptr;
+}
+
+std::shared_ptr FindArrowField(const std::shared_ptr& type,
+                                             const std::string& name) {
+    if (type == nullptr || type->id() != arrow::Type::STRUCT) {
+        return nullptr;
+    }
+    return std::static_pointer_cast(type)->GetFieldByName(name);
+}
+
+bool IsUntyped(const std::shared_ptr& schema) {
+    return schema == nullptr ||
+           (schema->scalar != nullptr && schema->scalar->id() == arrow::Type::NA);
+}
+
+std::shared_ptr MergeArrowScalars(const std::shared_ptr& first,
+                                                   const std::shared_ptr& second) {
+    if (first == nullptr) {
+        return second;
+    }
+    if (second == nullptr) {
+        return first;
+    }
+    bool first_decimal = first->id() == arrow::Type::DECIMAL128;
+    bool second_decimal = second->id() == arrow::Type::DECIMAL128;
+    if (first_decimal && second_decimal) {
+        auto merged = MergeDecimal(static_cast(*first),
+                                   static_cast(*second));
+        return merged->scalar;
+    }
+    if (first_decimal && second->id() == arrow::Type::INT64) {
+        return MergeDecimalWithLong(static_cast(*first))->scalar;
+    }
+    if (first->id() == arrow::Type::INT64 && second_decimal) {
+        return MergeDecimalWithLong(static_cast(*second))->scalar;
+    }
+    return first->Equals(*second) ? first : arrow::null();
+}
+
+std::shared_ptr WidenScalar(const std::shared_ptr& type) {
+    if (type == nullptr) {
+        return arrow::null();
+    }
+    if (type->id() == arrow::Type::DECIMAL128) {
+        const auto& decimal = static_cast(*type);
+        if (decimal.precision() <= 18 && decimal.scale() == 0) {
+            return arrow::int64();
+        }
+        return arrow::decimal128(
+            decimal.precision() <= 18 ? 18 : VariantDefs::kMaxDecimal16Precision, decimal.scale());
+    }
+    return type;
+}
+
+bool CompatibleTypeFamilies(const std::shared_ptr& previous,
+                            const std::shared_ptr& current) {
+    if (previous == nullptr || current == nullptr) {
+        return true;
+    }
+    if (previous->id() == arrow::Type::STRUCT || current->is_object) {
+        return previous->id() == arrow::Type::STRUCT && current->is_object;
+    }
+    if (previous->id() == arrow::Type::LIST || current->is_array) {
+        return previous->id() == arrow::Type::LIST && current->is_array;
+    }
+    if (current->scalar == nullptr) {
+        return false;
+    }
+    return MergeArrowScalars(previous, current->scalar)->id() != arrow::Type::NA;
+}
+
+std::shared_ptr SelectedSchemaToSimpleSchema(
+    const std::shared_ptr& selected, double field_count) {
+    if (selected == nullptr || selected->id() == arrow::Type::NA) {
+        return SimpleSchema::Variant();
+    }
+    if (selected->id() == arrow::Type::STRUCT) {
+        auto result = std::make_shared();
+        result->is_object = true;
+        for (const std::shared_ptr& field : selected->fields()) {
+            result->fields.push_back(SimpleSchema::Field{
+                field->name(), SelectedSchemaToSimpleSchema(field->type(), field_count),
+                field_count});
+        }
+        return result;
+    }
+    if (selected->id() == arrow::Type::LIST) {
+        auto result = std::make_shared();
+        result->is_array = true;
+        result->element = SelectedSchemaToSimpleSchema(
+            std::static_pointer_cast(selected)->value_type(), field_count);
+        return result;
+    }
+    return SimpleSchema::Scalar(selected);
+}
+
+std::shared_ptr FinalizeAdaptiveSchema(
+    std::shared_ptr combined, const std::shared_ptr& current,
+    std::shared_ptr previous_selected, double root_value_count,
+    double admission_ratio, double retention_ratio,
+    InferVariantShreddingSchema::MaxFields* max_fields) {
+    --max_fields->remaining;
+    if (max_fields->remaining <= 0) {
+        return arrow::null();
+    }
+
+    if (current != nullptr && previous_selected != nullptr &&
+        !CompatibleTypeFamilies(previous_selected, current)) {
+        combined = current;
+        previous_selected = nullptr;
+    }
+    if (IsUntyped(combined)) {
+        if (!IsUntyped(current)) {
+            combined = current;
+        } else if (previous_selected != nullptr) {
+            // Match Java's `combined = previousSelected`: retain the previous selection as the
+            // input to the remaining recursive width-budget and field-ordering logic.
+            combined = SelectedSchemaToSimpleSchema(previous_selected, root_value_count);
+        } else {
+            return arrow::null();
+        }
+    }
+
+    if (combined->is_object) {
+        struct Candidate {
+            const SimpleSchema::Field* field;
+            double ratio;
+            bool is_new;
+        };
+        std::vector candidates;
+        for (const auto& field : combined->fields) {
+            bool is_new = FindArrowField(previous_selected, field.name) == nullptr;
+            double threshold = is_new ? admission_ratio : retention_ratio;
+            double ratio = root_value_count == 0 ? 0 : field.count / root_value_count;
+            if (ratio >= threshold) {
+                candidates.push_back(Candidate{&field, ratio, is_new});
+            }
+        }
+        std::sort(candidates.begin(), candidates.end(),
+                  [](const Candidate& left, const Candidate& right) {
+                      if (left.ratio != right.ratio) {
+                          return left.ratio > right.ratio;
+                      }
+                      if (left.is_new != right.is_new) {
+                          return !left.is_new;
+                      }
+                      return left.field->name < right.field->name;
+                  });
+
+        arrow::FieldVector selected;
+        for (const Candidate& candidate : candidates) {
+            if (max_fields->remaining <= 0) {
+                break;
+            }
+            const SimpleSchema::Field* current_field =
+                FindSimpleField(current, candidate.field->name);
+            std::shared_ptr previous_field =
+                FindArrowField(previous_selected, candidate.field->name);
+            std::shared_ptr selected_type = FinalizeAdaptiveSchema(
+                candidate.field->schema, current_field == nullptr ? nullptr : current_field->schema,
+                previous_field == nullptr ? nullptr : previous_field->type(), root_value_count,
+                admission_ratio, retention_ratio, max_fields);
+            selected.push_back(arrow::field(candidate.field->name, selected_type));
+        }
+        std::sort(selected.begin(), selected.end(),
+                  [](const std::shared_ptr& left,
+                     const std::shared_ptr& right) {
+                      return left->name() < right->name();
+                  });
+        return selected.empty() ? arrow::null() : arrow::struct_(selected);
+    }
+
+    if (combined->is_array) {
+        std::shared_ptr current_element =
+            current != nullptr && current->is_array ? current->element : nullptr;
+        std::shared_ptr previous_element;
+        if (previous_selected != nullptr && previous_selected->id() == arrow::Type::LIST) {
+            previous_element =
+                std::static_pointer_cast(previous_selected)->value_type();
+        }
+        return arrow::list(FinalizeAdaptiveSchema(combined->element, current_element,
+                                                  previous_element, root_value_count,
+                                                  admission_ratio, retention_ratio, max_fields));
+    }
+
+    --max_fields->remaining;
+    std::shared_ptr current_scalar =
+        current == nullptr ? nullptr : current->scalar;
+    if (current_scalar == nullptr) {
+        return previous_selected == nullptr ? WidenScalar(combined->scalar) : previous_selected;
+    }
+    if (previous_selected == nullptr) {
+        return WidenScalar(current_scalar);
+    }
+    std::shared_ptr merged = MergeArrowScalars(previous_selected, current_scalar);
+    return merged->id() == arrow::Type::NA ? WidenScalar(current_scalar) : merged;
+}
+
 }  // namespace
 
+InferVariantShreddingSchema::InferVariantShreddingSchema(
+    const std::shared_ptr& logical_schema, const std::shared_ptr& pool,
+    int32_t max_schema_width, int32_t max_schema_depth, double min_field_cardinality_ratio)
+    : logical_schema_(logical_schema),
+      pool_(pool),
+      max_schema_width_(max_schema_width),
+      max_schema_depth_(max_schema_depth),
+      min_field_cardinality_ratio_(min_field_cardinality_ratio) {
+    Path current;
+    CollectVariantPaths(logical_schema_->fields(), ¤t, &paths_to_variant_);
+}
+
+void InferVariantShreddingSchema::CollectVariantPaths(
+    const std::vector>& fields, Path* current,
+    std::vector* paths) {
+    for (int32_t i = 0; i < static_cast(fields.size()); ++i) {
+        const std::shared_ptr& field = fields[i];
+        current->push_back(i);
+        if (VariantTypeUtils::IsVariantField(field)) {
+            paths->push_back(*current);
+        } else if (field->type()->id() == arrow::Type::STRUCT) {
+            CollectVariantPaths(field->type()->fields(), current, paths);
+        }
+        current->pop_back();
+    }
+}
+
+Result>>
+InferVariantShreddingSchema::CollectSamplesAtPath(const SampleBatches& sample_batches,
+                                                  const Path& path) const {
+    std::vector> samples;
+    for (const std::shared_ptr& sample_batch : sample_batches) {
+        std::shared_ptr column = sample_batch;
+        // Child slots below a null struct row are unspecified in Arrow and must not be decoded.
+        std::vector> ancestors;
+        for (int32_t index : path) {
+            if (column == nullptr || column->type_id() != arrow::Type::STRUCT) {
+                return Status::Invalid("sample batch does not match the variant column path");
+            }
+            if (column != sample_batch) {
+                ancestors.push_back(column);
+            }
+            column = arrow::internal::checked_cast(*column).field(index);
+        }
+        if (column == nullptr) {
+            return Status::Invalid("sample batch misses the planned variant column");
+        }
+        const auto& variant_array =
+            arrow::internal::checked_cast(*column);
+        const auto& value_array =
+            arrow::internal::checked_cast(*variant_array.field(0));
+        const auto& metadata_array =
+            arrow::internal::checked_cast(*variant_array.field(1));
+        for (int64_t row = 0; row < variant_array.length(); ++row) {
+            bool row_is_null = variant_array.IsNull(row);
+            for (const std::shared_ptr& ancestor : ancestors) {
+                row_is_null = row_is_null || ancestor->IsNull(row);
+            }
+            if (row_is_null) {
+                continue;
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                std::shared_ptr variant,
+                GenericVariant::Create(std::string_view(value_array.GetView(row)),
+                                       std::string_view(metadata_array.GetView(row)), pool_));
+            samples.push_back(std::move(variant));
+        }
+    }
+    return samples;
+}
+
+Result> InferVariantShreddingSchema::CreatePhysicalSchema(
+    const SelectedSchemas& selected_schemas) const {
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr plan,
+        VariantShreddingWritePlan::CreateFromPaths(logical_schema_, selected_schemas));
+    return plan->PhysicalSchema();
+}
+
+Result> InferVariantShreddingSchema::InferSchema(
+    const SampleBatches& samples) const {
+    PAIMON_ASSIGN_OR_RAISE(AdaptiveInferenceResult result,
+                           InferInitial(samples, std::numeric_limits::max()));
+    return result.physical_schema;
+}
+
+Result
+InferVariantShreddingSchema::InferInitial(const SampleBatches& samples,
+                                          int32_t effective_sample_size) const {
+    MaxFields max_fields = CreateMaxFieldsBudget();
+    InferenceEvidence evidence;
+    SelectedSchemas selected_schemas;
+    for (const Path& path : paths_to_variant_) {
+        PAIMON_ASSIGN_OR_RAISE(std::vector> column_samples,
+                               CollectSamplesAtPath(samples, path));
+        PAIMON_ASSIGN_OR_RAISE(
+            AdaptiveColumnResult result,
+            InferInitialColumn(column_samples, effective_sample_size, &max_fields));
+        evidence.columns.emplace(path, std::move(result.evidence));
+        selected_schemas.emplace(path, std::move(result.selected_schema));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_schema,
+                           CreatePhysicalSchema(selected_schemas));
+    return AdaptiveInferenceResult{std::move(physical_schema), std::move(evidence),
+                                   std::move(selected_schemas)};
+}
+
+Result
+InferVariantShreddingSchema::InferAdaptive(const InferenceEvidence& previous_evidence,
+                                           const SelectedSchemas& previous_selected_schemas,
+                                           const SampleBatches& samples,
+                                           int32_t effective_sample_size, double admission_ratio,
+                                           double retention_ratio) const {
+    MaxFields max_fields = CreateMaxFieldsBudget();
+    InferenceEvidence evidence;
+    SelectedSchemas selected_schemas;
+    for (const Path& path : paths_to_variant_) {
+        PAIMON_ASSIGN_OR_RAISE(std::vector> column_samples,
+                               CollectSamplesAtPath(samples, path));
+        auto evidence_it = previous_evidence.columns.find(path);
+        ColumnEvidence previous_column;
+        if (evidence_it != previous_evidence.columns.end()) {
+            previous_column = evidence_it->second;
+        }
+        auto selected_it = previous_selected_schemas.find(path);
+        std::shared_ptr previous_selected =
+            selected_it == previous_selected_schemas.end() ? nullptr : selected_it->second;
+        PAIMON_ASSIGN_OR_RAISE(AdaptiveColumnResult result,
+                               InferAdaptiveColumn(previous_column, previous_selected,
+                                                   column_samples, effective_sample_size,
+                                                   admission_ratio, retention_ratio, &max_fields));
+        evidence.columns.emplace(path, std::move(result.evidence));
+        selected_schemas.emplace(path, std::move(result.selected_schema));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr physical_schema,
+                           CreatePhysicalSchema(selected_schemas));
+    return AdaptiveInferenceResult{std::move(physical_schema), std::move(evidence),
+                                   std::move(selected_schemas)};
+}
+
 Result> InferVariantShreddingSchema::InferColumnShreddingType(
     const std::vector>& samples, MaxFields* max_fields) const {
-    int64_t num_non_null_values = 0;
-    std::shared_ptr simple_schema;
+    PAIMON_ASSIGN_OR_RAISE(ColumnEvidence evidence, AnalyzeColumn(samples));
+    double min_cardinality = std::ceil(evidence.root_value_count * min_field_cardinality_ratio_);
+    std::shared_ptr finalized =
+        FinalizeSimpleSchema(evidence.observed_schema, min_cardinality, max_fields);
+    if (finalized->id() == arrow::Type::NA) {
+        return std::shared_ptr(nullptr);
+    }
+    return finalized;
+}
+
+Result InferVariantShreddingSchema::AnalyzeColumn(
+    const std::vector>& samples) const {
+    ColumnEvidence evidence;
     for (const auto& sample : samples) {
         if (sample == nullptr) {
             continue;
         }
-        ++num_non_null_values;
+        ++evidence.root_value_count;
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_schema,
                                SchemaOf(*sample, max_schema_depth_));
-        simple_schema = MergeSchema(simple_schema, row_schema);
+        evidence.observed_schema = MergeSchema(evidence.observed_schema, row_schema);
     }
-    // Don't infer a schema for fields that appear in less than min_field_cardinality_ratio of
-    // the rows.
-    auto min_cardinality = static_cast(
-        std::ceil(static_cast(num_non_null_values) * min_field_cardinality_ratio_));
+    return evidence;
+}
+
+Result
+InferVariantShreddingSchema::InferInitialColumn(
+    const std::vector>& samples, int32_t effective_sample_size,
+    MaxFields* max_fields) const {
+    PAIMON_ASSIGN_OR_RAISE(ColumnEvidence evidence, AnalyzeColumn(samples));
+    double min_cardinality = std::ceil(evidence.root_value_count * min_field_cardinality_ratio_);
     std::shared_ptr finalized =
-        FinalizeSimpleSchema(simple_schema, min_cardinality, max_fields);
-    if (finalized->id() == arrow::Type::NA) {
-        // The whole column stays unshredded.
-        return std::shared_ptr(nullptr);
+        FinalizeSimpleSchema(evidence.observed_schema, min_cardinality, max_fields);
+    return AdaptiveColumnResult{ScaleToAtMost(evidence, effective_sample_size), finalized};
+}
+
+Result
+InferVariantShreddingSchema::InferAdaptiveColumn(
+    const ColumnEvidence& previous_evidence,
+    const std::shared_ptr& previous_selected,
+    const std::vector>& samples, int32_t effective_sample_size,
+    double admission_ratio, double retention_ratio, MaxFields* max_fields) const {
+    PAIMON_ASSIGN_OR_RAISE(ColumnEvidence current, AnalyzeColumn(samples));
+    ColumnEvidence combined;
+    if (current.root_value_count == 0) {
+        combined = previous_evidence;
+    } else {
+        ColumnEvidence bounded_previous = ScaleToAtMost(previous_evidence, effective_sample_size);
+        combined.root_value_count = current.root_value_count + bounded_previous.root_value_count;
+        combined.observed_schema =
+            MergeSchema(bounded_previous.observed_schema, current.observed_schema);
+        combined = ScaleToAtMost(combined, effective_sample_size);
     }
-    return finalized;
+    std::shared_ptr selected = FinalizeAdaptiveSchema(
+        combined.observed_schema, current.observed_schema, previous_selected,
+        combined.root_value_count, admission_ratio, retention_ratio, max_fields);
+    return AdaptiveColumnResult{std::move(combined), std::move(selected)};
 }
 
 }  // namespace paimon
diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema.h b/src/paimon/common/data/variant/infer_variant_shredding_schema.h
index 7b8d3d1a..8d8a45bc 100644
--- a/src/paimon/common/data/variant/infer_variant_shredding_schema.h
+++ b/src/paimon/common/data/variant/infer_variant_shredding_schema.h
@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include 
+#include 
 #include 
 #include 
 
@@ -26,17 +28,39 @@
 #include "paimon/result.h"
 
 namespace arrow {
+class Array;
 class DataType;
+class Field;
+class Schema;
 }  // namespace arrow
 
 namespace paimon {
 
-/// Infers a shredding type for a variant column from sampled values (mirroring the Java
+class MemoryPool;
+class VariantShreddingInferenceSession;
+
+/// Infers the complete physical write schema from sampled logical rows (mirroring the Java
 /// `InferVariantShreddingSchema`). Rare fields (below the cardinality ratio) stay in the
 /// un-shredded variant binary, integer types widen to int64, and the total number of shredded
-/// fields is limited.
+/// fields is limited across all Variant columns.
 class InferVariantShreddingSchema {
  public:
+    using Path = std::vector;
+    using SampleBatches = std::vector>;
+
+    struct SimpleSchema;
+
+    struct ColumnEvidence {
+        double root_value_count = 0;
+        std::shared_ptr observed_schema;
+    };
+
+    struct AdaptiveColumnResult {
+        ColumnEvidence evidence;
+        /// arrow::null() means the column remains unshredded.
+        std::shared_ptr selected_schema;
+    };
+
     /// The mutable budget of shredded fields remaining. One instance is shared across all
     /// variant columns of a schema so that the total inferred width stays within
     /// `variant.shredding.maxSchemaWidth` (mirroring the Java `MaxFields`).
@@ -44,11 +68,12 @@ class InferVariantShreddingSchema {
         int32_t remaining;
     };
 
-    InferVariantShreddingSchema(int32_t max_schema_width, int32_t max_schema_depth,
-                                double min_field_cardinality_ratio)
-        : max_schema_width_(max_schema_width),
-          max_schema_depth_(max_schema_depth),
-          min_field_cardinality_ratio_(min_field_cardinality_ratio) {}
+    InferVariantShreddingSchema(const std::shared_ptr& logical_schema,
+                                const std::shared_ptr& pool, int32_t max_schema_width,
+                                int32_t max_schema_depth, double min_field_cardinality_ratio);
+
+    /// Infers one complete physical schema from the sampled logical row batches.
+    Result> InferSchema(const SampleBatches& samples) const;
 
     /// Creates the shared shredded-field budget for one schema inference.
     MaxFields CreateMaxFieldsBudget() const {
@@ -62,7 +87,57 @@ class InferVariantShreddingSchema {
     Result> InferColumnShreddingType(
         const std::vector>& samples, MaxFields* max_fields) const;
 
+    /// Initial/adaptive inference primitives used by a rolling-writer-scoped session.
+    Result InferInitialColumn(
+        const std::vector>& samples, int32_t effective_sample_size,
+        MaxFields* max_fields) const;
+
+    Result InferAdaptiveColumn(
+        const ColumnEvidence& previous_evidence,
+        const std::shared_ptr& previous_selected,
+        const std::vector>& samples, int32_t effective_sample_size,
+        double admission_ratio, double retention_ratio, MaxFields* max_fields) const;
+
  private:
+    friend class VariantShreddingInferenceSession;
+
+    struct InferenceEvidence {
+        std::map columns;
+    };
+
+    using SelectedSchemas = std::map>;
+
+    struct AdaptiveInferenceResult {
+        std::shared_ptr physical_schema;
+        InferenceEvidence evidence;
+        SelectedSchemas selected_schemas;
+    };
+
+    Result InferInitial(const SampleBatches& samples,
+                                                 int32_t effective_sample_size) const;
+
+    Result InferAdaptive(const InferenceEvidence& previous_evidence,
+                                                  const SelectedSchemas& previous_selected_schemas,
+                                                  const SampleBatches& samples,
+                                                  int32_t effective_sample_size,
+                                                  double admission_ratio,
+                                                  double retention_ratio) const;
+
+    static void CollectVariantPaths(const std::vector>& fields,
+                                    Path* current, std::vector* paths);
+
+    Result>> CollectSamplesAtPath(
+        const SampleBatches& sample_batches, const Path& path) const;
+
+    Result> CreatePhysicalSchema(
+        const SelectedSchemas& selected_schemas) const;
+
+    Result AnalyzeColumn(
+        const std::vector>& samples) const;
+
+    std::shared_ptr logical_schema_;
+    std::shared_ptr pool_;
+    std::vector paths_to_variant_;
     int32_t max_schema_width_;
     int32_t max_schema_depth_;
     double min_field_cardinality_ratio_;
diff --git a/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp b/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp
index 9a55b866..13e77dbd 100644
--- a/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp
+++ b/src/paimon/common/data/variant/infer_variant_shredding_schema_test.cpp
@@ -70,7 +70,9 @@ class InferVariantShreddingSchemaTest : public ::testing::Test {
 
  protected:
     std::shared_ptr pool_ = GetDefaultPool();
-    InferVariantShreddingSchema infer_{/*max_schema_width=*/300, /*max_schema_depth=*/50,
+    std::shared_ptr empty_logical_schema_ = arrow::schema(arrow::FieldVector{});
+    InferVariantShreddingSchema infer_{empty_logical_schema_, pool_,
+                                       /*max_schema_width=*/300, /*max_schema_depth=*/50,
                                        /*min_field_cardinality_ratio=*/0.1};
 };
 
@@ -91,6 +93,17 @@ TEST_F(InferVariantShreddingSchemaTest, InferObjectSchema) {
     ASSERT_OK(VariantShreddingUtils::VariantShreddingSchema(inferred));
 }
 
+TEST_F(InferVariantShreddingSchemaTest, NullObjectFieldMergesWithTypedValue) {
+    auto samples = Samples({
+        R"({"a": 1, "b": null})",
+        R"({"a": 2, "b": 3})",
+    });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples));
+    auto expected =
+        arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
 TEST_F(InferVariantShreddingSchemaTest, MixedTypesFallToVariant) {
     auto samples = Samples({
         R"({"x": 1, "y": 1.5e0})",
@@ -120,6 +133,40 @@ TEST_F(InferVariantShreddingSchemaTest, RareFieldsDropped) {
     ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
 }
 
+TEST_F(InferVariantShreddingSchemaTest, FieldCardinalityAdmissionThreshold) {
+    std::vector jsons = {
+        R"({"common": 1, "rare": 99})",
+        R"({"common": 2, "rare": 88})",
+        R"({"common": 3})",
+        R"({"common": 4})",
+        R"({"common": 5})",
+        R"({"common": 6})",
+        R"({"common": 7})",
+        R"({"common": 8})",
+        R"({"common": 9})",
+        R"({"common": 10})",
+    };
+    auto samples = Samples(jsons);
+
+    InferVariantShreddingSchema permissive{empty_logical_schema_, pool_,
+                                           /*max_schema_width=*/300,
+                                           /*max_schema_depth=*/50,
+                                           /*min_field_cardinality_ratio=*/0.1};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr admitted,
+                         InferColumn(permissive, samples));
+    auto expected_admitted = arrow::struct_(
+        {arrow::field("common", arrow::int64()), arrow::field("rare", arrow::int64())});
+    ASSERT_TRUE(admitted->Equals(*expected_admitted)) << admitted->ToString();
+
+    InferVariantShreddingSchema strict{empty_logical_schema_, pool_,
+                                       /*max_schema_width=*/300,
+                                       /*max_schema_depth=*/50,
+                                       /*min_field_cardinality_ratio=*/0.25};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr rejected, InferColumn(strict, samples));
+    auto expected_rejected = arrow::struct_({arrow::field("common", arrow::int64())});
+    ASSERT_TRUE(rejected->Equals(*expected_rejected)) << rejected->ToString();
+}
+
 TEST_F(InferVariantShreddingSchemaTest, DecimalMerging) {
     auto samples = Samples({
         "{\"d\": 100.99}",
@@ -133,6 +180,82 @@ TEST_F(InferVariantShreddingSchemaTest, DecimalMerging) {
     ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
 }
 
+TEST_F(InferVariantShreddingSchemaTest, AllPrimitiveTypes) {
+    auto samples = Samples({R"({
+        "string": "test",
+        "long": 123456789,
+        "double": 3.14159,
+        "boolean": true,
+        "null": null
+    })"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples));
+    auto expected = arrow::struct_({
+        arrow::field("boolean", arrow::boolean()),
+        arrow::field("double", arrow::decimal128(18, 5)),
+        arrow::field("long", arrow::int64()),
+        arrow::field("null", arrow::null()),
+        arrow::field("string", arrow::utf8()),
+    });
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, MixedArrayElementTypesFallToVariant) {
+    auto samples = Samples({
+        R"({"arr": [1, 2, 3]})",
+        R"({"arr": ["a", "b", "c"]})",
+        R"({"arr": [true, false, true]})",
+        R"({"arr": [1, "mixed", true]})",
+    });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples));
+    auto expected = arrow::struct_({arrow::field("arr", arrow::list(arrow::null()))});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, NullInNestedArrays) {
+    auto samples = Samples({
+        R"({"arr": [1, 2, 3, null, 5]})",
+        R"({"arr": [null, null, null]})",
+        R"({"arr": [10, null, 20, null, 30]})",
+        R"({"arr": null})",
+    });
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples));
+    auto expected = arrow::struct_({arrow::field("arr", arrow::list(arrow::null()))});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, LargeDatasetWithManyFields) {
+    std::vector documents;
+    documents.reserve(500);
+    for (int32_t row = 0; row < 500; ++row) {
+        std::string json = "{";
+        for (int32_t field = 0; field < 50; ++field) {
+            if (field > 0) {
+                json += ",";
+            }
+            json += "\"field" + std::to_string(field) +
+                    "\":" + std::to_string((row * 50 + field) % 1000);
+        }
+        json += "}";
+        documents.push_back(std::move(json));
+    }
+    std::vector jsons;
+    jsons.reserve(documents.size());
+    for (const std::string& document : documents) {
+        jsons.push_back(document.c_str());
+    }
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred,
+                         InferColumn(infer_, Samples(jsons)));
+    ASSERT_EQ(arrow::Type::STRUCT, inferred->id());
+    const auto& struct_type = static_cast(*inferred);
+    ASSERT_EQ(50, struct_type.num_fields());
+    for (int32_t field = 0; field < 50; ++field) {
+        auto inferred_field = struct_type.GetFieldByName("field" + std::to_string(field));
+        ASSERT_NE(nullptr, inferred_field);
+        ASSERT_TRUE(inferred_field->type()->Equals(*arrow::int64()));
+    }
+}
+
 TEST_F(InferVariantShreddingSchemaTest, NoUsefulSchema) {
     auto scalar_samples = Samples({"1", "2"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr scalar_inferred,
@@ -154,7 +277,8 @@ TEST_F(InferVariantShreddingSchemaTest, NoUsefulSchema) {
 }
 
 TEST_F(InferVariantShreddingSchemaTest, MaxSchemaWidthLimit) {
-    InferVariantShreddingSchema narrow_infer{/*max_schema_width=*/3, /*max_schema_depth=*/50,
+    InferVariantShreddingSchema narrow_infer{empty_logical_schema_, pool_,
+                                             /*max_schema_width=*/3, /*max_schema_depth=*/50,
                                              /*min_field_cardinality_ratio=*/0.1};
     auto samples = Samples({R"({"a": 1, "b": 2, "c": 3, "d": 4, "e": 5})"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred,
@@ -180,7 +304,8 @@ TEST_F(InferVariantShreddingSchemaTest, MaxSchemaWidthLimit) {
 }
 
 TEST_F(InferVariantShreddingSchemaTest, MaxSchemaDepthLimit) {
-    InferVariantShreddingSchema shallow_infer{/*max_schema_width=*/300, /*max_schema_depth=*/1,
+    InferVariantShreddingSchema shallow_infer{empty_logical_schema_, pool_,
+                                              /*max_schema_width=*/300, /*max_schema_depth=*/1,
                                               /*min_field_cardinality_ratio=*/0.1};
     auto samples = Samples({R"({"outer": {"inner": 1}})"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred,
@@ -191,6 +316,43 @@ TEST_F(InferVariantShreddingSchemaTest, MaxSchemaDepthLimit) {
     ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
 }
 
+TEST_F(InferVariantShreddingSchemaTest, DeepNestedObjectSchema) {
+    auto samples = Samples({R"({"level1":{"level2":{"level3":{"value":42}}}})"});
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred, InferColumn(infer_, samples));
+    auto expected = arrow::struct_({arrow::field(
+        "level1",
+        arrow::struct_({arrow::field(
+            "level2",
+            arrow::struct_({arrow::field(
+                "level3", arrow::struct_({arrow::field("value", arrow::int64())}))}))}))});
+    ASSERT_TRUE(inferred->Equals(*expected)) << inferred->ToString();
+}
+
+TEST_F(InferVariantShreddingSchemaTest, AdaptivePreviousSelectionHonorsSharedWidthBudget) {
+    InferVariantShreddingSchema narrow_infer{empty_logical_schema_, pool_,
+                                             /*max_schema_width=*/6, /*max_schema_depth=*/50,
+                                             /*min_field_cardinality_ratio=*/0.1};
+    // A scalar-to-object transition can leave the combined evidence untyped while selecting the
+    // current object schema for writing.
+    InferVariantShreddingSchema::ColumnEvidence previous_evidence;
+    previous_evidence.root_value_count = 1;
+    auto previous_selected =
+        arrow::struct_({arrow::field("a", arrow::int64()), arrow::field("b", arrow::int64())});
+    InferVariantShreddingSchema::MaxFields max_fields = narrow_infer.CreateMaxFieldsBudget();
+    // In the preceding file, an earlier Variant column used one slot, leaving five slots for this
+    // column's root, a and b. In this file that earlier column expanded to use three slots, so only
+    // three remain and the previous selection must be trimmed.
+    max_fields.remaining = 3;
+
+    ASSERT_OK_AND_ASSIGN(
+        InferVariantShreddingSchema::AdaptiveColumnResult result,
+        narrow_infer.InferAdaptiveColumn(
+            previous_evidence, previous_selected, /*samples=*/{}, /*effective_sample_size=*/10,
+            /*admission_ratio=*/0.1, /*retention_ratio=*/0.05, &max_fields));
+    auto expected = arrow::struct_({arrow::field("a", arrow::int64())});
+    ASSERT_TRUE(result.selected_schema->Equals(*expected)) << result.selected_schema->ToString();
+}
+
 TEST_F(InferVariantShreddingSchemaTest, TrailingZeroDecimalNormalized) {
     // GetDecimal strips 100.00 to 1E+2 (scale -2); the inferred type must carry a non-negative
     // scale or reassembling the shredded file would be rejected. After normalization the value
@@ -265,7 +427,8 @@ TEST_F(InferVariantShreddingSchemaTest, ArraysMerge) {
 }
 
 TEST_F(InferVariantShreddingSchemaTest, ArrayBeyondDepthLimitStaysVariant) {
-    InferVariantShreddingSchema shallow_infer{/*max_schema_width=*/300, /*max_schema_depth=*/1,
+    InferVariantShreddingSchema shallow_infer{empty_logical_schema_, pool_,
+                                              /*max_schema_width=*/300, /*max_schema_depth=*/1,
                                               /*min_field_cardinality_ratio=*/0.1};
     auto samples = Samples({R"({"arr": [1, 2]})"});
     ASSERT_OK_AND_ASSIGN(std::shared_ptr inferred,
@@ -360,7 +523,8 @@ TEST_F(InferVariantShreddingSchemaTest, DecimalMergeOverflowFallsToVariant) {
 }
 
 TEST_F(InferVariantShreddingSchemaTest, ObjectWithAllRareFieldsStaysUnshredded) {
-    InferVariantShreddingSchema strict_infer{/*max_schema_width=*/300, /*max_schema_depth=*/50,
+    InferVariantShreddingSchema strict_infer{empty_logical_schema_, pool_,
+                                             /*max_schema_width=*/300, /*max_schema_depth=*/50,
                                              /*min_field_cardinality_ratio=*/0.6};
     // Two objects with disjoint single-occurrence keys: with a 0.6 ratio every field is below the
     // cardinality threshold, so the object contributes no typed field and the column is dropped.
diff --git a/src/paimon/common/data/variant/variant_defs.h b/src/paimon/common/data/variant/variant_defs.h
index 45f5614f..a9bc8564 100644
--- a/src/paimon/common/data/variant/variant_defs.h
+++ b/src/paimon/common/data/variant/variant_defs.h
@@ -23,6 +23,11 @@
 
 namespace paimon {
 
+enum class VariantShreddingInferenceMode {
+    PER_FILE,
+    ADAPTIVE,
+};
+
 /// Constants of the Paimon Variant type and the Variant Binary Encoding, which follows the
 /// parquet-format VariantEncoding.md specification (compatible with the Java / Spark
 /// implementation).
diff --git a/src/paimon/common/data/variant/variant_shredding_inference_session.cpp b/src/paimon/common/data/variant/variant_shredding_inference_session.cpp
new file mode 100644
index 00000000..5b24e046
--- /dev/null
+++ b/src/paimon/common/data/variant/variant_shredding_inference_session.cpp
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/data/variant/variant_shredding_inference_session.h"
+
+#include 
+
+namespace paimon {
+
+VariantShreddingInferenceSession::VariantShreddingInferenceSession(
+    InferVariantShreddingSchema inferrer, int32_t effective_sample_size, double admission_ratio,
+    double retention_ratio)
+    : inferrer_(std::move(inferrer)),
+      effective_sample_size_(effective_sample_size),
+      admission_ratio_(admission_ratio),
+      retention_ratio_(retention_ratio) {}
+
+Result> VariantShreddingInferenceSession::InferSchema(
+    const InferVariantShreddingSchema::SampleBatches& samples) {
+    InferVariantShreddingSchema::AdaptiveInferenceResult result;
+    if (!has_committed_evidence_) {
+        PAIMON_ASSIGN_OR_RAISE(result, inferrer_.InferInitial(samples, effective_sample_size_));
+    } else {
+        PAIMON_ASSIGN_OR_RAISE(
+            result,
+            inferrer_.InferAdaptive(committed_evidence_, committed_selected_schemas_, samples,
+                                    effective_sample_size_, admission_ratio_, retention_ratio_));
+    }
+    std::shared_ptr physical_schema = result.physical_schema;
+    pending_result_ = std::move(result);
+    return physical_schema;
+}
+
+Status VariantShreddingInferenceSession::CommitPendingInference() {
+    if (!pending_result_.has_value()) {
+        return Status::Invalid("No pending Variant inference to commit.");
+    }
+    committed_evidence_ = std::move(pending_result_->evidence);
+    committed_selected_schemas_ = std::move(pending_result_->selected_schemas);
+    pending_result_.reset();
+    has_committed_evidence_ = true;
+    return Status::OK();
+}
+
+}  // namespace paimon
diff --git a/src/paimon/common/data/variant/variant_shredding_inference_session.h b/src/paimon/common/data/variant/variant_shredding_inference_session.h
new file mode 100644
index 00000000..d0de3b93
--- /dev/null
+++ b/src/paimon/common/data/variant/variant_shredding_inference_session.h
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+
+#include "paimon/common/data/variant/infer_variant_shredding_schema.h"
+#include "paimon/result.h"
+
+namespace paimon {
+
+/// Rolling-writer-scoped Variant inference state. Evidence is bounded and is committed only
+/// after the corresponding file has closed successfully.
+class VariantShreddingInferenceSession {
+ public:
+    VariantShreddingInferenceSession(InferVariantShreddingSchema inferrer,
+                                     int32_t effective_sample_size, double admission_ratio,
+                                     double retention_ratio);
+
+    bool HasPrior() const {
+        return has_committed_evidence_;
+    }
+
+    /// Infers one complete physical row schema and retains the corresponding evidence as pending
+    /// state until the file using this schema has completed successfully.
+    Result> InferSchema(
+        const InferVariantShreddingSchema::SampleBatches& samples);
+
+    Status CommitPendingInference();
+
+ private:
+    InferVariantShreddingSchema inferrer_;
+    int32_t effective_sample_size_;
+    double admission_ratio_;
+    double retention_ratio_;
+
+    bool has_committed_evidence_ = false;
+    InferVariantShreddingSchema::InferenceEvidence committed_evidence_;
+    InferVariantShreddingSchema::SelectedSchemas committed_selected_schemas_;
+    std::optional pending_result_;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp
index f41adff7..b1e14b1a 100644
--- a/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp
+++ b/src/paimon/common/data/variant/variant_shredding_read_plan_factory.cpp
@@ -78,6 +78,40 @@ class FullVariantColumnReadPlan : public ShreddingColumnReadPlan {
     std::shared_ptr pool_;
 };
 
+/// Restores the logical `struct` field order of an untyped physical
+/// `struct` without copying either binary child.
+class UntypedVariantColumnReadPlan : public ShreddingColumnReadPlan {
+ public:
+    UntypedVariantColumnReadPlan(std::shared_ptr logical_field,
+                                 std::shared_ptr physical_field)
+        : logical_field_(std::move(logical_field)), physical_field_(std::move(physical_field)) {}
+
+    const std::shared_ptr& LogicalField() const override {
+        return logical_field_;
+    }
+
+    const std::shared_ptr& PhysicalField() const override {
+        return physical_field_;
+    }
+
+    Result> Assemble(const std::shared_ptr& physical,
+                                                   arrow::MemoryPool*) const override {
+        if (physical->type_id() != arrow::Type::STRUCT ||
+            physical->data()->child_data.size() != 2) {
+            return Status::Invalid(fmt::format("cannot reorder untyped physical variant field {}",
+                                               physical_field_->name()));
+        }
+        std::shared_ptr logical_data = physical->data()->Copy();
+        logical_data->type = logical_field_->type();
+        std::swap(logical_data->child_data[0], logical_data->child_data[1]);
+        return arrow::MakeArray(std::move(logical_data));
+    }
+
+ private:
+    std::shared_ptr logical_field_;
+    std::shared_ptr physical_field_;
+};
+
 /// A node of a nested variant plan tree: a variant position with its own leaf plan, or a nested
 /// container level to descend through.
 struct NestedVariantNode {
@@ -542,6 +576,9 @@ Result> CreateVariantColumnPlan(
         return std::make_shared(
             read_field, physical_field, std::move(schema), std::move(resolved), pool);
     }
+    if (VariantShreddingUtils::IsUntypedPhysicalVariantType(file_field->type())) {
+        return std::make_shared(read_field, file_field);
+    }
     if (!VariantShreddingUtils::IsShreddedFileType(file_field->type())) {
         return std::shared_ptr();
     }
diff --git a/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp b/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp
index 6085d1da..61107e7d 100644
--- a/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp
+++ b/src/paimon/common/data/variant/variant_shredding_read_plan_factory_test.cpp
@@ -155,6 +155,39 @@ TEST_F(VariantShreddingReadPlanFactoryTest, FullVariantReadOfShreddedFile) {
     ASSERT_NOK(plan->Assemble(ints, arrow::default_memory_pool()));
 }
 
+TEST_F(VariantShreddingReadPlanFactoryTest, FullVariantReadOfUntypedPhysicalFile) {
+    std::shared_ptr variant = Variant(R"({"a": 5})");
+    std::shared_ptr physical;
+    std::shared_ptr shredded;
+    MakeFullShredded(arrow::null(), variant, &physical, &shredded);
+    ASSERT_FALSE(HasFatalFailure());
+
+    const auto& physical_struct = static_cast(*physical);
+    ASSERT_EQ(physical_struct.num_fields(), 2);
+    ASSERT_EQ(physical_struct.field(0)->name(), VariantDefs::kMetadataFieldName);
+    ASSERT_EQ(physical_struct.field(1)->name(), VariantDefs::kValueFieldName);
+    ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(physical));
+    ASSERT_TRUE(VariantShreddingUtils::IsUntypedPhysicalVariantType(physical));
+
+    auto read_field = VariantTypeUtils::ToArrowField("v");
+    auto file_field = arrow::field("v", physical);
+    std::shared_ptr plan = CreatePlan(read_field, file_field);
+    ASSERT_NE(plan, nullptr);
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr assembled,
+                         plan->Assemble(shredded, arrow::default_memory_pool()));
+    auto assembled_struct = std::static_pointer_cast(assembled);
+    ASSERT_EQ(assembled_struct->data()->child_data[0], shredded->data()->child_data[1]);
+    ASSERT_EQ(assembled_struct->data()->child_data[1], shredded->data()->child_data[0]);
+    auto value_column = std::static_pointer_cast(assembled_struct->field(0));
+    auto metadata_column = std::static_pointer_cast(assembled_struct->field(1));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr rebuilt,
+        GenericVariant::Create(value_column->GetView(0), metadata_column->GetView(0), pool_));
+    ASSERT_OK_AND_ASSIGN(std::string json, rebuilt->ToJson());
+    EXPECT_EQ(json, R"({"a":5})");
+}
+
 TEST_F(VariantShreddingReadPlanFactoryTest, AccessProjectionOnUnshreddedFile) {
     std::shared_ptr variant = Variant(R"({"a": 5, "b": [10, 20], "c": "hi"})");
     std::shared_ptr unshredded = MakeUnshredded(variant);
diff --git a/src/paimon/common/data/variant/variant_shredding_test.cpp b/src/paimon/common/data/variant/variant_shredding_test.cpp
index a2c053ad..632223a9 100644
--- a/src/paimon/common/data/variant/variant_shredding_test.cpp
+++ b/src/paimon/common/data/variant/variant_shredding_test.cpp
@@ -200,9 +200,15 @@ TEST_F(VariantShreddingTest, ShreddingSchemaShape) {
     ASSERT_EQ(schema->object_schema.size(), 2);
     ASSERT_FALSE(schema->IsUnshredded());
     ASSERT_TRUE(VariantShreddingUtils::IsShreddedFileType(physical));
-    ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(
-        arrow::struct_({arrow::field("value", arrow::binary(), false),
-                        arrow::field("metadata", arrow::binary(), false)})));
+    auto logical_variant = arrow::struct_({arrow::field("value", arrow::binary(), false),
+                                           arrow::field("metadata", arrow::binary(), false)});
+    auto untyped_physical_variant =
+        arrow::struct_({arrow::field("metadata", arrow::binary(), false),
+                        arrow::field("value", arrow::binary(), false)});
+    ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(logical_variant));
+    ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(untyped_physical_variant));
+    ASSERT_FALSE(VariantShreddingUtils::IsUntypedPhysicalVariantType(logical_variant));
+    ASSERT_TRUE(VariantShreddingUtils::IsUntypedPhysicalVariantType(untyped_physical_variant));
 
     // Invalid shredding types are rejected.
     ASSERT_NOK(VariantShreddingUtils::VariantShreddingSchema(arrow::date32()));
diff --git a/src/paimon/common/data/variant/variant_shredding_utils.cpp b/src/paimon/common/data/variant/variant_shredding_utils.cpp
index 8d6f4382..bbb23135 100644
--- a/src/paimon/common/data/variant/variant_shredding_utils.cpp
+++ b/src/paimon/common/data/variant/variant_shredding_utils.cpp
@@ -309,4 +309,21 @@ bool VariantShreddingUtils::IsShreddedFileType(
     return struct_type->GetFieldByName(VariantDefs::kTypedValueFieldName) != nullptr;
 }
 
+bool VariantShreddingUtils::IsUntypedPhysicalVariantType(
+    const std::shared_ptr& file_variant_type) {
+    if (!file_variant_type || file_variant_type->id() != arrow::Type::STRUCT) {
+        return false;
+    }
+    const auto& struct_type = std::static_pointer_cast(file_variant_type);
+    if (struct_type->num_fields() != 2) {
+        return false;
+    }
+    const auto& metadata = struct_type->field(0);
+    const auto& value = struct_type->field(1);
+    return metadata->name() == VariantDefs::kMetadataFieldName &&
+           metadata->type()->id() == arrow::Type::BINARY &&
+           value->name() == VariantDefs::kValueFieldName &&
+           value->type()->id() == arrow::Type::BINARY;
+}
+
 }  // namespace paimon
diff --git a/src/paimon/common/data/variant/variant_shredding_utils.h b/src/paimon/common/data/variant/variant_shredding_utils.h
index 8f5b327c..7aa59e1d 100644
--- a/src/paimon/common/data/variant/variant_shredding_utils.h
+++ b/src/paimon/common/data/variant/variant_shredding_utils.h
@@ -59,6 +59,10 @@ class VariantShreddingUtils {
     /// Whether the physical struct type of a variant field in a data file is shredded (contains
     /// a `typed_value` child).
     static bool IsShreddedFileType(const std::shared_ptr& file_variant_type);
+
+    /// Whether the physical struct uses the untyped inference layout `{metadata, value}`.
+    static bool IsUntypedPhysicalVariantType(
+        const std::shared_ptr& file_variant_type);
 };
 
 }  // namespace paimon
diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan.cpp
index 96305db0..7fccc4ff 100644
--- a/src/paimon/common/data/variant/variant_shredding_write_plan.cpp
+++ b/src/paimon/common/data/variant/variant_shredding_write_plan.cpp
@@ -91,6 +91,52 @@ Result> ReplacePlannedFields(
     return field->WithType(arrow::struct_(new_fields));
 }
 
+Status CollectPlannedColumns(const std::shared_ptr& logical_field,
+                             const std::shared_ptr& physical_field,
+                             std::vector* path,
+                             std::vector* columns) {
+    if (logical_field->name() != physical_field->name()) {
+        return Status::Invalid(
+            fmt::format("variant shredding physical field '{}' does not match logical field '{}'",
+                        physical_field->name(), logical_field->name()));
+    }
+    if (VariantTypeUtils::IsVariantField(logical_field)) {
+        if (logical_field->type()->Equals(*physical_field->type())) {
+            return Status::OK();
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr variant_schema,
+                               VariantShreddingUtils::BuildVariantSchema(physical_field->type()));
+        columns->push_back(VariantShreddingWritePlan::PlannedColumn{
+            *path, std::move(variant_schema), physical_field->type()});
+        return Status::OK();
+    }
+    if (logical_field->type()->Equals(*physical_field->type())) {
+        return Status::OK();
+    }
+    if (logical_field->type()->id() != arrow::Type::STRUCT ||
+        physical_field->type()->id() != arrow::Type::STRUCT) {
+        return Status::Invalid(fmt::format(
+            "variant shredding physical type of field '{}' differs outside a Variant column",
+            logical_field->name()));
+    }
+    const auto& logical_type =
+        arrow::internal::checked_cast(*logical_field->type());
+    const auto& physical_type =
+        arrow::internal::checked_cast(*physical_field->type());
+    if (logical_type.num_fields() != physical_type.num_fields()) {
+        return Status::Invalid(
+            fmt::format("variant shredding physical struct '{}' has a different field count",
+                        logical_field->name()));
+    }
+    for (int32_t i = 0; i < logical_type.num_fields(); ++i) {
+        path->push_back(i);
+        PAIMON_RETURN_NOT_OK(
+            CollectPlannedColumns(logical_type.field(i), physical_type.field(i), path, columns));
+        path->pop_back();
+    }
+    return Status::OK();
+}
+
 }  // namespace
 
 Result> VariantShreddingWritePlan::Create(
@@ -114,15 +160,31 @@ Result> VariantShreddingWritePlan::Cr
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr new_root,
         ReplacePlannedFields(root_field, path_shredding_types, /*depth=*/0, &columns));
-    if (columns.empty()) {
-        // No planned path matches a variant column; the file is written unshredded.
-        return std::shared_ptr(nullptr);
-    }
     auto physical_schema = arrow::schema(new_root->type()->fields(), logical_schema->metadata());
     return std::shared_ptr(new VariantShreddingWritePlan(
         logical_schema, std::move(physical_schema), std::move(columns)));
 }
 
+Result>
+VariantShreddingWritePlan::CreateFromPhysicalSchema(
+    const std::shared_ptr& logical_schema,
+    const std::shared_ptr& physical_schema) {
+    if (logical_schema->num_fields() != physical_schema->num_fields()) {
+        return Status::Invalid(
+            "variant shredding logical and physical schemas have different field counts");
+    }
+    std::vector columns;
+    std::vector path;
+    for (int32_t i = 0; i < logical_schema->num_fields(); ++i) {
+        path.push_back(i);
+        PAIMON_RETURN_NOT_OK(CollectPlannedColumns(logical_schema->field(i),
+                                                   physical_schema->field(i), &path, &columns));
+        path.pop_back();
+    }
+    return std::shared_ptr(
+        new VariantShreddingWritePlan(logical_schema, physical_schema, std::move(columns)));
+}
+
 Result> VariantShreddingWritePlan::FromConfiguredSchema(
     const std::shared_ptr& logical_schema,
     const std::string& configured_schema_json) {
diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan.h b/src/paimon/common/data/variant/variant_shredding_write_plan.h
index d05faa25..2fa7a886 100644
--- a/src/paimon/common/data/variant/variant_shredding_write_plan.h
+++ b/src/paimon/common/data/variant/variant_shredding_write_plan.h
@@ -54,20 +54,25 @@ class VariantShreddingWritePlan {
     /// @param logical_schema The logical write schema.
     /// @param column_shredding_types The shredding type per variant column name, e.g.
     ///        `{"v": struct{a: int32, b: string}}`. Names that are not top-level variant columns
-    ///        of `logical_schema` are ignored. Returns nullptr when no name matches (the file is
-    ///        written unshredded, mirroring the Java behavior).
+    ///        of `logical_schema` are ignored. When no name matches, returns an identity plan.
     static Result> Create(
         const std::shared_ptr& logical_schema,
         const std::map>& column_shredding_types);
 
     /// Creates a plan shredding the variant columns at the given field-index paths (top-level or
-    /// nested inside structs). Paths that do not point at a variant field are ignored. Returns
-    /// nullptr when no path matches.
+    /// nested inside structs). Paths that do not point at a variant field are ignored. When no
+    /// path matches, returns an identity plan.
     static Result> CreateFromPaths(
         const std::shared_ptr& logical_schema,
         const std::map, std::shared_ptr>&
             path_shredding_types);
 
+    /// Creates a plan by comparing the complete logical and physical row schemas produced by
+    /// whole-row Variant schema inference.
+    static Result> CreateFromPhysicalSchema(
+        const std::shared_ptr& logical_schema,
+        const std::shared_ptr& physical_schema);
+
     /// Creates a plan from the `variant.shreddingSchema` option value: a ROW type JSON whose
     /// fields map top-level variant column names to their shredding types (nested variant
     /// columns cannot be configured, as in Java).
diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp
index bb7c5579..af7b73f0 100644
--- a/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp
+++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.cpp
@@ -19,13 +19,10 @@
 
 #include "paimon/common/data/variant/variant_shredding_write_plan_factory.h"
 
-#include 
 #include 
 
 #include "arrow/api.h"
-#include "arrow/util/checked_cast.h"
 #include "fmt/format.h"
-#include "paimon/common/data/variant/generic_variant.h"
 #include "paimon/common/data/variant/infer_variant_shredding_schema.h"
 #include "paimon/common/data/variant/variant_shredding_batch_converter.h"
 #include "paimon/common/data/variant/variant_shredding_write_plan.h"
@@ -35,78 +32,17 @@ namespace paimon {
 
 namespace {
 
-/// Collects the field-index paths of the shreddable variant fields: at the top level or nested
-/// inside structs only, mirroring the Java `InferVariantShreddingSchema.getPathsToVariant`.
-void CollectVariantPaths(const arrow::FieldVector& fields, std::vector* current,
-                         std::vector>* paths) {
-    for (int32_t i = 0; i < static_cast(fields.size()); ++i) {
-        const std::shared_ptr& field = fields[i];
-        current->push_back(i);
+bool ContainsVariantFields(const arrow::FieldVector& fields) {
+    for (const std::shared_ptr& field : fields) {
         if (VariantTypeUtils::IsVariantField(field)) {
-            paths->push_back(*current);
-        } else if (field->type()->id() == arrow::Type::STRUCT) {
-            CollectVariantPaths(field->type()->fields(), current, paths);
+            return true;
         }
-        current->pop_back();
-    }
-}
-
-std::vector> GetPathsToVariant(const arrow::Schema& schema) {
-    std::vector> paths;
-    std::vector current;
-    CollectVariantPaths(schema.fields(), ¤t, &paths);
-    return paths;
-}
-
-/// Collects the non-null variant values of one sample batch at the given field-index path,
-/// descending through struct arrays. Rows that are null at any level contribute no sample.
-Result>> CollectSamplesAtPath(
-    const std::vector>& sample_batches,
-    const std::vector& path, const std::shared_ptr& pool) {
-    std::vector> samples;
-    for (const auto& sample_batch : sample_batches) {
-        std::shared_ptr column = sample_batch;
-        // The structs enclosing the variant column (the batch root excluded): child slot
-        // contents under a null ancestor are unspecified in Arrow and must not be decoded.
-        std::vector> ancestors;
-        for (int32_t index : path) {
-            if (column == nullptr || column->type_id() != arrow::Type::STRUCT) {
-                return Status::Invalid("sample batch does not match the variant column path");
-            }
-            if (column != sample_batch) {
-                ancestors.push_back(column);
-            }
-            column = arrow::internal::checked_cast(*column).field(index);
-        }
-        if (column == nullptr) {
-            return Status::Invalid("sample batch misses the planned variant column");
-        }
-        const auto& variant_array =
-            arrow::internal::checked_cast(*column);
-        const auto& value_array =
-            arrow::internal::checked_cast(*variant_array.field(0));
-        const auto& metadata_array =
-            arrow::internal::checked_cast(*variant_array.field(1));
-        auto row_is_null = [&](int64_t row) {
-            for (const auto& ancestor : ancestors) {
-                if (ancestor->IsNull(row)) {
-                    return true;
-                }
-            }
-            return variant_array.IsNull(row);
-        };
-        for (int64_t row = 0; row < variant_array.length(); ++row) {
-            if (row_is_null(row)) {
-                continue;
-            }
-            PAIMON_ASSIGN_OR_RAISE(
-                std::shared_ptr variant,
-                GenericVariant::Create(std::string_view(value_array.GetView(row)),
-                                       std::string_view(metadata_array.GetView(row)), pool));
-            samples.push_back(std::move(variant));
+        if (field->type()->id() == arrow::Type::STRUCT &&
+            ContainsVariantFields(field->type()->fields())) {
+            return true;
         }
     }
-    return samples;
+    return false;
 }
 
 }  // namespace
@@ -114,7 +50,9 @@ Result>> CollectSamplesAtPath(
 VariantShreddingWritePlanFactory::VariantShreddingWritePlanFactory(
     std::optional configured_schema, bool infer_enabled, int32_t max_schema_width,
     int32_t max_schema_depth, double min_field_cardinality_ratio, int32_t max_infer_buffer_row,
-    const std::shared_ptr& write_schema, const std::shared_ptr& pool)
+    VariantShreddingInferenceMode inference_mode, int32_t adaptive_max_infer_buffer_row,
+    double adaptive_retention_ratio, const std::shared_ptr& write_schema,
+    const std::shared_ptr& pool)
     : write_schema_(write_schema),
       pool_(pool),
       configured_schema_(std::move(configured_schema)),
@@ -122,7 +60,16 @@ VariantShreddingWritePlanFactory::VariantShreddingWritePlanFactory(
       max_schema_width_(max_schema_width),
       max_schema_depth_(max_schema_depth),
       min_field_cardinality_ratio_(min_field_cardinality_ratio),
-      max_infer_buffer_row_(max_infer_buffer_row) {}
+      max_infer_buffer_row_(max_infer_buffer_row),
+      adaptive_max_infer_buffer_row_(adaptive_max_infer_buffer_row) {
+    if (!configured_schema_.has_value() && infer_enabled_ &&
+        inference_mode == VariantShreddingInferenceMode::ADAPTIVE) {
+        adaptive_session_ = std::make_unique(
+            InferVariantShreddingSchema(write_schema_, pool_, max_schema_width_, max_schema_depth_,
+                                        min_field_cardinality_ratio_),
+            adaptive_max_infer_buffer_row_, min_field_cardinality_ratio_, adaptive_retention_ratio);
+    }
+}
 
 std::shared_ptr VariantShreddingWritePlanFactory::Create(
     const CoreOptions& options, const std::shared_ptr& write_schema,
@@ -131,7 +78,9 @@ std::shared_ptr VariantShreddingWritePlanFacto
         options.GetVariantShreddingSchema(), options.VariantInferShreddingSchemaEnabled(),
         options.GetVariantShreddingMaxSchemaWidth(), options.GetVariantShreddingMaxSchemaDepth(),
         options.GetVariantShreddingMinFieldCardinalityRatio(),
-        options.GetVariantShreddingMaxInferBufferRow(), write_schema, pool));
+        options.GetVariantShreddingMaxInferBufferRow(), options.GetVariantShreddingInferenceMode(),
+        options.GetVariantShreddingAdaptiveMaxInferBufferRow(),
+        options.GetVariantShreddingAdaptiveRetentionRatio(), write_schema, pool));
 }
 
 bool VariantShreddingWritePlanFactory::ShouldCreateWritePlan() const {
@@ -143,6 +92,9 @@ bool VariantShreddingWritePlanFactory::ShouldInferWritePlan() const {
 }
 
 int32_t VariantShreddingWritePlanFactory::InferBufferRowCount() const {
+    if (adaptive_session_ != nullptr && adaptive_session_->HasPrior()) {
+        return adaptive_max_infer_buffer_row_;
+    }
     return max_infer_buffer_row_;
 }
 
@@ -151,12 +103,12 @@ bool VariantShreddingWritePlanFactory::HasConfiguredShreddingSchema() const {
 }
 
 bool VariantShreddingWritePlanFactory::ContainsShreddableVariantField() const {
-    return !GetPathsToVariant(*write_schema_).empty();
+    return ContainsVariantFields(write_schema_->fields());
 }
 
 Result> VariantShreddingWritePlanFactory::CreateConverter(
     const std::string& file_format_identifier,
-    const std::vector>& sample_batches) const {
+    const std::vector>& sample_batches) {
     if (file_format_identifier != "parquet") {
         return Status::NotImplemented(
             fmt::format("variant shredding is only supported by the parquet file format, got {}",
@@ -168,35 +120,40 @@ Result> VariantShreddingWritePlanFactor
         PAIMON_ASSIGN_OR_RAISE(plan, VariantShreddingWritePlan::FromConfiguredSchema(
                                          write_schema_, configured_schema_.value()));
     } else {
-        InferVariantShreddingSchema inferrer(max_schema_width_, max_schema_depth_,
-                                             min_field_cardinality_ratio_);
-        // One budget is shared across all variant columns so that the total inferred width stays
-        // within `variant.shredding.maxSchemaWidth` (as in Java).
-        InferVariantShreddingSchema::MaxFields max_fields = inferrer.CreateMaxFieldsBudget();
-        std::map, std::shared_ptr> path_shredding_types;
-        for (const std::vector& path : GetPathsToVariant(*write_schema_)) {
-            PAIMON_ASSIGN_OR_RAISE(std::vector> samples,
-                                   CollectSamplesAtPath(sample_batches, path, pool_));
-            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_type,
-                                   inferrer.InferColumnShreddingType(samples, &max_fields));
-            if (shredding_type != nullptr) {
-                path_shredding_types.emplace(path, std::move(shredding_type));
-            }
-        }
-        if (path_shredding_types.empty()) {
-            // No useful shredding schema was found; write the file unshredded.
-            return std::shared_ptr(nullptr);
+        std::shared_ptr physical_schema;
+        if (adaptive_session_ != nullptr) {
+            PAIMON_ASSIGN_OR_RAISE(physical_schema, adaptive_session_->InferSchema(sample_batches));
+            has_pending_adaptive_inference_ = true;
+        } else {
+            InferVariantShreddingSchema inferrer(write_schema_, pool_, max_schema_width_,
+                                                 max_schema_depth_, min_field_cardinality_ratio_);
+            PAIMON_ASSIGN_OR_RAISE(physical_schema, inferrer.InferSchema(sample_batches));
         }
-        PAIMON_ASSIGN_OR_RAISE(
-            plan, VariantShreddingWritePlan::CreateFromPaths(write_schema_, path_shredding_types));
-    }
-    if (plan == nullptr) {
-        // The configured schema names no variant column; write the file unshredded.
-        return std::shared_ptr(nullptr);
+        PAIMON_ASSIGN_OR_RAISE(plan, VariantShreddingWritePlan::CreateFromPhysicalSchema(
+                                         write_schema_, physical_schema));
     }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr converter,
                            VariantShreddingBatchConverter::Create(plan, pool_));
-    return std::shared_ptr(std::move(converter));
+    std::shared_ptr result = std::move(converter);
+    if (adaptive_session_ != nullptr) {
+        pending_adaptive_converter_ = result;
+    }
+    return result;
+}
+
+Status VariantShreddingWritePlanFactory::OnFileCompleted(
+    const std::shared_ptr& converter) {
+    if (adaptive_session_ == nullptr) {
+        return Status::OK();
+    }
+    if (!has_pending_adaptive_inference_ || converter != pending_adaptive_converter_) {
+        return Status::Invalid(
+            "Completed Variant write plan does not match the pending adaptive inference.");
+    }
+    PAIMON_RETURN_NOT_OK(adaptive_session_->CommitPendingInference());
+    has_pending_adaptive_inference_ = false;
+    pending_adaptive_converter_.reset();
+    return Status::OK();
 }
 
 }  // namespace paimon
diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h
index 098da8f0..69b0fa79 100644
--- a/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h
+++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory.h
@@ -25,6 +25,7 @@
 #include 
 
 #include "paimon/common/data/shredding/shredding_write_plan_factory.h"
+#include "paimon/common/data/variant/variant_shredding_inference_session.h"
 #include "paimon/core/core_options.h"
 #include "paimon/memory/memory_pool.h"
 #include "paimon/result.h"
@@ -57,21 +58,24 @@ class VariantShreddingWritePlanFactory : public ShreddingWritePlanFactory {
 
     Result> CreateConverter(
         const std::string& file_format_identifier,
-        const std::vector>& sample_batches) const override;
+        const std::vector>& sample_batches) override;
 
     MetadataFinalizer CreateMetadataFinalizer(
-        const std::shared_ptr& converter) const override {
+        const std::shared_ptr& converter,
+        const std::string& compression) const override {
         // The shredded physical schema is self-describing; no per-file metadata is needed.
         return nullptr;
     }
 
+    Status OnFileCompleted(const std::shared_ptr& converter) override;
+
  private:
-    VariantShreddingWritePlanFactory(std::optional configured_schema,
-                                     bool infer_enabled, int32_t max_schema_width,
-                                     int32_t max_schema_depth, double min_field_cardinality_ratio,
-                                     int32_t max_infer_buffer_row,
-                                     const std::shared_ptr& write_schema,
-                                     const std::shared_ptr& pool);
+    VariantShreddingWritePlanFactory(
+        std::optional configured_schema, bool infer_enabled, int32_t max_schema_width,
+        int32_t max_schema_depth, double min_field_cardinality_ratio, int32_t max_infer_buffer_row,
+        VariantShreddingInferenceMode inference_mode, int32_t adaptive_max_infer_buffer_row,
+        double adaptive_retention_ratio, const std::shared_ptr& write_schema,
+        const std::shared_ptr& pool);
 
     bool HasConfiguredShreddingSchema() const;
     /// Whether the write schema holds a shreddable variant field: at the top level or nested
@@ -87,6 +91,10 @@ class VariantShreddingWritePlanFactory : public ShreddingWritePlanFactory {
     int32_t max_schema_depth_ = 0;
     double min_field_cardinality_ratio_ = 0.0;
     int32_t max_infer_buffer_row_ = 0;
+    int32_t adaptive_max_infer_buffer_row_ = 0;
+    std::unique_ptr adaptive_session_;
+    bool has_pending_adaptive_inference_ = false;
+    std::shared_ptr pending_adaptive_converter_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp
index b5b09e33..a3d3c6c5 100644
--- a/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp
+++ b/src/paimon/common/data/variant/variant_shredding_write_plan_factory_test.cpp
@@ -27,6 +27,7 @@
 #include "arrow/c/bridge.h"
 #include "gtest/gtest.h"
 #include "paimon/common/data/variant/generic_variant.h"
+#include "paimon/common/data/variant/variant_shredding_utils.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/core/core_options.h"
@@ -57,6 +58,17 @@ class VariantShreddingWritePlanFactoryTest : public ::testing::Test {
         return std::move(result).value();
     }
 
+    static std::shared_ptr ShreddedTypedValue(
+        const arrow::StructType& typed_object, const std::string& field_name) {
+        auto field = typed_object.GetFieldByName(field_name);
+        if (field == nullptr || field->type()->id() != arrow::Type::STRUCT) {
+            return nullptr;
+        }
+        return std::static_pointer_cast(field->type())
+            ->GetFieldByName("typed_value")
+            ->type();
+    }
+
  protected:
     std::shared_ptr pool_;
     std::shared_ptr schema_;
@@ -124,14 +136,316 @@ TEST_F(VariantShreddingWritePlanFactoryTest, InferredSchema) {
     ASSERT_NE(typed_struct.GetFieldByName("city"), nullptr);
 }
 
+// Verifies that adaptive inference returns a complete physical row schema to the write-plan
+// factory, and that committed evidence is reused only after the preceding file completes.
+TEST_F(VariantShreddingWritePlanFactoryTest, AdaptiveInferenceReturnsCompletePhysicalSchema) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         MakeOptions({{"variant.inferShreddingSchema", "true"},
+                                      {"variant.shredding.inferenceMode", "adaptive"}}));
+    auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_);
+    std::vector> samples = {
+        BuildBatch({R"({"age": 35, "city": "Chicago"})"})};
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr expected_variant_type,
+        VariantShreddingUtils::VariantShreddingSchema(arrow::struct_(
+            {arrow::field("age", arrow::int64()), arrow::field("city", arrow::utf8())})));
+    auto expected_schema =
+        arrow::schema({schema_->field(0), schema_->field(1)->WithType(expected_variant_type)},
+                      schema_->metadata());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr first,
+                         factory->CreateConverter("parquet", samples));
+    ASSERT_NE(first, nullptr);
+    ASSERT_TRUE(first->GetPhysicalSchema()->Equals(*expected_schema))
+        << first->GetPhysicalSchema()->ToString();
+
+    ASSERT_OK(factory->OnFileCompleted(first));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr second,
+                         factory->CreateConverter("parquet", {}));
+    ASSERT_NE(second, nullptr);
+    ASSERT_TRUE(second->GetPhysicalSchema()->Equals(*expected_schema))
+        << second->GetPhysicalSchema()->ToString();
+    ASSERT_OK(factory->OnFileCompleted(second));
+}
+
+TEST_F(VariantShreddingWritePlanFactoryTest, AdaptiveInferenceUsesAdmissionAndRetentionThresholds) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         MakeOptions({{"variant.inferShreddingSchema", "true"},
+                                      {"variant.shredding.inferenceMode", "adaptive"},
+                                      {"variant.shredding.maxInferBufferRow", "10"},
+                                      {"variant.shredding.adaptive.maxInferBufferRow", "10"},
+                                      {"variant.shredding.minFieldCardinalityRatio", "0.4"},
+                                      {"variant.shredding.adaptive.retentionRatio", "0.2"}}));
+    auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_);
+    auto typed_value = [](const std::shared_ptr& converter) {
+        const auto& variant_type = static_cast(
+            *converter->GetPhysicalSchema()->GetFieldByName("v")->type());
+        return variant_type.GetFieldByName("typed_value");
+    };
+
+    std::vector> first_samples = {
+        BuildBatch({R"({"legacy":"v","stable":1})", R"({"legacy":"v","stable":1})",
+                    R"({"legacy":"v","stable":1})", R"({"legacy":"v","stable":1})",
+                    R"({"legacy":"v","stable":1})", R"({"stable":1})", R"({"stable":1})",
+                    R"({"stable":1})", R"({"stable":1})", R"({"stable":1})"})};
+    ASSERT_OK_AND_ASSIGN(auto first, factory->CreateConverter("parquet", first_samples));
+    auto first_typed = typed_value(first);
+    ASSERT_NE(nullptr, first_typed);
+    const auto& first_struct = static_cast(*first_typed->type());
+    ASSERT_TRUE(ShreddedTypedValue(first_struct, "legacy")->Equals(*arrow::utf8()));
+    ASSERT_TRUE(ShreddedTypedValue(first_struct, "stable")->Equals(*arrow::int64()));
+    ASSERT_OK(factory->OnFileCompleted(first));
+
+    std::vector> second_samples = {
+        BuildBatch({R"({"emerging":true,"stable":2})", R"({"emerging":true,"stable":2})",
+                    R"({"emerging":true,"stable":2})", R"({"emerging":true,"stable":2})",
+                    R"({"emerging":true,"stable":2})", R"({"emerging":true,"stable":2})",
+                    R"({"emerging":true,"stable":2})", R"({"emerging":true,"stable":2})",
+                    R"({"emerging":true,"stable":2})", R"({"stable":2})"})};
+    ASSERT_OK_AND_ASSIGN(auto second, factory->CreateConverter("parquet", second_samples));
+    auto second_typed = typed_value(second);
+    ASSERT_NE(nullptr, second_typed);
+    const auto& second_struct = static_cast(*second_typed->type());
+    ASSERT_TRUE(ShreddedTypedValue(second_struct, "emerging")->Equals(*arrow::boolean()));
+    ASSERT_TRUE(ShreddedTypedValue(second_struct, "legacy")->Equals(*arrow::utf8()));
+    ASSERT_TRUE(ShreddedTypedValue(second_struct, "stable")->Equals(*arrow::int64()));
+    ASSERT_OK(factory->OnFileCompleted(second));
+
+    std::vector> third_samples = {
+        BuildBatch({R"({"stable":3})", R"({"stable":3})", R"({"stable":3})", R"({"stable":3})",
+                    R"({"stable":3})", R"({"stable":3})", R"({"stable":3})", R"({"stable":3})",
+                    R"({"stable":3})", R"({"stable":3})"})};
+    ASSERT_OK_AND_ASSIGN(auto third, factory->CreateConverter("parquet", third_samples));
+    auto third_typed = typed_value(third);
+    ASSERT_NE(nullptr, third_typed);
+    const auto& third_struct = static_cast(*third_typed->type());
+    ASSERT_TRUE(ShreddedTypedValue(third_struct, "emerging")->Equals(*arrow::boolean()));
+    ASSERT_EQ(nullptr, third_struct.GetFieldByName("legacy"));
+    ASSERT_TRUE(ShreddedTypedValue(third_struct, "stable")->Equals(*arrow::int64()));
+    ASSERT_OK(factory->OnFileCompleted(third));
+}
+
+TEST_F(VariantShreddingWritePlanFactoryTest, AdaptiveInferenceOnShortRolledFile) {
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         MakeOptions({{"variant.inferShreddingSchema", "true"},
+                                      {"variant.shredding.inferenceMode", "adaptive"},
+                                      {"variant.shredding.maxInferBufferRow", "4"},
+                                      {"variant.shredding.adaptive.maxInferBufferRow", "4"},
+                                      {"variant.shredding.minFieldCardinalityRatio", "0.4"},
+                                      {"variant.shredding.adaptive.retentionRatio", "0.2"}}));
+    auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_);
+
+    std::vector> first_samples = {
+        BuildBatch({R"({"legacy":"a","stable":1})", R"({"legacy":"b","stable":2})",
+                    R"({"legacy":"c","stable":3})", R"({"legacy":"d","stable":4})"})};
+    ASSERT_OK_AND_ASSIGN(auto first, factory->CreateConverter("parquet", first_samples));
+    ASSERT_OK(factory->OnFileCompleted(first));
+
+    std::vector> short_samples = {
+        BuildBatch({R"({"emerging":true,"stable":5})", R"({"emerging":false,"stable":6})",
+                    R"({"emerging":true,"stable":7})"})};
+    ASSERT_OK_AND_ASSIGN(auto second, factory->CreateConverter("parquet", short_samples));
+    const auto& variant_type = static_cast(
+        *second->GetPhysicalSchema()->GetFieldByName("v")->type());
+    const auto& typed_object =
+        static_cast(*variant_type.GetFieldByName("typed_value")->type());
+    ASSERT_TRUE(ShreddedTypedValue(typed_object, "emerging")->Equals(*arrow::boolean()));
+    ASSERT_TRUE(ShreddedTypedValue(typed_object, "legacy")->Equals(*arrow::utf8()));
+    ASSERT_TRUE(ShreddedTypedValue(typed_object, "stable")->Equals(*arrow::int64()));
+    ASSERT_OK(factory->OnFileCompleted(second));
+}
+
+TEST_F(VariantShreddingWritePlanFactoryTest,
+       AdaptiveInferenceWidensScalarSelectedFromPriorEvidence) {
+    std::vector fields = {DataField(1, arrow::field("id", arrow::int32())),
+                                     DataField(2, VariantTypeUtils::ToArrowField("first")),
+                                     DataField(3, VariantTypeUtils::ToArrowField("second"))};
+    auto schema = DataField::ConvertDataFieldsToArrowSchema(fields);
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         MakeOptions({{"variant.inferShreddingSchema", "true"},
+                                      {"variant.shredding.inferenceMode", "adaptive"},
+                                      {"variant.shredding.maxSchemaWidth", "6"}}));
+    auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_);
+    auto build_batch = [&](const std::vector& first_jsons,
+                           const std::vector& second_jsons) {
+        auto first = VariantTestData::BuildVariantBatch(schema->field(0), schema->field(1),
+                                                        first_jsons, pool_);
+        EXPECT_TRUE(first.ok()) << first.status().ToString();
+        auto second = VariantTestData::BuildVariantBatch(schema->field(0), schema->field(2),
+                                                         second_jsons, pool_);
+        EXPECT_TRUE(second.ok()) << second.status().ToString();
+        return std::shared_ptr(
+            arrow::StructArray::Make(
+                {first.value()->field(0), first.value()->field(1), second.value()->field(1)},
+                schema->fields())
+                .ValueOrDie());
+    };
+
+    std::vector> first_samples = {
+        build_batch({R"({"a":1,"b":2})"}, {R"({"historical":12345})"})};
+    ASSERT_OK_AND_ASSIGN(auto initial, factory->CreateConverter("parquet", first_samples));
+    const auto& initial_first = static_cast(
+        *initial->GetPhysicalSchema()->GetFieldByName("first")->type());
+    const auto& initial_first_typed =
+        static_cast(*initial_first.GetFieldByName("typed_value")->type());
+    ASSERT_NE(nullptr, initial_first_typed.GetFieldByName("a"));
+    ASSERT_NE(nullptr, initial_first_typed.GetFieldByName("b"));
+    const auto& initial_second = static_cast(
+        *initial->GetPhysicalSchema()->GetFieldByName("second")->type());
+    ASSERT_EQ(nullptr, initial_second.GetFieldByName("typed_value"));
+    ASSERT_OK(factory->OnFileCompleted(initial));
+
+    std::vector> adaptive_samples = {build_batch({"1"}, {nullptr})};
+    ASSERT_OK_AND_ASSIGN(auto adaptive, factory->CreateConverter("parquet", adaptive_samples));
+    const auto& adaptive_first = static_cast(
+        *adaptive->GetPhysicalSchema()->GetFieldByName("first")->type());
+    ASSERT_TRUE(adaptive_first.GetFieldByName("typed_value")->type()->Equals(*arrow::int64()));
+    const auto& adaptive_second = static_cast(
+        *adaptive->GetPhysicalSchema()->GetFieldByName("second")->type());
+    const auto& adaptive_second_typed = static_cast(
+        *adaptive_second.GetFieldByName("typed_value")->type());
+    ASSERT_TRUE(ShreddedTypedValue(adaptive_second_typed, "historical")->Equals(*arrow::int64()));
+    ASSERT_OK(factory->OnFileCompleted(adaptive));
+}
+
+TEST_F(VariantShreddingWritePlanFactoryTest, AdaptiveInferenceWithNestedVariant) {
+    auto nested_variant = VariantTypeUtils::ToArrowField("payload");
+    auto nested_field = arrow::field(
+        "nested", arrow::struct_({arrow::field("label", arrow::utf8()), nested_variant}));
+    std::vector fields = {DataField(1, arrow::field("id", arrow::int32())),
+                                     DataField(2, nested_field)};
+    auto schema = DataField::ConvertDataFieldsToArrowSchema(fields);
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         MakeOptions({{"variant.inferShreddingSchema", "true"},
+                                      {"variant.shredding.inferenceMode", "adaptive"},
+                                      {"variant.shredding.maxInferBufferRow", "2"},
+                                      {"variant.shredding.adaptive.maxInferBufferRow", "2"},
+                                      {"variant.shredding.minFieldCardinalityRatio", "0.4"},
+                                      {"variant.shredding.adaptive.retentionRatio", "0.2"}}));
+    auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_);
+    auto build_nested_batch = [&](const std::vector& labels,
+                                  const std::vector& jsons) {
+        auto variants =
+            VariantTestData::BuildVariantBatch(schema->field(0), nested_variant, jsons, pool_);
+        EXPECT_TRUE(variants.ok()) << variants.status().ToString();
+        arrow::StringBuilder label_builder;
+        for (const char* label : labels) {
+            EXPECT_TRUE(label_builder.Append(label).ok());
+        }
+        std::shared_ptr label_array;
+        EXPECT_TRUE(label_builder.Finish(&label_array).ok());
+        auto nested = arrow::StructArray::Make({label_array, variants.value()->field(1)},
+                                               nested_field->type()->fields())
+                          .ValueOrDie();
+        return std::shared_ptr(
+            arrow::StructArray::Make({variants.value()->field(0), nested}, schema->fields())
+                .ValueOrDie());
+    };
+    auto nested_typed_object = [](const std::shared_ptr& converter) {
+        const auto& physical_nested = static_cast(
+            *converter->GetPhysicalSchema()->GetFieldByName("nested")->type());
+        const auto& physical_variant = static_cast(
+            *physical_nested.GetFieldByName("payload")->type());
+        return std::static_pointer_cast(
+            physical_variant.GetFieldByName("typed_value")->type());
+    };
+
+    std::vector> first_samples = {build_nested_batch(
+        {"first", "second"}, {R"({"legacy":"a","stable":1})", R"({"legacy":"b","stable":2})"})};
+    ASSERT_OK_AND_ASSIGN(auto first, factory->CreateConverter("parquet", first_samples));
+    auto first_typed = nested_typed_object(first);
+    ASSERT_TRUE(ShreddedTypedValue(*first_typed, "legacy")->Equals(*arrow::utf8()));
+    ASSERT_TRUE(ShreddedTypedValue(*first_typed, "stable")->Equals(*arrow::int64()));
+    ASSERT_OK(factory->OnFileCompleted(first));
+
+    std::vector> second_samples = {build_nested_batch(
+        {"third", "fourth"},
+        {R"({"emerging":true,"stable":3})", R"({"emerging":false,"stable":4})"})};
+    ASSERT_OK_AND_ASSIGN(auto second, factory->CreateConverter("parquet", second_samples));
+    auto second_typed = nested_typed_object(second);
+    ASSERT_TRUE(ShreddedTypedValue(*second_typed, "emerging")->Equals(*arrow::boolean()));
+    ASSERT_TRUE(ShreddedTypedValue(*second_typed, "legacy")->Equals(*arrow::utf8()));
+    ASSERT_TRUE(ShreddedTypedValue(*second_typed, "stable")->Equals(*arrow::int64()));
+    ASSERT_OK(factory->OnFileCompleted(second));
+}
+
 TEST_F(VariantShreddingWritePlanFactoryTest, InferredSchemaWithoutSamples) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          MakeOptions({{"variant.inferShreddingSchema", "true"}}));
     auto factory = VariantShreddingWritePlanFactory::Create(options, schema_, pool_);
-    // With no useful samples the file stays unshredded.
+    // With no useful samples the complete physical plan keeps the Variant untyped.
     ASSERT_OK_AND_ASSIGN(std::shared_ptr converter,
                          factory->CreateConverter("parquet", {}));
-    ASSERT_EQ(converter, nullptr);
+    ASSERT_NE(converter, nullptr);
+    const auto& physical_type = static_cast(
+        *converter->GetPhysicalSchema()->GetFieldByName("v")->type());
+    ASSERT_EQ(physical_type.GetFieldByName("typed_value"), nullptr);
+}
+
+TEST_F(VariantShreddingWritePlanFactoryTest, MultipleVariantFieldsInferredIndependently) {
+    std::vector fields = {DataField(1, arrow::field("id", arrow::int32())),
+                                     DataField(2, VariantTypeUtils::ToArrowField("v1")),
+                                     DataField(3, VariantTypeUtils::ToArrowField("v2"))};
+    auto schema = DataField::ConvertDataFieldsToArrowSchema(fields);
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         MakeOptions({{"variant.inferShreddingSchema", "true"},
+                                      {"variant.shredding.inferenceMode", "adaptive"},
+                                      {"variant.shredding.maxInferBufferRow", "2"},
+                                      {"variant.shredding.adaptive.maxInferBufferRow", "2"},
+                                      {"variant.shredding.minFieldCardinalityRatio", "0.4"},
+                                      {"variant.shredding.adaptive.retentionRatio", "0.2"}}));
+    auto factory = VariantShreddingWritePlanFactory::Create(options, schema, pool_);
+
+    auto v1 = VariantTestData::BuildVariantBatch(
+        schema->field(0), schema->field(1), {R"({"name":"Alice"})", R"({"name":"Bob"})"}, pool_);
+    ASSERT_TRUE(v1.ok()) << v1.status().ToString();
+    auto v2 = VariantTestData::BuildVariantBatch(schema->field(0), schema->field(2),
+                                                 {R"({"age":30})", R"({"age":25})"}, pool_);
+    ASSERT_TRUE(v2.ok()) << v2.status().ToString();
+    std::shared_ptr batch =
+        arrow::StructArray::Make({v1.value()->field(0), v1.value()->field(1), v2.value()->field(1)},
+                                 schema->fields())
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(auto converter, factory->CreateConverter("parquet", {batch}));
+
+    const auto& v1_physical = static_cast(
+        *converter->GetPhysicalSchema()->GetFieldByName("v1")->type());
+    const auto& v1_typed =
+        static_cast(*v1_physical.GetFieldByName("typed_value")->type());
+    ASSERT_TRUE(ShreddedTypedValue(v1_typed, "name")->Equals(*arrow::utf8()));
+
+    const auto& v2_physical = static_cast(
+        *converter->GetPhysicalSchema()->GetFieldByName("v2")->type());
+    const auto& v2_typed =
+        static_cast(*v2_physical.GetFieldByName("typed_value")->type());
+    ASSERT_TRUE(ShreddedTypedValue(v2_typed, "age")->Equals(*arrow::int64()));
+    ASSERT_OK(factory->OnFileCompleted(converter));
+
+    auto next_v1 = VariantTestData::BuildVariantBatch(
+        schema->field(0), schema->field(1),
+        {R"({"emerging":true,"name":"Carol"})", R"({"emerging":false,"name":"Dave"})"}, pool_);
+    ASSERT_TRUE(next_v1.ok()) << next_v1.status().ToString();
+    auto next_v2 = VariantTestData::BuildVariantBatch(schema->field(0), schema->field(2),
+                                                      {R"({"age":40})", R"({"age":45})"}, pool_);
+    ASSERT_TRUE(next_v2.ok()) << next_v2.status().ToString();
+    std::shared_ptr next_batch =
+        arrow::StructArray::Make(
+            {next_v1.value()->field(0), next_v1.value()->field(1), next_v2.value()->field(1)},
+            schema->fields())
+            .ValueOrDie();
+    ASSERT_OK_AND_ASSIGN(auto adaptive, factory->CreateConverter("parquet", {next_batch}));
+    const auto& adaptive_v1 = static_cast(
+        *adaptive->GetPhysicalSchema()->GetFieldByName("v1")->type());
+    const auto& adaptive_v1_typed =
+        static_cast(*adaptive_v1.GetFieldByName("typed_value")->type());
+    ASSERT_TRUE(ShreddedTypedValue(adaptive_v1_typed, "emerging")->Equals(*arrow::boolean()));
+    ASSERT_TRUE(ShreddedTypedValue(adaptive_v1_typed, "name")->Equals(*arrow::utf8()));
+    const auto& adaptive_v2 = static_cast(
+        *adaptive->GetPhysicalSchema()->GetFieldByName("v2")->type());
+    const auto& adaptive_v2_typed =
+        static_cast(*adaptive_v2.GetFieldByName("typed_value")->type());
+    ASSERT_TRUE(ShreddedTypedValue(adaptive_v2_typed, "age")->Equals(*arrow::int64()));
+    ASSERT_OK(factory->OnFileCompleted(adaptive));
 }
 
 TEST_F(VariantShreddingWritePlanFactoryTest, SharedWidthBudgetAcrossColumns) {
@@ -236,11 +550,16 @@ TEST_F(VariantShreddingWritePlanFactoryTest, NestedVariantInsideStruct) {
                                  {schema->field(0), schema->field(1)})
             .ValueOrDie();
 
-    // Sampling sees no usable value: the file stays unshredded.
+    // Sampling sees no usable value: the complete physical plan keeps the Variant untyped.
     std::vector> garbage_samples = {garbage_batch};
     ASSERT_OK_AND_ASSIGN(std::shared_ptr unshredded_converter,
                          factory->CreateConverter("parquet", garbage_samples));
-    ASSERT_EQ(unshredded_converter, nullptr);
+    ASSERT_NE(unshredded_converter, nullptr);
+    const auto& unshredded_struct = static_cast(
+        *unshredded_converter->GetPhysicalSchema()->GetFieldByName("s")->type());
+    const auto& unshredded_variant =
+        static_cast(*unshredded_struct.GetFieldByName("nv")->type());
+    ASSERT_EQ(unshredded_variant.GetFieldByName("typed_value"), nullptr);
 
     // Conversion with the previously inferred plan shreds the row to null instead of failing.
     auto c_garbage_batch = std::make_unique();
@@ -272,7 +591,8 @@ TEST_F(VariantShreddingWritePlanFactoryTest, ConfiguredSchemaMatchingNoColumn) {
     ASSERT_TRUE(factory->ShouldCreateWritePlan());
     ASSERT_OK_AND_ASSIGN(std::shared_ptr converter,
                          factory->CreateConverter("parquet", {}));
-    ASSERT_EQ(converter, nullptr);
+    ASSERT_NE(converter, nullptr);
+    ASSERT_TRUE(converter->GetPhysicalSchema()->Equals(*schema_));
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp
index 11823d5b..209a08fa 100644
--- a/src/paimon/common/defs.cpp
+++ b/src/paimon/common/defs.cpp
@@ -35,6 +35,7 @@ const char Options::BUCKET_KEY[] = "bucket-key";
 const char Options::FILE_FORMAT[] = "file.format";
 const char Options::FILE_SYSTEM[] = "file-system";
 const char Options::TARGET_FILE_SIZE[] = "target-file-size";
+const char Options::TARGET_FILE_ROW_NUM[] = "target-file-row-num";
 const char Options::BLOB_TARGET_FILE_SIZE[] = "blob.target-file-size";
 const char Options::BLOB_SPLIT_BY_FILE_SIZE[] = "blob.split-by-file-size";
 const char Options::PAGE_SIZE[] = "page-size";
@@ -110,12 +111,17 @@ const char Options::MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY[] =
 const char Options::VARIANT_SHREDDING_SCHEMA[] = "variant.shreddingSchema";
 const char Options::PARQUET_VARIANT_SHREDDING_SCHEMA[] = "parquet.variant.shreddingSchema";
 const char Options::VARIANT_INFER_SHREDDING_SCHEMA[] = "variant.inferShreddingSchema";
+const char Options::VARIANT_SHREDDING_INFERENCE_MODE[] = "variant.shredding.inferenceMode";
 const char Options::VARIANT_SHREDDING_MAX_SCHEMA_WIDTH[] = "variant.shredding.maxSchemaWidth";
 const char Options::VARIANT_SHREDDING_MAX_SCHEMA_DEPTH[] = "variant.shredding.maxSchemaDepth";
 const char Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO[] =
     "variant.shredding.minFieldCardinalityRatio";
 const char Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW[] =
     "variant.shredding.maxInferBufferRow";
+const char Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW[] =
+    "variant.shredding.adaptive.maxInferBufferRow";
+const char Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO[] =
+    "variant.shredding.adaptive.retentionRatio";
 const char Options::BLOB_AS_DESCRIPTOR[] = "blob-as-descriptor";
 const char Options::BLOB_FIELD[] = "blob-field";
 const char Options::BLOB_DESCRIPTOR_FIELD[] = "blob-descriptor-field";
diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp
index 249b431d..e6d6de79 100644
--- a/src/paimon/core/append/append_only_writer.cpp
+++ b/src/paimon/core/append/append_only_writer.cpp
@@ -19,6 +19,7 @@
 #include "paimon/core/append/append_only_writer.h"
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -53,14 +54,13 @@
 
 namespace paimon {
 
-AppendOnlyWriter::AppendOnlyWriter(
-    const CoreOptions& options, int64_t schema_id,
-    const std::shared_ptr& write_schema,
-    const std::optional>& write_cols, int64_t max_sequence_number,
-    const std::shared_ptr& path_factory,
-    const std::shared_ptr& compact_manager,
-    const std::shared_ptr& shredding_context,
-    const std::shared_ptr& memory_pool)
+AppendOnlyWriter::AppendOnlyWriter(const CoreOptions& options, int64_t schema_id,
+                                   const std::shared_ptr& write_schema,
+                                   const std::optional>& write_cols,
+                                   int64_t max_sequence_number,
+                                   const std::shared_ptr& path_factory,
+                                   const std::shared_ptr& compact_manager,
+                                   const std::shared_ptr& memory_pool)
     : options_(options),
       schema_id_(schema_id),
       write_schema_(write_schema),
@@ -69,8 +69,7 @@ AppendOnlyWriter::AppendOnlyWriter(
       path_factory_(path_factory),
       compact_manager_(compact_manager),
       memory_pool_(memory_pool),
-      metrics_(std::make_shared()),
-      shredding_context_(shredding_context) {}
+      metrics_(std::make_shared()) {}
 
 AppendOnlyWriter::~AppendOnlyWriter() = default;
 
@@ -189,16 +188,20 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingRowWrit
     }
 
     // No BLOB fields, or all BLOB fields are inline and no .blob files are needed.
+    PAIMON_ASSIGN_OR_RAISE(WriterFactory writer_factory,
+                           GetDataFileWriterFactory(write_schema_, write_cols_));
     return std::make_unique>>(
-        options_.GetTargetFileSize(/*has_primary_key=*/false),
-        GetDataFileWriterFactory(write_schema_, write_cols_));
+        options_.GetTargetFileSize(/*has_primary_key=*/false), options_.GetTargetFileRowNum(),
+        writer_factory);
 }
 
-AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetDataFileWriterFactory(
+Result AppendOnlyWriter::GetDataFileWriterFactory(
     const std::shared_ptr& schema,
     const std::optional>& write_cols) const {
-    if (auto plan_factory = ShreddingWritePlanFactories::SelectActive(
-            options_, schema, shredding_context_, memory_pool_)) {
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr plan_factory,
+        ShreddingWritePlanFactories::SelectActive(options_, schema, memory_pool_));
+    if (plan_factory != nullptr) {
         return std::make_shared(
             options_, schema_id_, schema, write_cols, seq_num_counter_, FileSource::Append(),
             path_factory_, plan_factory, memory_pool_);
@@ -237,17 +240,21 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri
         auto single_blob_file_writer_factory =
             GetBlobFileWriterFactory(single_field_schema, write_cols);
         return std::make_unique>>(
-            options_.GetBlobTargetFileSize(), single_blob_file_writer_factory);
+            options_.GetBlobTargetFileSize(),
+            /*target_file_row_num=*/std::numeric_limits::max(),
+            single_blob_file_writer_factory);
     };
 
     WriterFactory main_writer_factory;
     if (schemas.main_schema->num_fields() > 0) {
-        main_writer_factory =
-            GetDataFileWriterFactory(schemas.main_schema, schemas.main_schema->field_names());
+        PAIMON_ASSIGN_OR_RAISE(
+            main_writer_factory,
+            GetDataFileWriterFactory(schemas.main_schema, schemas.main_schema->field_names()));
     }
     return std::make_unique(
-        options_.GetTargetFileSize(/*has_primary_key=*/false), main_writer_factory, blob_schema,
-        blob_writer_creator, arrow::struct_(write_schema_->fields()), inline_fields);
+        options_.GetTargetFileSize(/*has_primary_key=*/false), options_.GetTargetFileRowNum(),
+        main_writer_factory, blob_schema, blob_writer_creator,
+        arrow::struct_(write_schema_->fields()), inline_fields);
 }
 
 Status AppendOnlyWriter::Sync() {
diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h
index a0229b0b..eb4c2de6 100644
--- a/src/paimon/core/append/append_only_writer.h
+++ b/src/paimon/core/append/append_only_writer.h
@@ -44,7 +44,6 @@ class Schema;
 namespace paimon {
 
 class CommitIncrement;
-class MapSharedShreddingContext;
 class RecordBatch;
 template 
 class RollingFileWriter;
@@ -63,7 +62,6 @@ class AppendOnlyWriter : public BatchWriter {
                      int64_t max_sequence_number,
                      const std::shared_ptr& path_factory,
                      const std::shared_ptr& compact_manager,
-                     const std::shared_ptr& shredding_context,
                      const std::shared_ptr& memory_pool);
 
     ~AppendOnlyWriter() override;
@@ -106,7 +104,7 @@ class AppendOnlyWriter : public BatchWriter {
     Result DrainIncrement();
     Status Flush(bool wait_for_latest_compaction, bool forced_full_compaction);
 
-    WriterFactory GetDataFileWriterFactory(
+    Result GetDataFileWriterFactory(
         const std::shared_ptr& schema,
         const std::optional>& write_cols) const;
 
@@ -136,11 +134,6 @@ class AppendOnlyWriter : public BatchWriter {
     std::unique_ptr>> writer_;
     std::set inline_descriptor_fields_;
     std::set inline_view_fields_;
-
-    // ---- Shared-shredding MAP support ----
-    /// Cross-file context for K adaptation and shredding column tracking.
-    /// nullptr when no shared-shredding MAP columns are configured.
-    std::shared_ptr shredding_context_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/append/append_only_writer_test.cpp b/src/paimon/core/append/append_only_writer_test.cpp
index 8488efff..068398b3 100644
--- a/src/paimon/core/append/append_only_writer_test.cpp
+++ b/src/paimon/core/append/append_only_writer_test.cpp
@@ -40,7 +40,6 @@
 #include "paimon/common/data/blob_descriptor.h"
 #include "paimon/common/data/blob_utils.h"
 #include "paimon/common/data/blob_view_struct.h"
-#include "paimon/common/data/shredding/map_shared_shredding_context.h"
 #include "paimon/common/data/shredding/map_shared_shredding_utils.h"
 #include "paimon/common/data/shredding/map_shredding_defs.h"
 #include "paimon/common/fs/external_path_provider.h"
@@ -305,10 +304,8 @@ class AppendOnlyWriterTest : public testing::Test {
         // Deserialize and compare the per-field shared-shredding map metadata.
         auto metadata = file_schema->field(field_index)->metadata();
         ASSERT_NE(nullptr, metadata);
-        ASSERT_OK_AND_ASSIGN(
-            auto deserialized_meta,
-            MapSharedShreddingUtils::DeserializeMetadata(
-                metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression));
+        ASSERT_OK_AND_ASSIGN(auto deserialized_meta,
+                             MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()));
         ASSERT_EQ(expected_meta, deserialized_meta);
     }
 
@@ -319,12 +316,9 @@ class AppendOnlyWriterTest : public testing::Test {
         const std::shared_ptr& path_factory,
         const std::shared_ptr& compact_manager,
         const std::shared_ptr& memory_pool) const {
-        PAIMON_ASSIGN_OR_RAISE(
-            auto shredding_context,
-            MapSharedShreddingUtils::CreateShreddingContext(write_schema, options));
         return std::make_unique(options, schema_id, write_schema, write_cols,
                                                   max_sequence_number, path_factory,
-                                                  compact_manager, shredding_context, memory_pool);
+                                                  compact_manager, memory_pool);
     }
 
  protected:
@@ -707,8 +701,9 @@ TEST_F(AppendOnlyWriterTest, TestCompactPassesFullCompactionFlag) {
 }
 
 TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) {
-    auto options =
-        CreateOptions({{Options::FILE_FORMAT, "orc"}, {Options::MANIFEST_FORMAT, "orc"}});
+    auto options = CreateOptions({{Options::FILE_FORMAT, "orc"},
+                                  {Options::MANIFEST_FORMAT, "orc"},
+                                  {Options::TARGET_FILE_ROW_NUM, "1"}});
     auto dir = UniqueTestDirectory::Create();
     ASSERT_TRUE(dir);
     auto path_factory = CreatePathFactory(dir->Str(), "orc", options);
@@ -730,16 +725,18 @@ TEST_F(AppendOnlyWriterTest, TestWriteWithSingleBlobField) {
     ASSERT_TRUE(blob_builder.Append("bb", 2).ok());
     auto blob_array = blob_builder.Finish().ValueOrDie();
 
-    ASSERT_OK(writer->Write(CreateStructBatch(schema, {int_array, blob_array})));
+    ASSERT_OK(writer->Write(
+        CreateStructBatch(schema, {int_array->Slice(0, 1), blob_array->Slice(0, 1)})));
+    ASSERT_OK(writer->Write(
+        CreateStructBatch(schema, {int_array->Slice(1, 1), blob_array->Slice(1, 1)})));
     ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true));
 
-    ASSERT_EQ(inc.GetNewFilesIncrement().NewFiles().size(), 2);
-    const auto& main_file = inc.GetNewFilesIncrement().NewFiles()[0];
-    const auto& blob_file = inc.GetNewFilesIncrement().NewFiles()[1];
-    ASSERT_TRUE(
-        options.GetFileSystem()->Exists(path_factory->ToPath(main_file->file_name)).value());
-    ASSERT_TRUE(
-        options.GetFileSystem()->Exists(path_factory->ToPath(blob_file->file_name)).value());
+    const auto& files = inc.GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(files.size(), 4);
+    for (const auto& file : files) {
+        ASSERT_EQ(file->row_count, 1);
+        ASSERT_TRUE(options.GetFileSystem()->Exists(path_factory->ToPath(file->file_name)).value());
+    }
     ASSERT_OK(writer->Close());
 }
 
@@ -977,6 +974,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapFieldContent) {
     auto options = CreateOptions({
         {Options::FILE_FORMAT, format},
         {Options::MANIFEST_FORMAT, format},
+        {Options::TARGET_FILE_ROW_NUM, "2"},
         {"fields.tags.map.storage-layout", "shared-shredding"},
         {"fields.tags.map.shared-shredding.max-columns", "3"},
         {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -1108,6 +1106,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF
     auto options = CreateOptions({
         {Options::FILE_FORMAT, format},
         {Options::MANIFEST_FORMAT, format},
+        {Options::TARGET_FILE_ROW_NUM, "2"},
         {"fields.tags.map.storage-layout", "shared-shredding"},
         {"fields.tags.map.shared-shredding.max-columns", "3"},
         {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -1134,21 +1133,39 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF
         [2, null]
     ])");
     ASSERT_OK(writer->Write(std::move(null_batch)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement null_inc, writer->PrepareCommit(/*wait_compaction=*/true));
-    ASSERT_EQ(1, null_inc.GetNewFilesIncrement().NewFiles().size());
-    std::string null_file_path =
-        path_factory->ToPath(null_inc.GetNewFilesIncrement().NewFiles()[0]->file_name);
+
+    auto empty_batch = CreateBatch(logical_schema, R"([
+        [3, []],
+        [4, []]
+    ])");
+    ASSERT_OK(writer->Write(std::move(empty_batch)));
+
+    auto null_value_batch = CreateBatch(logical_schema, R"([
+        [5, [["a", null]]],
+        [6, [["b", null]]],
+        [7, [["c", 7], ["d", null]]]
+    ])");
+    ASSERT_OK(writer->Write(std::move(null_value_batch)));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement inc, writer->PrepareCommit(/*wait_compaction=*/true));
+    ASSERT_OK(writer->Close());
+
+    const auto& files = inc.GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(3, files.size());
+    std::string null_file_path = path_factory->ToPath(files[0]->file_name);
+    std::string empty_file_path = path_factory->ToPath(files[1]->file_name);
+    std::string null_value_file_path = path_factory->ToPath(files[2]->file_name);
 
     std::map first_file_k = {{"tags", 3}};
     ASSERT_OK_AND_ASSIGN(auto first_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
                                                 logical_schema, first_file_k));
+    auto first_physical_type = arrow::struct_(first_schema->fields());
+
     MapSharedShreddingFieldMeta empty_meta;
     empty_meta.num_columns = 3;
     empty_meta.max_row_width = 0;
     CheckShreddingFileSchema(null_file_path, format, first_schema, /*field_index=*/1, empty_meta,
                              options.GetFileCompression());
 
-    auto first_physical_type = arrow::struct_(first_schema->fields());
     std::shared_ptr expected_null_array;
     ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(first_physical_type, {R"([
         [1, null],
@@ -1158,29 +1175,16 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF
                     .ok());
     CheckFileContent(null_file_path, format, expected_null_array);
 
-    auto empty_batch = CreateBatch(logical_schema, R"([
-        [3, []],
-        [4, []]
-    ])");
-    ASSERT_OK(writer->Write(std::move(empty_batch)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement empty_inc,
-                         writer->PrepareCommit(/*wait_compaction=*/true));
-    ASSERT_EQ(1, empty_inc.GetNewFilesIncrement().NewFiles().size());
-    std::string empty_file_path =
-        path_factory->ToPath(empty_inc.GetNewFilesIncrement().NewFiles()[0]->file_name);
-
-    // Previous file observed max_row_width=0, but the next file must still keep at least one
-    // physical value column so shared-shredding never produces a K=0 schema.
-    std::map second_file_k = {{"tags", 1}};
-    ASSERT_OK_AND_ASSIGN(auto second_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
-                                                 logical_schema, second_file_k));
+    std::map subsequent_file_k = {{"tags", 1}};
+    ASSERT_OK_AND_ASSIGN(auto subsequent_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                                                     logical_schema, subsequent_file_k));
+    auto subsequent_physical_type = arrow::struct_(subsequent_schema->fields());
     empty_meta.num_columns = 1;
-    CheckShreddingFileSchema(empty_file_path, format, second_schema, /*field_index=*/1, empty_meta,
-                             options.GetFileCompression());
+    CheckShreddingFileSchema(empty_file_path, format, subsequent_schema, /*field_index=*/1,
+                             empty_meta, options.GetFileCompression());
 
-    auto second_physical_type = arrow::struct_(second_schema->fields());
     std::shared_ptr expected_empty_array;
-    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(second_physical_type, {R"([
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(subsequent_physical_type, {R"([
         [3, [[-1], null, null]],
         [4, [[-1], null, null]]
     ])"},
@@ -1188,29 +1192,17 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF
                     .ok());
     CheckFileContent(empty_file_path, format, expected_empty_array);
 
-    auto null_value_batch = CreateBatch(logical_schema, R"([
-        [5, [["a", null]]],
-        [6, [["b", null]]],
-        [7, [["c", 7], ["d", null]]]
-    ])");
-    ASSERT_OK(writer->Write(std::move(null_value_batch)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement null_value_inc,
-                         writer->PrepareCommit(/*wait_compaction=*/true));
-    ASSERT_EQ(1, null_value_inc.GetNewFilesIncrement().NewFiles().size());
-    std::string null_value_file_path =
-        path_factory->ToPath(null_value_inc.GetNewFilesIncrement().NewFiles()[0]->file_name);
-
     MapSharedShreddingFieldMeta null_value_meta;
     null_value_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}, {"d", 3}};
     null_value_meta.field_to_columns = {{0, {0}}, {1, {0}}, {2, {0}}};
     null_value_meta.overflow_field_set = {3};
     null_value_meta.num_columns = 1;
     null_value_meta.max_row_width = 2;
-    CheckShreddingFileSchema(null_value_file_path, format, second_schema, /*field_index=*/1,
+    CheckShreddingFileSchema(null_value_file_path, format, subsequent_schema, /*field_index=*/1,
                              null_value_meta, options.GetFileCompression());
 
     std::shared_ptr expected_null_value_array;
-    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(second_physical_type, {R"([
+    ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(subsequent_physical_type, {R"([
         [5, [[0], null, null]],
         [6, [[1], null, null]],
         [7, [[2], 7, [[3, null]]]]
@@ -1218,7 +1210,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapAllNullThenAllEmptyF
                                                                  &expected_null_value_array)
                     .ok());
     CheckFileContent(null_value_file_path, format, expected_null_value_array);
-
     ASSERT_OK(writer->Close());
 }
 
@@ -1362,11 +1353,12 @@ TEST_P(AppendOnlyWriterShreddingTest, TestWriteSharedShreddingMapWithLruPlacemen
     CheckFileContent(data_file_path, format, expected_array);
 }
 
-TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFiles) {
+TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossRollingFiles) {
     std::string format = GetFormat();
     auto options = CreateOptions({
         {Options::FILE_FORMAT, format},
         {Options::MANIFEST_FORMAT, format},
+        {Options::TARGET_FILE_ROW_NUM, "1"},
         {"fields.tags.map.storage-layout", "shared-shredding"},
         {"fields.tags.map.shared-shredding.max-columns", "10"},
         {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -1394,20 +1386,29 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil
         [2, [["c", 30], ["a", 40], ["b", 50]]]
     ])");
     ASSERT_OK(writer->Write(std::move(batch1)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement inc1, writer->PrepareCommit(/*wait_compaction=*/true));
-    ASSERT_EQ(1, inc1.GetNewFilesIncrement().NewFiles().size());
 
-    std::string file1_path =
-        path_factory->ToPath(inc1.GetNewFilesIncrement().NewFiles()[0]->file_name);
+    // File 2 adapts to file 1's max row width.
+    auto batch2 = CreateBatch(logical_schema, R"([
+        [3, [["x", 100], ["y", 200], ["z", 300], ["w", 400], ["v", 500]]]
+    ])");
+    ASSERT_OK(writer->Write(std::move(batch2)));
+
+    // File 3 sees both preceding files in the same rolling session.
+    auto batch3 = CreateBatch(logical_schema, R"([
+        [4, [["p", 1000], ["q", 2000], ["r", 3000], ["s", 4000]]]
+    ])");
+    ASSERT_OK(writer->Write(std::move(batch3)));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement increment,
+                         writer->PrepareCommit(/*wait_compaction=*/true));
+    const auto& files = increment.GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(3, files.size());
+    std::string file1_path = path_factory->ToPath(files[0]->file_name);
+    std::string file2_path = path_factory->ToPath(files[1]->file_name);
+    std::string file3_path = path_factory->ToPath(files[2]->file_name);
 
-    // File 1 should have K=10 (first file uses K_max).
     std::map column_to_k_file1 = {{"tags", 10}};
     ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema(
                                                 logical_schema, column_to_k_file1));
-    // Verify file1 physical schema has 10 columns.
-    auto struct_type1 = std::static_pointer_cast(phys_schema1->field(1)->type());
-    ASSERT_EQ(12, struct_type1->num_fields());  // mapping + 10 cols + overflow
-
     MapSharedShreddingFieldMeta meta1;
     meta1.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}};
     meta1.field_to_columns = {{0, {0, 1}}, {1, {1, 2}}, {2, {0}}};
@@ -1417,25 +1418,9 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil
     CheckShreddingFileSchema(file1_path, format, phys_schema1, /*field_index=*/1, meta1,
                              compression);
 
-    // --- File 2: K should adapt to min(max_window=3, K_max=10) = 3 ---
-    // Write 5 keys → 3 fit in columns, 2 overflow.
-    auto batch2 = CreateBatch(logical_schema, R"([
-        [3, [["x", 100], ["y", 200], ["z", 300], ["w", 400], ["v", 500]]]
-    ])");
-    ASSERT_OK(writer->Write(std::move(batch2)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement inc2, writer->PrepareCommit(/*wait_compaction=*/true));
-    ASSERT_EQ(1, inc2.GetNewFilesIncrement().NewFiles().size());
-
-    std::string file2_path =
-        path_factory->ToPath(inc2.GetNewFilesIncrement().NewFiles()[0]->file_name);
-
-    // File 2 should have K=3 (adapted from file1's max_row_width=3).
     std::map column_to_k_file2 = {{"tags", 3}};
     ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema(
                                                 logical_schema, column_to_k_file2));
-    auto struct_type2 = std::static_pointer_cast(phys_schema2->field(1)->type());
-    ASSERT_EQ(5, struct_type2->num_fields());  // mapping + 3 cols + overflow
-
     MapSharedShreddingFieldMeta meta2;
     meta2.name_to_id = {{"x", 0}, {"y", 1}, {"z", 2}, {"w", 3}, {"v", 4}};
     meta2.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}};
@@ -1444,8 +1429,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil
     meta2.max_row_width = 5;
     CheckShreddingFileSchema(file2_path, format, phys_schema2, /*field_index=*/1, meta2,
                              compression);
-
-    // Verify data: 5 keys, K=3, so w and v overflow.
     auto physical_type2 = arrow::struct_(phys_schema2->fields());
     std::shared_ptr expected_array2;
     ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type2, {R"([
@@ -1455,26 +1438,9 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil
                     .ok());
     CheckFileContent(file2_path, format, expected_array2);
 
-    // --- File 3: K should adapt to min(max_window=max(3,5)=5, K_max=10) = 5 ---
-    // File2 reported max_row_width=5, so window now has [3, 5], max=5.
-    // Write 4 keys → all fit in K=5, no overflow.
-    auto batch3 = CreateBatch(logical_schema, R"([
-        [4, [["p", 1000], ["q", 2000], ["r", 3000], ["s", 4000]]]
-    ])");
-    ASSERT_OK(writer->Write(std::move(batch3)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement inc3, writer->PrepareCommit(/*wait_compaction=*/true));
-    ASSERT_EQ(1, inc3.GetNewFilesIncrement().NewFiles().size());
-
-    std::string file3_path =
-        path_factory->ToPath(inc3.GetNewFilesIncrement().NewFiles()[0]->file_name);
-
-    // File 3 should have K=5 (window max grew from file2's max_row_width=5).
     std::map column_to_k_file3 = {{"tags", 5}};
     ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema(
                                                 logical_schema, column_to_k_file3));
-    auto struct_type3 = std::static_pointer_cast(phys_schema3->field(1)->type());
-    ASSERT_EQ(7, struct_type3->num_fields());  // mapping + 5 cols + overflow
-
     MapSharedShreddingFieldMeta meta3;
     meta3.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}, {"s", 3}};
     meta3.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}, {3, {3}}};
@@ -1482,8 +1448,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil
     meta3.max_row_width = 4;
     CheckShreddingFileSchema(file3_path, format, phys_schema3, /*field_index=*/1, meta3,
                              compression);
-
-    // Verify data: 4 keys fit in K=5, col4 unused, no overflow.
     auto physical_type3 = arrow::struct_(phys_schema3->fields());
     std::shared_ptr expected_array3;
     ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type3, {R"([
@@ -1496,7 +1460,7 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapKAdaptationAcrossFil
     ASSERT_OK(writer->Close());
 }
 
-TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapUsesInitialContextForFirstFile) {
+TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapFirstFileUsesConfiguredMaxK) {
     std::string format = GetFormat();
     auto options = CreateOptions({
         {Options::FILE_FORMAT, format},
@@ -1516,12 +1480,9 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapUsesInitialContextFo
     ASSERT_TRUE(dir);
     auto path_factory = CreatePathFactory(dir->Str(), format, options);
 
-    auto initial_context =
-        std::make_shared(std::map{{"tags", 10}});
-    initial_context->ReportFileStats("tags", 2);
     auto writer = std::make_unique(
         options, /*schema_id=*/0, logical_schema, /*write_cols=*/std::nullopt,
-        /*max_sequence_number=*/-1, path_factory, compact_manager_, initial_context, memory_pool_);
+        /*max_sequence_number=*/-1, path_factory, compact_manager_, memory_pool_);
 
     auto batch = CreateBatch(logical_schema, R"([
         [1, [["a", 10], ["b", 20], ["c", 30]]]
@@ -1533,14 +1494,13 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapUsesInitialContextFo
     std::string file_path =
         path_factory->ToPath(inc.GetNewFilesIncrement().NewFiles()[0]->file_name);
 
-    std::map column_to_k = {{"tags", 2}};
+    std::map column_to_k = {{"tags", 10}};
     ASSERT_OK_AND_ASSIGN(auto physical_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
                                                    logical_schema, column_to_k));
     MapSharedShreddingFieldMeta expected_meta;
     expected_meta.name_to_id = {{"a", 0}, {"b", 1}, {"c", 2}};
-    expected_meta.field_to_columns = {{0, {0}}, {1, {1}}};
-    expected_meta.overflow_field_set = {2};
-    expected_meta.num_columns = 2;
+    expected_meta.field_to_columns = {{0, {0}}, {1, {1}}, {2, {2}}};
+    expected_meta.num_columns = 10;
     expected_meta.max_row_width = 3;
     CheckShreddingFileSchema(file_path, format, physical_schema, /*field_index=*/1, expected_meta,
                              options.GetFileCompression());
@@ -1548,7 +1508,8 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapUsesInitialContextFo
     auto physical_type = arrow::struct_(physical_schema->fields());
     std::shared_ptr expected_array;
     ASSERT_TRUE(arrow::ipc::internal::json::ChunkedArrayFromJSON(physical_type, {R"([
-        [1, [[0, 1], 10, 20, [[2, 30]]]]
+        [1, [[0, 1, 2, -1, -1, -1, -1, -1, -1, -1],
+             10, 20, 30, null, null, null, null, null, null, null, null]]
     ])"},
                                                                  &expected_array)
                     .ok());
@@ -1557,12 +1518,13 @@ TEST_P(AppendOnlyWriterShreddingTest, TestSharedShreddingMapUsesInitialContextFo
     ASSERT_OK(writer->Close());
 }
 
-TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKAdaptation) {
+TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsAdaptAcrossRollingFiles) {
     std::string format = GetFormat();
     // Two shared-shredding MAP fields with different initial K: tags(K=8), attrs(K=4).
     auto options = CreateOptions({
         {Options::FILE_FORMAT, format},
         {Options::MANIFEST_FORMAT, format},
+        {Options::TARGET_FILE_ROW_NUM, "1"},
         {"fields.tags.map.storage-layout", "shared-shredding"},
         {"fields.tags.map.shared-shredding.max-columns", "8"},
         {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -1596,17 +1558,27 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA
         [2, [["a", 30]],            [["x", "v2"]]]
     ])");
     ASSERT_OK(writer->Write(std::move(batch1)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement inc1, writer->PrepareCommit(/*wait_compaction=*/true));
-    ASSERT_EQ(1, inc1.GetNewFilesIncrement().NewFiles().size());
 
-    std::string file1_path =
-        path_factory->ToPath(inc1.GetNewFilesIncrement().NewFiles()[0]->file_name);
+    auto batch2 = CreateBatch(logical_schema, R"([
+        [3, [["c", 100], ["d", 200], ["e", 300]], [["p", "a1"], ["q", "a2"], ["r", "a3"]]]
+    ])");
+    ASSERT_OK(writer->Write(std::move(batch2)));
+
+    auto batch3 = CreateBatch(logical_schema, R"([
+        [4, [["f", 400], ["g", 500]], [["s", "b1"], ["t", "b2"]]]
+    ])");
+    ASSERT_OK(writer->Write(std::move(batch3)));
+    ASSERT_OK_AND_ASSIGN(CommitIncrement increment,
+                         writer->PrepareCommit(/*wait_compaction=*/true));
+    const auto& files = increment.GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(3, files.size());
+    std::string file1_path = path_factory->ToPath(files[0]->file_name);
+    std::string file2_path = path_factory->ToPath(files[1]->file_name);
+    std::string file3_path = path_factory->ToPath(files[2]->file_name);
 
-    // Verify file1: tags K=8, attrs K=4 (first file uses K_max).
     std::map col_to_k_file1 = {{"tags", 8}, {"attrs", 4}};
     ASSERT_OK_AND_ASSIGN(auto phys_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema(
                                                 logical_schema, col_to_k_file1));
-
     MapSharedShreddingFieldMeta meta1_tags;
     meta1_tags.name_to_id = {{"a", 0}, {"b", 1}};
     meta1_tags.field_to_columns = {{0, {0}}, {1, {1}}};
@@ -1614,7 +1586,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA
     meta1_tags.max_row_width = 2;
     CheckShreddingFileSchema(file1_path, format, phys_schema1, /*field_index=*/1, meta1_tags,
                              compression);
-
     MapSharedShreddingFieldMeta meta1_attrs;
     meta1_attrs.name_to_id = {{"x", 0}};
     meta1_attrs.field_to_columns = {{0, {0}}};
@@ -1623,22 +1594,9 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA
     CheckShreddingFileSchema(file1_path, format, phys_schema1, /*field_index=*/2, meta1_attrs,
                              compression);
 
-    // --- File 2: tags K=min(2,8)=2, attrs K=min(1,4)=1 ---
-    // tags: 3 keys → 1 overflow; attrs: 3 keys → 2 overflow
-    auto batch2 = CreateBatch(logical_schema, R"([
-        [3, [["c", 100], ["d", 200], ["e", 300]], [["p", "a1"], ["q", "a2"], ["r", "a3"]]]
-    ])");
-    ASSERT_OK(writer->Write(std::move(batch2)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement inc2, writer->PrepareCommit(/*wait_compaction=*/true));
-    ASSERT_EQ(1, inc2.GetNewFilesIncrement().NewFiles().size());
-
-    std::string file2_path =
-        path_factory->ToPath(inc2.GetNewFilesIncrement().NewFiles()[0]->file_name);
-
     std::map col_to_k_file2 = {{"tags", 2}, {"attrs", 1}};
     ASSERT_OK_AND_ASSIGN(auto phys_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema(
                                                 logical_schema, col_to_k_file2));
-
     MapSharedShreddingFieldMeta meta2_tags;
     meta2_tags.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}};
     meta2_tags.field_to_columns = {{0, {0}}, {1, {1}}};
@@ -1647,7 +1605,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA
     meta2_tags.max_row_width = 3;
     CheckShreddingFileSchema(file2_path, format, phys_schema2, /*field_index=*/1, meta2_tags,
                              compression);
-
     MapSharedShreddingFieldMeta meta2_attrs;
     meta2_attrs.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}};
     meta2_attrs.field_to_columns = {{0, {0}}};
@@ -1657,18 +1614,6 @@ TEST_P(AppendOnlyWriterShreddingTest, TestMultipleSharedShreddingMapFieldsWithKA
     CheckShreddingFileSchema(file2_path, format, phys_schema2, /*field_index=*/2, meta2_attrs,
                              compression);
 
-    // --- File 3: tags K=min(max(2,3),8)=3, attrs K=min(max(1,3),4)=3 ---
-    // tags: 2 keys, fits; attrs: 2 keys, fits.
-    auto batch3 = CreateBatch(logical_schema, R"([
-        [4, [["f", 400], ["g", 500]], [["s", "b1"], ["t", "b2"]]]
-    ])");
-    ASSERT_OK(writer->Write(std::move(batch3)));
-    ASSERT_OK_AND_ASSIGN(CommitIncrement inc3, writer->PrepareCommit(/*wait_compaction=*/true));
-    ASSERT_EQ(1, inc3.GetNewFilesIncrement().NewFiles().size());
-
-    std::string file3_path =
-        path_factory->ToPath(inc3.GetNewFilesIncrement().NewFiles()[0]->file_name);
-
     std::map col_to_k_file3 = {{"tags", 3}, {"attrs", 3}};
     ASSERT_OK_AND_ASSIGN(auto phys_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema(
                                                 logical_schema, col_to_k_file3));
diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp
index e38766fa..8130dc45 100644
--- a/src/paimon/core/core_options.cpp
+++ b/src/paimon/core/core_options.cpp
@@ -229,6 +229,23 @@ class ConfigParser {
         return Status::OK();
     }
 
+    // Parse VariantShreddingInferenceMode
+    Status ParseVariantShreddingInferenceMode(VariantShreddingInferenceMode* inference_mode) const {
+        auto iter = config_map_.find(Options::VARIANT_SHREDDING_INFERENCE_MODE);
+        if (iter != config_map_.end()) {
+            std::string str = StringUtils::ToLowerCase(iter->second);
+            if (str == "per-file") {
+                *inference_mode = VariantShreddingInferenceMode::PER_FILE;
+            } else if (str == "adaptive") {
+                *inference_mode = VariantShreddingInferenceMode::ADAPTIVE;
+            } else {
+                return Status::Invalid(
+                    fmt::format("invalid variant shredding inference mode: {}", str));
+            }
+        }
+        return Status::OK();
+    }
+
     // Parse ChangelogProducer
     Status ParseChangelogProducer(ChangelogProducer* changelog_producer) const {
         auto iter = config_map_.find(Options::CHANGELOG_PRODUCER);
@@ -362,6 +379,7 @@ class ConfigParser {
 struct CoreOptions::Impl {
     int64_t page_size = 64 * 1024;
     std::optional target_file_size;
+    int64_t target_file_row_num = std::numeric_limits::max();
     std::optional blob_target_file_size;
     int64_t source_split_target_size = 128 * 1024 * 1024;
     int64_t source_split_open_file_cost = 4 * 1024 * 1024;
@@ -450,10 +468,14 @@ struct CoreOptions::Impl {
     bool row_tracking_partition_group_on_commit = true;
     bool data_evolution_enabled = false;
     bool variant_infer_shredding_schema = false;
+    VariantShreddingInferenceMode variant_shredding_inference_mode =
+        VariantShreddingInferenceMode::PER_FILE;
     int32_t variant_shredding_max_schema_width = 300;
     int32_t variant_shredding_max_schema_depth = 50;
     double variant_shredding_min_field_cardinality_ratio = 0.1;
     int32_t variant_shredding_max_infer_buffer_row = 4096;
+    int32_t variant_shredding_adaptive_max_infer_buffer_row = 256;
+    double variant_shredding_adaptive_retention_ratio = 0.05;
     bool blob_view_resolve_enabled = true;
     bool blob_as_descriptor = false;
     std::optional blob_split_by_file_size;
@@ -502,6 +524,13 @@ struct CoreOptions::Impl {
         PAIMON_RETURN_NOT_OK(parser.ParseMemorySize(Options::PAGE_SIZE, &page_size));
         // Parse target-file-size - target size of a data file
         PAIMON_RETURN_NOT_OK(parser.ParseMemorySize(Options::TARGET_FILE_SIZE, &target_file_size));
+        // Parse target-file-row-num - target rows of a newly written data file
+        PAIMON_RETURN_NOT_OK(
+            parser.Parse(Options::TARGET_FILE_ROW_NUM, &target_file_row_num));
+        if (target_file_row_num <= 0) {
+            return Status::Invalid(
+                fmt::format("{} should be at least 1", Options::TARGET_FILE_ROW_NUM));
+        }
         // Parse blob.target-file-size - target size of a blob file
         PAIMON_RETURN_NOT_OK(
             parser.ParseMemorySize(Options::BLOB_TARGET_FILE_SIZE, &blob_target_file_size));
@@ -870,6 +899,8 @@ struct CoreOptions::Impl {
         // Parse variant.inferShreddingSchema - infer the shredding schema from sampled rows
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_INFER_SHREDDING_SCHEMA,
                                                 &variant_infer_shredding_schema));
+        PAIMON_RETURN_NOT_OK(
+            parser.ParseVariantShreddingInferenceMode(&variant_shredding_inference_mode));
         // Parse variant.shredding.maxSchemaWidth - max number of shredded fields, default 300
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_SHREDDING_MAX_SCHEMA_WIDTH,
                                                    &variant_shredding_max_schema_width));
@@ -885,6 +916,12 @@ struct CoreOptions::Impl {
         // default 4096
         PAIMON_RETURN_NOT_OK(parser.Parse(Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW,
                                                    &variant_shredding_max_infer_buffer_row));
+        PAIMON_RETURN_NOT_OK(
+            parser.Parse(Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW,
+                                  &variant_shredding_adaptive_max_infer_buffer_row));
+        PAIMON_RETURN_NOT_OK(
+            parser.Parse(Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO,
+                                 &variant_shredding_adaptive_retention_ratio));
         if (variant_shredding_max_schema_width <= 0) {
             return Status::Invalid(fmt::format(
                 "The option '{}' should be positive, while input is {}",
@@ -908,6 +945,23 @@ struct CoreOptions::Impl {
                             Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW,
                             variant_shredding_max_infer_buffer_row));
         }
+        if (variant_shredding_inference_mode == VariantShreddingInferenceMode::ADAPTIVE) {
+            if (variant_shredding_adaptive_max_infer_buffer_row <= 0) {
+                return Status::Invalid(
+                    fmt::format("The option '{}' should be positive, while input is {}",
+                                Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW,
+                                variant_shredding_adaptive_max_infer_buffer_row));
+            }
+            if (variant_shredding_adaptive_retention_ratio < 0.0 ||
+                variant_shredding_adaptive_retention_ratio >
+                    variant_shredding_min_field_cardinality_ratio) {
+                return Status::Invalid(
+                    fmt::format("The option '{}' should be in the range [0, {}], while input is {}",
+                                Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO,
+                                Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO,
+                                variant_shredding_adaptive_retention_ratio));
+            }
+        }
         return Status::OK();
     }
 
@@ -1050,6 +1104,10 @@ int64_t CoreOptions::GetTargetFileSize(bool has_primary_key) const {
     return impl_->target_file_size.value();
 }
 
+int64_t CoreOptions::GetTargetFileRowNum() const {
+    return impl_->target_file_row_num;
+}
+
 int64_t CoreOptions::GetBlobTargetFileSize() const {
     if (impl_->blob_target_file_size == std::nullopt) {
         return GetTargetFileSize(/*has_primary_key=*/false);
@@ -1355,6 +1413,10 @@ bool CoreOptions::VariantInferShreddingSchemaEnabled() const {
     return impl_->variant_infer_shredding_schema;
 }
 
+VariantShreddingInferenceMode CoreOptions::GetVariantShreddingInferenceMode() const {
+    return impl_->variant_shredding_inference_mode;
+}
+
 int32_t CoreOptions::GetVariantShreddingMaxSchemaWidth() const {
     return impl_->variant_shredding_max_schema_width;
 }
@@ -1371,6 +1433,14 @@ int32_t CoreOptions::GetVariantShreddingMaxInferBufferRow() const {
     return impl_->variant_shredding_max_infer_buffer_row;
 }
 
+int32_t CoreOptions::GetVariantShreddingAdaptiveMaxInferBufferRow() const {
+    return impl_->variant_shredding_adaptive_max_infer_buffer_row;
+}
+
+double CoreOptions::GetVariantShreddingAdaptiveRetentionRatio() const {
+    return impl_->variant_shredding_adaptive_retention_ratio;
+}
+
 Result CoreOptions::GetMapStorageLayout(const std::string& field_name) const {
     std::string key = std::string(Options::FIELDS_PREFIX) + "." + field_name + "." +
                       std::string(Options::MAP_STORAGE_LAYOUT);
diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h
index 050685ce..e62a0323 100644
--- a/src/paimon/core/core_options.h
+++ b/src/paimon/core/core_options.h
@@ -27,6 +27,7 @@
 
 #include "paimon/bucket/bucket_function_type.h"
 #include "paimon/cache/cache.h"
+#include "paimon/common/data/variant/variant_defs.h"
 #include "paimon/core/options/changelog_producer.h"
 #include "paimon/core/options/compress_options.h"
 #include "paimon/core/options/external_path_strategy.h"
@@ -78,6 +79,7 @@ class PAIMON_EXPORT CoreOptions {
     int32_t GetFileCompressionZstdLevel() const;
     int64_t GetPageSize() const;
     int64_t GetTargetFileSize(bool has_primary_key) const;
+    int64_t GetTargetFileRowNum() const;
     int64_t GetBlobTargetFileSize() const;
     bool BlobSplitByFileSize() const;
     int64_t GetCompactionFileSize(bool has_primary_key) const;
@@ -151,10 +153,13 @@ class PAIMON_EXPORT CoreOptions {
     /// "parquet.variant.shreddingSchema").
     std::optional GetVariantShreddingSchema() const;
     bool VariantInferShreddingSchemaEnabled() const;
+    VariantShreddingInferenceMode GetVariantShreddingInferenceMode() const;
     int32_t GetVariantShreddingMaxSchemaWidth() const;
     int32_t GetVariantShreddingMaxSchemaDepth() const;
     double GetVariantShreddingMinFieldCardinalityRatio() const;
     int32_t GetVariantShreddingMaxInferBufferRow() const;
+    int32_t GetVariantShreddingAdaptiveMaxInferBufferRow() const;
+    double GetVariantShreddingAdaptiveRetentionRatio() const;
 
     bool DeletionVectorsEnabled() const;
     bool DeletionVectorsBitmap64() const;
diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp
index 90e905de..f8230c6f 100644
--- a/src/paimon/core/core_options_test.cpp
+++ b/src/paimon/core/core_options_test.cpp
@@ -42,6 +42,7 @@ TEST(CoreOptionsTest, TestDefaultValue) {
     ASSERT_EQ(64 * 1024L, core_options.GetPageSize());
     ASSERT_EQ(256 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/false));
     ASSERT_EQ(128 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/true));
+    ASSERT_EQ(std::numeric_limits::max(), core_options.GetTargetFileRowNum());
     ASSERT_EQ(256 * 1024 * 1024L, core_options.GetBlobTargetFileSize());
     ASSERT_TRUE(core_options.BlobSplitByFileSize());
     ASSERT_EQ(187904815, core_options.GetCompactionFileSize(/*has_primary_key=*/false));
@@ -178,6 +179,7 @@ TEST(CoreOptionsTest, TestFromMap) {
         {Options::BUCKET, "3"},
         {Options::PAGE_SIZE, "128 kb"},
         {Options::TARGET_FILE_SIZE, "512MB"},
+        {Options::TARGET_FILE_ROW_NUM, "123"},
         {Options::BLOB_TARGET_FILE_SIZE, "1G"},
         {Options::PARTITION_DEFAULT_NAME, "foo"},
         {Options::MANIFEST_TARGET_FILE_SIZE, "16MB"},
@@ -307,6 +309,7 @@ TEST(CoreOptionsTest, TestFromMap) {
     ASSERT_EQ(128 * 1024L, core_options.GetPageSize());
     ASSERT_EQ(512 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/true));
     ASSERT_EQ(512 * 1024 * 1024L, core_options.GetTargetFileSize(/*has_primary_key=*/false));
+    ASSERT_EQ(123, core_options.GetTargetFileRowNum());
     ASSERT_EQ(1024 * 1024 * 1024L, core_options.GetBlobTargetFileSize());
     ASSERT_EQ("foo", core_options.GetPartitionDefaultName());
     ASSERT_EQ(16 * 1024 * 1024L, core_options.GetManifestTargetFileSize());
@@ -447,6 +450,8 @@ TEST(CoreOptionsTest, TestFromMap) {
 }
 
 TEST(CoreOptionsTest, TestInvalidCase) {
+    ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::TARGET_FILE_ROW_NUM, "0"}}),
+                        "target-file-row-num should be at least 1");
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::BUCKET, "3.5"}}),
                         "Invalid Config [bucket: 3.5]");
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{Options::SCAN_SNAPSHOT_ID, "3.5"}}),
@@ -850,6 +855,7 @@ TEST(CoreOptionsTest, TestCopyAssignmentOperator) {
         {Options::BUCKET, "3"},
         {Options::PAGE_SIZE, "128 kb"},
         {Options::TARGET_FILE_SIZE, "512MB"},
+        {Options::TARGET_FILE_ROW_NUM, "4321"},
         {Options::FILE_FORMAT, "ORC"},
         {Options::FILE_COMPRESSION, "lz4"},
         {Options::FILE_COMPRESSION_ZSTD_LEVEL, "5"},
@@ -878,6 +884,9 @@ TEST(CoreOptionsTest, TestCopyAssignmentOperator) {
         {Options::DATA_FILE_PREFIX, "test-data-"},
         {Options::ROW_TRACKING_ENABLED, "true"},
         {Options::DATA_EVOLUTION_ENABLED, "true"},
+        {Options::VARIANT_SHREDDING_INFERENCE_MODE, "adaptive"},
+        {Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW, "77"},
+        {Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO, "0.02"},
         {Options::BUCKET_FUNCTION_TYPE, "mod"},
     };
     ASSERT_OK_AND_ASSIGN(CoreOptions source, CoreOptions::FromMap(options));
@@ -891,6 +900,7 @@ TEST(CoreOptionsTest, TestCopyAssignmentOperator) {
     // Verify all fields are correctly copied
     ASSERT_EQ(3, target.GetBucket());
     ASSERT_EQ(128 * 1024L, target.GetPageSize());
+    ASSERT_EQ(4321, target.GetTargetFileRowNum());
     ASSERT_EQ("orc", target.GetFileFormat()->Identifier());
     ASSERT_EQ("lz4", target.GetFileCompression());
     ASSERT_EQ(5, target.GetFileCompressionZstdLevel());
@@ -919,6 +929,9 @@ TEST(CoreOptionsTest, TestCopyAssignmentOperator) {
     ASSERT_EQ("test-data-", target.DataFilePrefix());
     ASSERT_TRUE(target.RowTrackingEnabled());
     ASSERT_TRUE(target.DataEvolutionEnabled());
+    ASSERT_EQ(VariantShreddingInferenceMode::ADAPTIVE, target.GetVariantShreddingInferenceMode());
+    ASSERT_EQ(77, target.GetVariantShreddingAdaptiveMaxInferBufferRow());
+    ASSERT_DOUBLE_EQ(0.02, target.GetVariantShreddingAdaptiveRetentionRatio());
     ASSERT_EQ(BucketFunctionType::MOD, target.GetBucketFunctionType());
 
     // Verify the target's ToMap matches the source's ToMap
@@ -1118,10 +1131,14 @@ TEST(CoreOptionsTest, TestVariantOptions) {
         ASSERT_OK_AND_ASSIGN(CoreOptions options, CoreOptions::FromMap({}));
         ASSERT_EQ(options.GetVariantShreddingSchema(), std::nullopt);
         ASSERT_FALSE(options.VariantInferShreddingSchemaEnabled());
+        ASSERT_EQ(options.GetVariantShreddingInferenceMode(),
+                  VariantShreddingInferenceMode::PER_FILE);
         ASSERT_EQ(options.GetVariantShreddingMaxSchemaWidth(), 300);
         ASSERT_EQ(options.GetVariantShreddingMaxSchemaDepth(), 50);
         ASSERT_DOUBLE_EQ(options.GetVariantShreddingMinFieldCardinalityRatio(), 0.1);
         ASSERT_EQ(options.GetVariantShreddingMaxInferBufferRow(), 4096);
+        ASSERT_EQ(options.GetVariantShreddingAdaptiveMaxInferBufferRow(), 256);
+        ASSERT_DOUBLE_EQ(options.GetVariantShreddingAdaptiveRetentionRatio(), 0.05);
     }
     {
         // Configured values.
@@ -1129,16 +1146,23 @@ TEST(CoreOptionsTest, TestVariantOptions) {
             CoreOptions options,
             CoreOptions::FromMap({{"variant.shreddingSchema", "{\"type\": \"ROW\"}"},
                                   {"variant.inferShreddingSchema", "true"},
+                                  {"variant.shredding.inferenceMode", "ADAPTIVE"},
                                   {"variant.shredding.maxSchemaWidth", "20"},
                                   {"variant.shredding.maxSchemaDepth", "5"},
                                   {"variant.shredding.minFieldCardinalityRatio", "0.25"},
-                                  {"variant.shredding.maxInferBufferRow", "128"}}));
+                                  {"variant.shredding.maxInferBufferRow", "128"},
+                                  {"variant.shredding.adaptive.maxInferBufferRow", "64"},
+                                  {"variant.shredding.adaptive.retentionRatio", "0.2"}}));
         ASSERT_EQ(options.GetVariantShreddingSchema(), "{\"type\": \"ROW\"}");
         ASSERT_TRUE(options.VariantInferShreddingSchemaEnabled());
+        ASSERT_EQ(options.GetVariantShreddingInferenceMode(),
+                  VariantShreddingInferenceMode::ADAPTIVE);
         ASSERT_EQ(options.GetVariantShreddingMaxSchemaWidth(), 20);
         ASSERT_EQ(options.GetVariantShreddingMaxSchemaDepth(), 5);
         ASSERT_DOUBLE_EQ(options.GetVariantShreddingMinFieldCardinalityRatio(), 0.25);
         ASSERT_EQ(options.GetVariantShreddingMaxInferBufferRow(), 128);
+        ASSERT_EQ(options.GetVariantShreddingAdaptiveMaxInferBufferRow(), 64);
+        ASSERT_DOUBLE_EQ(options.GetVariantShreddingAdaptiveRetentionRatio(), 0.2);
     }
     {
         // The legacy parquet-prefixed key is a fallback for the shredding schema.
@@ -1149,6 +1173,8 @@ TEST(CoreOptionsTest, TestVariantOptions) {
     // Invalid values fail when the options are parsed, not when they are used.
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.inferShreddingSchema", "not_a_bool"}}),
                         "variant.inferShreddingSchema");
+    ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.inferenceMode", "invalid"}}),
+                        "invalid variant shredding inference mode: invalid");
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxSchemaWidth", "abc"}}),
                         "variant.shredding.maxSchemaWidth");
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxSchemaWidth", "0"}}),
@@ -1160,6 +1186,18 @@ TEST(CoreOptionsTest, TestVariantOptions) {
         "should be in the range [0, 1]");
     ASSERT_NOK_WITH_MSG(CoreOptions::FromMap({{"variant.shredding.maxInferBufferRow", "0"}}),
                         "should be positive");
+    ASSERT_NOK_WITH_MSG(
+        CoreOptions::FromMap({{"variant.shredding.inferenceMode", "adaptive"},
+                              {"variant.shredding.adaptive.maxInferBufferRow", "0"}}),
+        "variant.shredding.adaptive.maxInferBufferRow");
+    ASSERT_NOK_WITH_MSG(
+        CoreOptions::FromMap({{"variant.shredding.inferenceMode", "adaptive"},
+                              {"variant.shredding.adaptive.retentionRatio", "-0.01"}}),
+        "variant.shredding.adaptive.retentionRatio");
+    ASSERT_NOK_WITH_MSG(
+        CoreOptions::FromMap({{"variant.shredding.inferenceMode", "adaptive"},
+                              {"variant.shredding.adaptive.retentionRatio", "0.11"}}),
+        "should be in the range [0, variant.shredding.minFieldCardinalityRatio]");
 }
 
 }  // namespace paimon::test
diff --git a/src/paimon/core/io/infer_shredding_file_writer.h b/src/paimon/core/io/infer_shredding_file_writer.h
index 4ce7925a..4acf9b48 100644
--- a/src/paimon/core/io/infer_shredding_file_writer.h
+++ b/src/paimon/core/io/infer_shredding_file_writer.h
@@ -38,7 +38,8 @@ namespace paimon {
 /// batches are buffered until `ShreddingWritePlanFactory::InferBufferRowCount` rows have been
 /// collected (or the writer is closed); the buffered batches are then sampled to create the
 /// batch converter, the actual file writer is created with the resulting physical schema, and
-/// the buffered batches are replayed into it. The file never rolls while buffering.
+/// the buffered batches are replayed into it. File-size rolling is suppressed while buffering;
+/// row-count rolling remains controlled by the outer RollingFileWriter.
 template 
 class InferShreddingFileWriter : public SingleFileWriter {
  public:
@@ -81,7 +82,7 @@ class InferShreddingFileWriter : public SingleFileWriter {
 
     Result ReachTargetSize(bool suggested_check, int64_t target_size) override {
         if (!plan_finalized_) {
-            // Never roll the file while rows are being buffered for inference.
+            // File-size rolling is unavailable until the inner writer has been created.
             return false;
         }
         return inner_->ReachTargetSize(suggested_check, target_size);
diff --git a/src/paimon/core/io/infer_shredding_file_writer_test.cpp b/src/paimon/core/io/infer_shredding_file_writer_test.cpp
index fe98eb97..b0e82756 100644
--- a/src/paimon/core/io/infer_shredding_file_writer_test.cpp
+++ b/src/paimon/core/io/infer_shredding_file_writer_test.cpp
@@ -160,7 +160,7 @@ class InferShreddingFileWriterTest : public ::testing::Test {
 
 TEST_F(InferShreddingFileWriterTest, BuffersUntilThresholdThenReplays) {
     auto writer = MakeWriter(/*buffer_rows=*/4);
-    // Never rolls while buffering.
+    // File-size rolling is suppressed while buffering.
     ASSERT_OK_AND_ASSIGN(bool reach, writer->ReachTargetSize(true, 1));
     ASSERT_FALSE(reach);
 
@@ -200,13 +200,15 @@ TEST_F(InferShreddingFileWriterTest, CloseFlushesPartialBuffer) {
     ASSERT_TRUE(inner_->closed);
 }
 
-TEST_F(InferShreddingFileWriterTest, EmptyFileFallsBackToLogicalSchema) {
+TEST_F(InferShreddingFileWriterTest, EmptyFileUsesUntypedVariantPlan) {
     auto writer = MakeWriter(/*buffer_rows=*/4);
     ASSERT_OK(writer->Close());
-    // With no samples there is no useful shredding schema; the writer is created without a
-    // converter.
+    // Even without typed fields, inference creates the complete untyped Variant physical plan.
     ASSERT_EQ(captured_converters_.size(), 1);
-    ASSERT_EQ(captured_converters_[0], nullptr);
+    ASSERT_NE(captured_converters_[0], nullptr);
+    const auto& physical_type = static_cast(
+        *captured_converters_[0]->GetPhysicalSchema()->GetFieldByName("v")->type());
+    ASSERT_EQ(physical_type.GetFieldByName("typed_value"), nullptr);
     ASSERT_TRUE(sink_.empty());
     ASSERT_TRUE(inner_->closed);
 }
diff --git a/src/paimon/core/io/map_shared_shredding_core_utils.cpp b/src/paimon/core/io/map_shared_shredding_core_utils.cpp
deleted file mode 100644
index 38c9518b..00000000
--- a/src/paimon/core/io/map_shared_shredding_core_utils.cpp
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/core/io/map_shared_shredding_core_utils.h"
-
-#include 
-#include 
-
-#include "arrow/c/bridge.h"
-#include "arrow/type.h"
-#include "paimon/common/data/shredding/map_shared_shredding_context.h"
-#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
-#include "paimon/common/utils/arrow/status_utils.h"
-#include "paimon/core/core_options.h"
-#include "paimon/core/io/data_file_meta.h"
-#include "paimon/core/io/data_file_path_factory.h"
-#include "paimon/format/file_format.h"
-#include "paimon/format/file_format_factory.h"
-#include "paimon/format/reader_builder.h"
-#include "paimon/fs/file_system.h"
-#include "paimon/memory/memory_pool.h"
-#include "paimon/reader/file_batch_reader.h"
-
-namespace paimon {
-namespace {
-
-bool ContainsWriteColumn(const std::vector& write_cols, const std::string& field) {
-    return std::find(write_cols.begin(), write_cols.end(), field) != write_cols.end();
-}
-
-Result> ReadFileSchema(
-    const std::shared_ptr& file,
-    const std::shared_ptr& path_factory, const CoreOptions& options,
-    const std::shared_ptr& pool) {
-    PAIMON_ASSIGN_OR_RAISE(std::string format_str, file->FileFormat());
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format,
-                           FileFormatFactory::Get(format_str, options.ToMap()));
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder,
-                           format->CreateReaderBuilder(options.GetReadBatchSize()));
-    reader_builder->WithMemoryPool(pool);
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream,
-                           options.GetFileSystem()->Open(path_factory->ToPath(file)));
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader,
-                           reader_builder->Build(input_stream));
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader->GetFileSchema());
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema,
-                                      arrow::ImportSchema(c_file_schema.get()));
-    return file_schema;
-}
-
-// Restores each shared-shredding field from the newest data file that carries its file metadata.
-// In data evolution mode, the newest file may only contain a subset of write_cols, so restoring
-// from files.back() alone can miss other shared-shredding fields. Use write_cols to avoid opening
-// unrelated files, and fall back to Kmax for fields whose metadata cannot be found.
-Status RestoreContextFromRecentFiles(const std::vector>& files,
-                                     const std::shared_ptr& path_factory,
-                                     const CoreOptions& options,
-                                     const std::shared_ptr& pool,
-                                     MapSharedShreddingContext* context) {
-    if (!context || files.empty()) {
-        return Status::OK();
-    }
-
-    std::vector shredding_fields = context->GetShreddingColumnNames();
-    std::set pending_fields(shredding_fields.begin(), shredding_fields.end());
-
-    for (auto file_it = files.rbegin(); file_it != files.rend() && !pending_fields.empty();
-         ++file_it) {
-        const auto& file = *file_it;
-        std::vector candidate_fields;
-        if (!file->write_cols) {
-            candidate_fields.assign(pending_fields.begin(), pending_fields.end());
-        } else {
-            for (const auto& field : pending_fields) {
-                if (ContainsWriteColumn(file->write_cols.value(), field)) {
-                    candidate_fields.push_back(field);
-                }
-            }
-        }
-        if (candidate_fields.empty()) {
-            continue;
-        }
-
-        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file_schema,
-                               ReadFileSchema(file, path_factory, options, pool));
-        for (const auto& field_name : candidate_fields) {
-            std::shared_ptr field = file_schema->GetFieldByName(field_name);
-            if (!field) {
-                continue;
-            }
-            const auto& metadata = field->metadata();
-            if (!metadata) {
-                continue;
-            }
-            auto metadata_copy = metadata->Copy();
-            if (!MapSharedShreddingUtils::HasShreddingMetadata(metadata_copy)) {
-                continue;
-            }
-            PAIMON_ASSIGN_OR_RAISE(
-                MapSharedShreddingFieldMeta field_meta,
-                MapSharedShreddingUtils::DeserializeMetadata(
-                    metadata_copy, MapSharedShreddingDefine::kDefaultDictCompression));
-            context->ReportFileStats(field->name(), field_meta.max_row_width);
-            pending_fields.erase(field_name);
-        }
-    }
-    return Status::OK();
-}
-
-}  // namespace
-
-Result>
-MapSharedShreddingCoreUtils::CreateAndRestoreContext(
-    const std::shared_ptr& write_schema,
-    const std::vector>& restore_files,
-    const std::shared_ptr& path_factory, const CoreOptions& options,
-    const std::shared_ptr& pool) {
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr context,
-                           MapSharedShreddingUtils::CreateShreddingContext(write_schema, options));
-    PAIMON_RETURN_NOT_OK(
-        RestoreContextFromRecentFiles(restore_files, path_factory, options, pool, context.get()));
-    return context;
-}
-
-}  // namespace paimon
diff --git a/src/paimon/core/io/map_shared_shredding_core_utils.h b/src/paimon/core/io/map_shared_shredding_core_utils.h
deleted file mode 100644
index 413d6347..00000000
--- a/src/paimon/core/io/map_shared_shredding_core_utils.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#pragma once
-
-#include 
-#include 
-
-#include "paimon/result.h"
-
-namespace arrow {
-class Schema;
-}  // namespace arrow
-
-namespace paimon {
-
-class CoreOptions;
-struct DataFileMeta;
-class DataFilePathFactory;
-class MapSharedShreddingContext;
-class MemoryPool;
-
-class MapSharedShreddingCoreUtils {
- public:
-    MapSharedShreddingCoreUtils() = delete;
-    ~MapSharedShreddingCoreUtils() = delete;
-
-    static Result> CreateAndRestoreContext(
-        const std::shared_ptr& write_schema,
-        const std::vector>& restore_files,
-        const std::shared_ptr& path_factory, const CoreOptions& options,
-        const std::shared_ptr& pool);
-};
-
-}  // namespace paimon
diff --git a/src/paimon/core/io/rolling_blob_file_writer.cpp b/src/paimon/core/io/rolling_blob_file_writer.cpp
index 4bc7c5c0..36336eef 100644
--- a/src/paimon/core/io/rolling_blob_file_writer.cpp
+++ b/src/paimon/core/io/rolling_blob_file_writer.cpp
@@ -42,12 +42,13 @@ class DataType;
 namespace paimon {
 
 RollingBlobFileWriter::RollingBlobFileWriter(
-    int64_t target_file_size, const std::shared_ptr& writer_factory,
+    int64_t target_file_size, int64_t target_file_row_num,
+    const std::shared_ptr& writer_factory,
     const std::shared_ptr& blob_schema,
     MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator,
     const std::shared_ptr& data_type, const std::set& inline_fields)
-    : RollingFileWriter<::ArrowArray*, std::shared_ptr>(target_file_size,
-                                                                      writer_factory),
+    : RollingFileWriter<::ArrowArray*, std::shared_ptr>(
+          target_file_size, target_file_row_num, writer_factory),
       blob_schema_(blob_schema),
       blob_writer_creator_(std::move(blob_writer_creator)),
       data_type_(data_type),
@@ -88,11 +89,14 @@ Status RollingBlobFileWriter::Write(::ArrowArray* record) {
     PAIMON_RETURN_NOT_OK(blob_writer_->Write(&c_blob_array));
 
     record_count_ += record_count;
-    if (current_writer_ != nullptr) {
-        PAIMON_ASSIGN_OR_RAISE(bool need_rolling_file, NeedRollingFile());
-        if (need_rolling_file) {
-            PAIMON_RETURN_NOT_OK(CloseCurrentWriter());
-        }
+    current_file_record_count_ += record_count;
+    bool need_rolling_file = current_file_record_count_ >= target_file_row_num_;
+    if (!need_rolling_file && current_writer_ != nullptr) {
+        PAIMON_ASSIGN_OR_RAISE(bool main_writer_needs_rolling, NeedRollingFile());
+        need_rolling_file = main_writer_needs_rolling;
+    }
+    if (need_rolling_file) {
+        PAIMON_RETURN_NOT_OK(CloseCurrentWriter());
     }
     guard.Release();
     return Status::OK();
@@ -116,6 +120,7 @@ Status RollingBlobFileWriter::CloseCurrentWriter() {
     results_.insert(results_.end(), blob_metas.begin(), blob_metas.end());
 
     current_writer_.reset();
+    current_file_record_count_ = 0;
     return Status::OK();
 }
 
diff --git a/src/paimon/core/io/rolling_blob_file_writer.h b/src/paimon/core/io/rolling_blob_file_writer.h
index d091c17d..4f3e6020 100644
--- a/src/paimon/core/io/rolling_blob_file_writer.h
+++ b/src/paimon/core/io/rolling_blob_file_writer.h
@@ -63,7 +63,7 @@ class RollingBlobFileWriter
     using MainWriter = SingleFileWriter<::ArrowArray*, std::shared_ptr>;
     using MainWriterFactory = SingleFileWriterFactory<::ArrowArray*, std::shared_ptr>;
 
-    RollingBlobFileWriter(int64_t target_file_size,
+    RollingBlobFileWriter(int64_t target_file_size, int64_t target_file_row_num,
                           const std::shared_ptr& writer_factory,
                           const std::shared_ptr& blob_schema,
                           MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator,
diff --git a/src/paimon/core/io/rolling_file_writer.h b/src/paimon/core/io/rolling_file_writer.h
index ac53bead..3a945632 100644
--- a/src/paimon/core/io/rolling_file_writer.h
+++ b/src/paimon/core/io/rolling_file_writer.h
@@ -18,6 +18,7 @@
 
 #pragma once
 
+#include 
 #include 
 #include 
 #include 
@@ -37,9 +38,10 @@ namespace paimon {
 template 
 class RollingFileWriter : public FileWriter> {
  public:
-    RollingFileWriter(int64_t target_file_size,
+    RollingFileWriter(int64_t target_file_size, int64_t target_file_row_num,
                       const std::shared_ptr>& writer_factory)
         : target_file_size_(target_file_size),
+          target_file_row_num_(target_file_row_num),
           writer_factory_(writer_factory),
           metrics_(std::make_shared()),
           logger_(Logger::GetLogger("RollingFileWriter")) {}
@@ -59,10 +61,6 @@ class RollingFileWriter : public FileWriter> {
         return metrics_;
     }
 
-    int64_t TargetFileSize() const {
-        return target_file_size_;
-    }
-
  protected:
     static constexpr int32_t CHECK_ROLLING_RECORD_CNT = 1000;
 
@@ -72,10 +70,12 @@ class RollingFileWriter : public FileWriter> {
     Status OpenCurrentWriter();
 
     int64_t target_file_size_ = 0;
+    int64_t target_file_row_num_ = std::numeric_limits::max();
     std::shared_ptr> writer_factory_;
     std::shared_ptr metrics_;
 
     int64_t record_count_ = 0;
+    int64_t current_file_record_count_ = 0;
     int64_t last_need_rolling_record_count_ = 0;
     bool closed_ = false;
 
@@ -101,6 +101,9 @@ bool RollingFileWriter::SuggestCheck() {
 
 template 
 Result RollingFileWriter::NeedRollingFile() {
+    if (current_file_record_count_ >= target_file_row_num_) {
+        return true;
+    }
     return current_writer_->ReachTargetSize(SuggestCheck(), target_file_size_);
 }
 
@@ -121,6 +124,7 @@ Status RollingFileWriter::Write(T record) {
     }
     PAIMON_RETURN_NOT_OK(current_writer_->Write(std::move(record)));
     record_count_ += record_count;
+    current_file_record_count_ += record_count;
     PAIMON_ASSIGN_OR_RAISE(bool need_rolling_file, NeedRollingFile());
     if (need_rolling_file) {
         PAIMON_RETURN_NOT_OK(CloseCurrentWriter());
@@ -165,6 +169,7 @@ Status RollingFileWriter::CloseCurrentWriter() {
     PAIMON_ASSIGN_OR_RAISE(R result, current_writer_->GetResult());
     results_.push_back(result);
     current_writer_.reset();
+    current_file_record_count_ = 0;
     if (metrics_) {
         metrics_->Merge(current_metrics);
     }
diff --git a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp
index 5e9b5718..6e4843bb 100644
--- a/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp
+++ b/src/paimon/core/io/shredding_append_data_file_writer_factory.cpp
@@ -68,7 +68,11 @@ ShreddingAppendDataFileWriterFactory::CreateShreddedWriter(
     const std::shared_ptr& converter) const {
     if (converter == nullptr) {
         // No conversion is useful for this file; fall back to the plain writer.
-        return AppendDataFileWriterFactory::CreateWriter();
+        std::unique_ptr>> writer;
+        PAIMON_ASSIGN_OR_RAISE(writer, AppendDataFileWriterFactory::CreateWriter());
+        writer->SetCompletionCallback(
+            [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); });
+        return writer;
     }
     std::shared_ptr seq_num_counter = ResolveSeqNumCounter();
     std::shared_ptr file_schema = converter->GetPhysicalSchema();
@@ -88,10 +92,12 @@ ShreddingAppendDataFileWriterFactory::CreateShreddedWriter(
     PAIMON_RETURN_NOT_OK(
         writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
     ShreddingWritePlanFactory::MetadataFinalizer finalizer =
-        plan_factory_->CreateMetadataFinalizer(converter);
+        plan_factory_->CreateMetadataFinalizer(converter, options_.GetFileCompression());
     if (finalizer) {
         writer->SetMetadataFinalizer(std::move(finalizer));
     }
+    writer->SetCompletionCallback(
+        [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); });
     return std::unique_ptr>>(
         std::move(writer));
 }
diff --git a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
index 10c7f8b2..8ac583ee 100644
--- a/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
+++ b/src/paimon/core/io/shredding_key_value_data_file_writer_factory.cpp
@@ -67,7 +67,11 @@ ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter(
     const std::shared_ptr& converter) const {
     if (converter == nullptr) {
         // No conversion is useful for this file; fall back to the plain writer.
-        return KeyValueDataFileWriterFactory::CreateWriter();
+        std::unique_ptr>> writer;
+        PAIMON_ASSIGN_OR_RAISE(writer, KeyValueDataFileWriterFactory::CreateWriter());
+        writer->SetCompletionCallback(
+            [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); });
+        return writer;
     }
     auto format = options_.GetWriteFileFormat(level_);
     std::shared_ptr file_schema = converter->GetPhysicalSchema();
@@ -87,10 +91,12 @@ ShreddingKeyValueDataFileWriterFactory::CreateShreddedWriter(
     PAIMON_RETURN_NOT_OK(
         writer->Init(options_.GetFileSystem(), path_factory_->NewPath(), resources.writer_builder));
     ShreddingWritePlanFactory::MetadataFinalizer finalizer =
-        plan_factory_->CreateMetadataFinalizer(converter);
+        plan_factory_->CreateMetadataFinalizer(converter, options_.GetWriteFileCompression(level_));
     if (finalizer) {
         writer->SetMetadataFinalizer(std::move(finalizer));
     }
+    writer->SetCompletionCallback(
+        [factory = plan_factory_, converter]() { return factory->OnFileCompleted(converter); });
     return std::unique_ptr>>(
         std::move(writer));
 }
diff --git a/src/paimon/core/io/single_file_writer.h b/src/paimon/core/io/single_file_writer.h
index a92b827a..224e8d9e 100644
--- a/src/paimon/core/io/single_file_writer.h
+++ b/src/paimon/core/io/single_file_writer.h
@@ -99,6 +99,12 @@ class SingleFileWriter : public FileWriter {
     void Abort() override;
     Status Close() override;
 
+    /// Sets a callback invoked only after the format writer and output stream have both
+    /// completed successfully.
+    void SetCompletionCallback(std::function callback) {
+        completion_callback_ = std::move(callback);
+    }
+
     std::shared_ptr GetMetrics() const override {
         if (writer_) {
             return writer_->GetWriterMetrics();
@@ -136,6 +142,7 @@ class SingleFileWriter : public FileWriter {
     std::shared_ptr out_;  // nullptr for DirectWriterBuilder
     bool closed_ = false;
     std::string path_;
+    std::function completion_callback_;
 
  private:
     int64_t record_count_ = 0;
@@ -222,12 +229,19 @@ Status SingleFileWriter::Close() {
     if (out_) {
         PAIMON_RETURN_NOT_OK(out_->Flush());
         PAIMON_ASSIGN_OR_RAISE(output_bytes_, out_->GetPos());
-        PAIMON_RETURN_NOT_OK(out_->Close());
+        std::shared_ptr out = std::move(out_);
+        PAIMON_RETURN_NOT_OK(out->Close());
     } else {
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_status, fs_->GetFileStatus(path_));
         output_bytes_ = file_status->GetLen();
     }
+    // Completing the format writer and stream is terminal even if publication fails. The scope
+    // guard still removes the file on a callback error, while a repeated Close() does not publish
+    // the same file again.
     closed_ = true;
+    if (completion_callback_) {
+        PAIMON_RETURN_NOT_OK(completion_callback_());
+    }
     guard.Release();
     return Status::OK();
 }
@@ -254,7 +268,8 @@ Status SingleFileWriter::UpdateSchema(const std::shared_ptr
 template 
 void SingleFileWriter::Abort() {
     if (out_) {
-        auto status = out_->Close();
+        std::shared_ptr out = std::move(out_);
+        auto status = out->Close();
         if (!status.ok()) {
             PAIMON_LOG_WARN(logger_, "Exception occurs when closing %s: %s", path_.c_str(),
                             status.ToString().c_str());
diff --git a/src/paimon/core/io/single_file_writer_test.cpp b/src/paimon/core/io/single_file_writer_test.cpp
index c3ef9eb0..8ed940e2 100644
--- a/src/paimon/core/io/single_file_writer_test.cpp
+++ b/src/paimon/core/io/single_file_writer_test.cpp
@@ -106,4 +106,42 @@ TEST(SingleFileWriterTest, TestInvalidConvert) {
     ASSERT_FALSE(exist);
 }
 
+TEST(SingleFileWriterTest, CompletionCallbackFailureIsTerminal) {
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    std::string file_path = dir->Str() + "/single-file";
+    auto data_type = arrow::struct_({arrow::field("col", arrow::int32())});
+    auto converter = [&](int32_t value, ::ArrowArray* dest) -> Status {
+        std::string value_str = "[[" + std::to_string(value) + "]]";
+        auto array =
+            arrow::ipc::internal::json::ArrayFromJSON(data_type, value_str.c_str()).ValueOrDie();
+        PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, dest));
+        return Status::OK();
+    };
+    SimpleSingleFileWriter writer("zstd", converter);
+    ASSERT_OK_AND_ASSIGN(
+        CoreOptions options,
+        CoreOptions::FromMap({{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, "orc"}}));
+    auto file_format = options.GetWriteFileFormat(/*level=*/0);
+    auto file_system = options.GetFileSystem();
+    ArrowSchema arrow_schema;
+    ASSERT_TRUE(arrow::ExportType(*data_type, &arrow_schema).ok());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr writer_builder,
+                         file_format->CreateWriterBuilder(&arrow_schema, /*batch_size=*/100));
+    ASSERT_OK(writer.Init(file_system, file_path, writer_builder));
+    ASSERT_OK(writer.Write(100));
+
+    int32_t callback_count = 0;
+    writer.SetCompletionCallback([&callback_count]() -> Status {
+        ++callback_count;
+        return Status::Invalid("completion failed");
+    });
+    ASSERT_NOK_WITH_MSG(writer.Close(), "completion failed");
+    ASSERT_EQ(callback_count, 1);
+    ASSERT_OK(writer.Close());
+    ASSERT_EQ(callback_count, 1);
+    ASSERT_OK_AND_ASSIGN(auto exists, file_system->Exists(file_path));
+    ASSERT_FALSE(exists);
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/manifest/manifest_file.cpp b/src/paimon/core/manifest/manifest_file.cpp
index a952888c..22f2681f 100644
--- a/src/paimon/core/manifest/manifest_file.cpp
+++ b/src/paimon/core/manifest/manifest_file.cpp
@@ -19,6 +19,7 @@
 #include "paimon/core/manifest/manifest_file.h"
 
 #include 
+#include 
 #include 
 
 #include "arrow/c/abi.h"
@@ -107,7 +108,8 @@ Result> ManifestFile::Write(
         options_.GetFileSystem(), path_factory_, writer_builder_);
     std::unique_ptr> writer =
         std::make_unique>(
-            target_file_size_, writer_factory);
+            target_file_size_, /*target_file_row_num=*/std::numeric_limits::max(),
+            writer_factory);
     for (const auto& entry : entries) {
         auto s = writer->Write(entry);
         if (!s.ok()) {
diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp
index f6b71311..3efd40f6 100644
--- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.cpp
@@ -27,13 +27,11 @@ ChangelogMergeTreeRewriter::ChangelogMergeTreeRewriter(
     std::unique_ptr&& merge_file_split_read,
     MergeFunctionWrapperFactory merge_function_wrapper_factory,
     const std::shared_ptr& cancellation_controller,
-    const std::shared_ptr& shredding_context,
     const std::shared_ptr& pool)
-    : MergeTreeCompactRewriter(partition, bucket, schema_id, trimmed_primary_keys, options,
-                               data_schema, write_schema, std::move(dv_factory), path_factory_cache,
-                               std::move(merge_file_split_read),
-                               std::move(merge_function_wrapper_factory), cancellation_controller,
-                               shredding_context, pool),
+    : MergeTreeCompactRewriter(
+          partition, bucket, schema_id, trimmed_primary_keys, options, data_schema, write_schema,
+          std::move(dv_factory), path_factory_cache, std::move(merge_file_split_read),
+          std::move(merge_function_wrapper_factory), cancellation_controller, pool),
       max_level_(max_level),
       force_drop_delete_(force_drop_delete) {}
 
@@ -85,7 +83,6 @@ Result ChangelogMergeTreeRewriter::RewriteOrProduceChangelog(
     auto before = ExtractFilesFromSections(sections);
     std::unique_ptr compact_file_writer;
     if (rewrite_compact_file) {
-        PAIMON_RETURN_NOT_OK(RestoreShreddingContextFromFiles(before));
         PAIMON_ASSIGN_OR_RAISE(compact_file_writer, CreateRollingRowWriter(output_level));
     }
     // TODO(xinyu.lxy): produce changelog
diff --git a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h
index 1e6b9034..c5d8e891 100644
--- a/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h
+++ b/src/paimon/core/mergetree/compact/changelog_merge_tree_rewriter.h
@@ -43,7 +43,6 @@ class ChangelogMergeTreeRewriter : public MergeTreeCompactRewriter {
         std::unique_ptr&& merge_file_split_read,
         MergeFunctionWrapperFactory merge_function_wrapper_factory,
         const std::shared_ptr& cancellation_controller,
-        const std::shared_ptr& shredding_context,
         const std::shared_ptr& pool);
 
     struct UpgradeStrategy {
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
index 19cfe0e4..994b0e3f 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.cpp
@@ -20,8 +20,6 @@
 
 #include 
 
-#include "paimon/common/data/shredding/map_shared_shredding_context.h"
-#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/core/mergetree/compact/first_row_merge_function_wrapper.h"
 #include "paimon/core/mergetree/compact/lookup_changelog_merge_function_wrapper.h"
@@ -42,14 +40,13 @@ LookupMergeTreeCompactRewriter::LookupMergeTreeCompactRewriter(
     MergeFunctionWrapperFactory merge_function_wrapper_factory,
     const std::shared_ptr& cancellation_controller,
     const std::shared_ptr& remote_lookup_file_manager,
-    const std::shared_ptr& shredding_context,
     const std::shared_ptr& pool)
     : ChangelogMergeTreeRewriter(
           max_level, /*force_drop_delete=*/dv_maintainer != nullptr, partition, bucket, schema_id,
           trimmed_primary_keys, options, data_schema, write_schema,
           DeletionVector::CreateFactory(dv_maintainer), path_factory_cache,
           std::move(merge_file_split_read), std::move(merge_function_wrapper_factory),
-          cancellation_controller, shredding_context, pool),
+          cancellation_controller, pool),
       lookup_levels_(std::move(lookup_levels)),
       dv_maintainer_(dv_maintainer),
       remote_lookup_file_manager_(remote_lookup_file_manager) {}
@@ -96,14 +93,11 @@ LookupMergeTreeCompactRewriter::Create(
         std::unique_ptr merge_file_split_read,
         MergeFileSplitRead::Create(path_factory, internal_context, pool, CreateDefaultExecutor()));
 
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context,
-                           MapSharedShreddingUtils::CreateShreddingContext(write_schema, options));
-
     return std::unique_ptr(new LookupMergeTreeCompactRewriter(
         std::move(lookup_levels), dv_maintainer, max_level, partition, bucket, table_schema->Id(),
         trimmed_primary_keys, options, data_schema, write_schema, path_factory_cache,
         std::move(merge_file_split_read), std::move(merge_function_wrapper_factory),
-        cancellation_controller, remote_lookup_file_manager, shredding_context, pool));
+        cancellation_controller, remote_lookup_file_manager, pool));
 }
 
 template 
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h
index 85e9aba0..06c5f47f 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter.h
@@ -74,7 +74,6 @@ class LookupMergeTreeCompactRewriter : public ChangelogMergeTreeRewriter {
         MergeFunctionWrapperFactory merge_function_wrapper_factory,
         const std::shared_ptr& cancellation_controller,
         const std::shared_ptr& remote_lookup_file_manager,
-        const std::shared_ptr& shredding_context,
         const std::shared_ptr& pool);
 
     bool RewriteChangelog(int32_t output_level, bool drop_delete,
diff --git a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
index f33b8b86..c734d5e9 100644
--- a/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
+++ b/src/paimon/core/mergetree/compact/lookup_merge_tree_compact_rewriter_test.cpp
@@ -106,7 +106,7 @@ class LookupMergeTreeCompactRewriterTest : public ::testing::TestWithParamId(), arrow_schema_,
                 options, std::make_shared(), /*io_manager=*/nullptr,
-                /*enable_multi_thread_spill=*/false, /*shredding_context=*/nullptr, pool_));
+                /*enable_multi_thread_spill=*/false, pool_));
 
         // write data
         ArrowArray c_src_array;
@@ -1080,8 +1080,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr,
-            /*shredding_context=*/nullptr, pool_);
+            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/1, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::NoChangelogNoRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file));
@@ -1096,8 +1095,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr,
-            /*shredding_context=*/nullptr, pool_);
+            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file));
@@ -1116,8 +1114,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr,
-            /*shredding_context=*/nullptr, pool_);
+            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/1);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file));
@@ -1132,8 +1129,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr,
-            /*shredding_context=*/nullptr, pool_);
+            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/5, file));
@@ -1148,8 +1144,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr,
-            /*shredding_context=*/nullptr, pool_);
+            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogNoRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file));
@@ -1165,8 +1160,7 @@ TEST_F(LookupMergeTreeCompactRewriterTest, TestGenerateUpgradeStrategy) {
             /*trimmed_primary_keys=*/{"key"}, core_options, /*data_schema=*/nullptr,
             /*write_schema=*/nullptr, /*path_factory_cache=*/nullptr,
             /*merge_file_split_read=*/nullptr, /*merge_function_wrapper_factory=*/nullptr,
-            cancellation_controller, /*remote_lookup_file_manager=*/nullptr,
-            /*shredding_context=*/nullptr, pool_);
+            cancellation_controller, /*remote_lookup_file_manager=*/nullptr, pool_);
         auto file = create_meta(/*level=*/0, /*delete_row_count=*/std::nullopt);
         ASSERT_EQ(ChangelogMergeTreeRewriter::UpgradeStrategy::ChangelogWithRewrite(),
                   rewriter.GenerateUpgradeStrategy(/*output_level=*/2, file));
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
index d17d7a0e..1b64be2d 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.cpp
@@ -18,18 +18,17 @@
 #include "paimon/core/mergetree/compact/merge_tree_compact_rewriter.h"
 
 #include 
+#include 
 #include 
 
 #include "arrow/c/bridge.h"
 #include "arrow/c/helpers.h"
-#include "paimon/common/data/shredding/map_shared_shredding_utils.h"
 #include "paimon/common/data/shredding/shredding_write_plan_factories.h"
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/scope_guard.h"
 #include "paimon/core/io/key_value_data_file_writer_factory.h"
 #include "paimon/core/io/key_value_meta_projection_consumer.h"
 #include "paimon/core/io/key_value_record_reader.h"
-#include "paimon/core/io/map_shared_shredding_core_utils.h"
 #include "paimon/core/io/row_to_arrow_array_converter.h"
 #include "paimon/core/io/shredding_key_value_data_file_writer_factory.h"
 #include "paimon/core/manifest/file_source.h"
@@ -46,7 +45,6 @@ MergeTreeCompactRewriter::MergeTreeCompactRewriter(
     std::unique_ptr&& merge_file_split_read,
     MergeFunctionWrapperFactory merge_function_wrapper_factory,
     const std::shared_ptr& cancellation_controller,
-    const std::shared_ptr& shredding_context,
     const std::shared_ptr& pool)
     : options_(options),
       merge_file_split_read_(std::move(merge_file_split_read)),
@@ -60,8 +58,7 @@ MergeTreeCompactRewriter::MergeTreeCompactRewriter(
       dv_factory_(std::move(dv_factory)),
       path_factory_cache_(path_factory_cache),
       merge_function_wrapper_factory_(std::move(merge_function_wrapper_factory)),
-      cancellation_controller_(cancellation_controller),
-      shredding_context_(shredding_context) {
+      cancellation_controller_(cancellation_controller) {
     assert(cancellation_controller_ != nullptr);
 }
 
@@ -106,13 +103,10 @@ Result> MergeTreeCompactRewriter::Crea
         return std::shared_ptr>();
     };
 
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context,
-                           MapSharedShreddingUtils::CreateShreddingContext(write_schema, options));
-
     return std::unique_ptr(new MergeTreeCompactRewriter(
         partition, bucket, table_schema->Id(), trimmed_primary_keys, options, data_schema,
         write_schema, std::move(dv_factory), path_factory_cache, std::move(merge_file_split_read),
-        merge_function_wrapper_factory, cancellation_controller, shredding_context, pool));
+        merge_function_wrapper_factory, cancellation_controller, pool));
 }
 
 Result MergeTreeCompactRewriter::Upgrade(int32_t output_level,
@@ -139,24 +133,16 @@ std::vector> MergeTreeCompactRewriter::ExtractFile
     return files;
 }
 
-Status MergeTreeCompactRewriter::RestoreShreddingContextFromFiles(
-    const std::vector>& files) {
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory,
-                           CreateDataFilePathFactory(options_.GetFileFormat()->Identifier()));
-    PAIMON_ASSIGN_OR_RAISE(shredding_context_,
-                           MapSharedShreddingCoreUtils::CreateAndRestoreContext(
-                               write_schema_, files, data_file_path_factory, options_, pool_));
-    return Status::OK();
-}
-
 Result>
 MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) {
     auto format = options_.GetWriteFileFormat(level);
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory,
                            CreateDataFilePathFactory(format->Identifier()));
     std::shared_ptr>> factory;
-    if (auto plan_factory = ShreddingWritePlanFactories::SelectActive(options_, write_schema_,
-                                                                      shredding_context_, pool_)) {
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr plan_factory,
+        ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_));
+    if (plan_factory != nullptr) {
         factory = std::make_shared(
             options_, schema_id_, write_schema_, level, FileSource::Compact(),
             trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true,
@@ -167,7 +153,8 @@ MergeTreeCompactRewriter::CreateRollingRowWriter(int32_t level) {
             trimmed_primary_keys_, data_file_path_factory, /*create_stats_extractor=*/true, pool_);
     }
     return std::make_unique(
-        options_.GetTargetFileSize(/*has_primary_key=*/true), factory);
+        options_.GetTargetFileSize(/*has_primary_key=*/true),
+        /*target_file_row_num=*/std::numeric_limits::max(), factory);
 }
 
 Result
@@ -266,8 +253,6 @@ Result MergeTreeCompactRewriter::RewriteCompaction(
     PAIMON_ASSIGN_OR_RAISE(MergeTreeCompactRewriter::KeyValueConsumerCreator create_consumer,
                            GenerateKeyValueConsumer());
     auto before = ExtractFilesFromSections(sections);
-    PAIMON_RETURN_NOT_OK(RestoreShreddingContextFromFiles(before));
-
     std::vector> reader_holders;
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr rolling_writer,
                            CreateRollingRowWriter(output_level));
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h
index a6af11d7..c9c62470 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter.h
@@ -32,7 +32,6 @@
 #include "paimon/core/utils/file_store_path_factory.h"
 #include "paimon/core/utils/file_store_path_factory_cache.h"
 namespace paimon {
-class MapSharedShreddingContext;
 
 /// Default `CompactRewriter` for merge trees.
 class MergeTreeCompactRewriter : public CompactRewriter {
@@ -73,9 +72,6 @@ class MergeTreeCompactRewriter : public CompactRewriter {
     static std::vector> ExtractFilesFromSections(
         const std::vector>& sections);
 
-    Status RestoreShreddingContextFromFiles(
-        const std::vector>& files);
-
     MergeTreeCompactRewriter(const BinaryRow& partition, int32_t bucket, int64_t schema_id,
                              const std::vector& trimmed_primary_keys,
                              const CoreOptions& options,
@@ -86,7 +82,6 @@ class MergeTreeCompactRewriter : public CompactRewriter {
                              std::unique_ptr&& merge_file_split_read,
                              MergeFunctionWrapperFactory merge_function_wrapper_factory,
                              const std::shared_ptr& cancellation_controller,
-                             const std::shared_ptr& shredding_context,
                              const std::shared_ptr& pool);
 
     using KeyValueRollingFileWriter =
@@ -127,9 +122,6 @@ class MergeTreeCompactRewriter : public CompactRewriter {
     std::shared_ptr path_factory_cache_;
     MergeFunctionWrapperFactory merge_function_wrapper_factory_;
     std::shared_ptr cancellation_controller_;
-
-    /// Cross-file shared context for shared-shredding MAP columns (nullable).
-    std::shared_ptr shredding_context_;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp
index 891b46c4..cb26540a 100644
--- a/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp
+++ b/src/paimon/core/mergetree/compact/merge_tree_compact_rewriter_test.cpp
@@ -112,6 +112,14 @@ TEST_F(MergeTreeCompactRewriterTest, TestSimple) {
     // load table schema
     SchemaManager schema_manager(fs, table_path);
     ASSERT_OK_AND_ASSIGN(auto table_schema, schema_manager.ReadSchema(0));
+    auto options = table_schema->Options();
+    options[Options::TARGET_FILE_ROW_NUM] = "1";
+    auto logical_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr configured_schema,
+        TableSchema::Create(table_schema->Id(), logical_schema, table_schema->PartitionKeys(),
+                            table_schema->PrimaryKeys(), options));
+    table_schema = std::shared_ptr(std::move(configured_schema));
     ASSERT_OK_AND_ASSIGN(
         auto rewriter,
         CreateCompactRewriter(table_path, table_schema, /*bucket=*/1,
@@ -124,6 +132,7 @@ TEST_F(MergeTreeCompactRewriterTest, TestSimple) {
                                                   /*output_level=*/5, /*drop_delete=*/true, runs));
     // check compact result
     ASSERT_EQ(4, compact_result.Before().size());
+    // Compaction must not honor target-file-row-num; all seven rows stay in one output file.
     ASSERT_EQ(1, compact_result.After().size());
     const auto& compact_file_meta = compact_result.After()[0];
     auto expected_file_meta = std::make_shared(
diff --git a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp
index c28ea61d..cd8f3e18 100644
--- a/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp
+++ b/src/paimon/core/mergetree/lookup/remote_lookup_file_manager_test.cpp
@@ -84,8 +84,7 @@ class RemoteLookupFileManagerTest : public testing::Test {
                              std::vector({"key"}), data_path_factory, key_comparator,
                              /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper,
                              /*schema_id=*/0, arrow_schema_, options, noop_compact_manager_,
-                             /*io_manager=*/nullptr, /*enable_multi_thread_spill=*/false,
-                             /*shredding_context=*/nullptr, pool_));
+                             /*io_manager=*/nullptr, /*enable_multi_thread_spill=*/false, pool_));
 
         ArrowArray c_src_array;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*src_array, &c_src_array));
diff --git a/src/paimon/core/mergetree/lookup_levels_test.cpp b/src/paimon/core/mergetree/lookup_levels_test.cpp
index ee1d0b4e..a136b625 100644
--- a/src/paimon/core/mergetree/lookup_levels_test.cpp
+++ b/src/paimon/core/mergetree/lookup_levels_test.cpp
@@ -88,8 +88,7 @@ class LookupLevelsTest : public testing::Test {
                              std::vector({"key"}), data_path_factory, key_comparator,
                              /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper,
                              /*schema_id=*/0, arrow_schema_, options, noop_compact_manager_,
-                             /*io_manager=*/nullptr, /*enable_multi_thread_spill=*/false,
-                             /*shredding_context=*/nullptr, pool_));
+                             /*io_manager=*/nullptr, /*enable_multi_thread_spill=*/false, pool_));
 
         // write data
         ArrowArray c_src_array;
diff --git a/src/paimon/core/mergetree/merge_tree_writer.cpp b/src/paimon/core/mergetree/merge_tree_writer.cpp
index 7db94de1..3b6806c7 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer.cpp
@@ -58,7 +58,6 @@ Result> MergeTreeWriter::Create(
     int64_t schema_id, const std::shared_ptr& value_schema,
     const CoreOptions& options, const std::shared_ptr& compact_manager,
     const std::shared_ptr& io_manager, bool enable_multi_thread_spill,
-    const std::shared_ptr& shredding_context,
     const std::shared_ptr& pool) {
     auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema);
 
@@ -68,10 +67,10 @@ Result> MergeTreeWriter::Create(
                             options.GetSequenceField(), key_comparator, user_defined_seq_comparator,
                             merge_function_wrapper, options, io_manager, enable_multi_thread_spill,
                             pool));
-    return std::shared_ptr(new MergeTreeWriter(
-        trimmed_primary_keys, options, path_factory, key_comparator, user_defined_seq_comparator,
-        merge_function_wrapper, schema_id, write_schema, compact_manager, std::move(write_buffer),
-        shredding_context, pool));
+    return std::shared_ptr(
+        new MergeTreeWriter(trimmed_primary_keys, options, path_factory, key_comparator,
+                            user_defined_seq_comparator, merge_function_wrapper, schema_id,
+                            write_schema, compact_manager, std::move(write_buffer), pool));
 }
 
 MergeTreeWriter::MergeTreeWriter(
@@ -82,9 +81,7 @@ MergeTreeWriter::MergeTreeWriter(
     const std::shared_ptr>& merge_function_wrapper,
     int64_t schema_id, const std::shared_ptr& write_schema,
     const std::shared_ptr& compact_manager,
-    std::unique_ptr&& write_buffer,
-    const std::shared_ptr& shredding_context,
-    const std::shared_ptr& pool)
+    std::unique_ptr&& write_buffer, const std::shared_ptr& pool)
     : pool_(pool),
       trimmed_primary_keys_(trimmed_primary_keys),
       options_(options),
@@ -96,8 +93,7 @@ MergeTreeWriter::MergeTreeWriter(
       write_schema_(write_schema),
       compact_manager_(compact_manager),
       write_buffer_(std::move(write_buffer)),
-      metrics_(std::make_shared()),
-      shredding_context_(shredding_context) {}
+      metrics_(std::make_shared()) {}
 
 Status MergeTreeWriter::DoClose() {
     // Request cancellation and wait for running compaction to exit.
@@ -277,7 +273,9 @@ Status MergeTreeWriter::FlushWriteBuffer(bool wait_for_latest_compaction,
             std::make_unique>(
                 std::move(sort_merge_reader), create_consumer, options_.GetWriteBatchSize(),
                 /*projection_thread_num=*/1, pool_);
-        auto rolling_writer = CreateRollingRowWriter();
+        std::unique_ptr>>
+            rolling_writer;
+        PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
         ScopeGuard write_guard([&]() -> void {
             rolling_writer->Abort();
             async_key_value_producer_consumer->Close();
@@ -321,11 +319,13 @@ Result MergeTreeWriter::DrainIncrement() {
     return CommitIncrement(data_increment, compact_increment, drain_deletion_file);
 }
 
-std::unique_ptr>>
+Result>>>
 MergeTreeWriter::CreateRollingRowWriter() const {
     std::shared_ptr>> factory;
-    if (auto plan_factory = ShreddingWritePlanFactories::SelectActive(options_, write_schema_,
-                                                                      shredding_context_, pool_)) {
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr plan_factory,
+        ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_));
+    if (plan_factory != nullptr) {
         factory = std::make_shared(
             options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
             trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, plan_factory,
@@ -336,7 +336,8 @@ MergeTreeWriter::CreateRollingRowWriter() const {
             trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/true, pool_);
     }
     return std::make_unique>>(
-        options_.GetTargetFileSize(/*has_primary_key=*/true), factory);
+        options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(),
+        factory);
 }
 
 }  // namespace paimon
diff --git a/src/paimon/core/mergetree/merge_tree_writer.h b/src/paimon/core/mergetree/merge_tree_writer.h
index d522f679..febce2af 100644
--- a/src/paimon/core/mergetree/merge_tree_writer.h
+++ b/src/paimon/core/mergetree/merge_tree_writer.h
@@ -49,7 +49,6 @@ namespace paimon {
 class DataFilePathFactory;
 class IOManager;
 class FieldsComparator;
-class MapSharedShreddingContext;
 class MemoryPool;
 class Metrics;
 template 
@@ -66,7 +65,6 @@ class MergeTreeWriter : public BatchWriter {
         int64_t schema_id, const std::shared_ptr& value_schema,
         const CoreOptions& options, const std::shared_ptr& compact_manager,
         const std::shared_ptr& io_manager, bool enable_multi_thread_spill,
-        const std::shared_ptr& shredding_context,
         const std::shared_ptr& pool);
 
     Status Write(std::unique_ptr&& batch) override;
@@ -99,7 +97,7 @@ class MergeTreeWriter : public BatchWriter {
     Status FlushWriteBuffer(bool wait_for_latest_compaction, bool forced_full_compaction);
     Result DrainIncrement();
 
-    std::unique_ptr>>
+    Result>>>
     CreateRollingRowWriter() const;
 
     Status TrySyncLatestCompaction(bool blocking);
@@ -116,7 +114,6 @@ class MergeTreeWriter : public BatchWriter {
                     int64_t schema_id, const std::shared_ptr& write_schema,
                     const std::shared_ptr& compact_manager,
                     std::unique_ptr&& write_buffer,
-                    const std::shared_ptr& shredding_context,
                     const std::shared_ptr& pool);
 
     std::shared_ptr pool_;
@@ -142,8 +139,5 @@ class MergeTreeWriter : public BatchWriter {
     std::vector> compact_after_;
 
     std::shared_ptr compact_deletion_file_;
-
-    /// Cross-file shared context for shared-shredding MAP columns (nullable).
-    std::shared_ptr shredding_context_;
 };
 }  // namespace paimon
diff --git a/src/paimon/core/mergetree/merge_tree_writer_test.cpp b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
index ddcba958..57a62654 100644
--- a/src/paimon/core/mergetree/merge_tree_writer_test.cpp
+++ b/src/paimon/core/mergetree/merge_tree_writer_test.cpp
@@ -178,10 +178,8 @@ class MergeTreeWriterTest : public ::testing::TestWithParam {
 
         auto metadata = file_schema->field(field_index)->metadata();
         ASSERT_NE(nullptr, metadata);
-        ASSERT_OK_AND_ASSIGN(
-            auto deserialized_meta,
-            MapSharedShreddingUtils::DeserializeMetadata(
-                metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression));
+        ASSERT_OK_AND_ASSIGN(auto deserialized_meta,
+                             MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()));
         ASSERT_EQ(expected_meta, deserialized_meta);
     }
 
@@ -210,8 +208,7 @@ class MergeTreeWriterTest : public ::testing::TestWithParam {
         return MergeTreeWriter::Create(
             last_sequence_number, primary_keys_, path_factory, key_comparator_,
             user_defined_seq_comparator, merge_function_wrapper_, schema_id, value_schema_, options,
-            writer_compact_manager, io_manager, /*enable_multi_thread_spill=*/false,
-            /*shredding_context=*/nullptr, pool_);
+            writer_compact_manager, io_manager, /*enable_multi_thread_spill=*/false, pool_);
     }
 
  private:
@@ -405,9 +402,6 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {
     ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator,
                          FieldsComparator::Create({value_fields[0]},
                                                   /*is_ascending_order=*/true));
-    ASSERT_OK_AND_ASSIGN(auto shredding_context,
-                         MapSharedShreddingUtils::CreateShreddingContext(write_schema, options));
-
     ASSERT_OK_AND_ASSIGN(
         auto merge_writer,
         MergeTreeWriter::Create(
@@ -416,7 +410,7 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {
             /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/5,
             value_schema, options, noop_compact_manager_,
             GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr,
-            /*enable_multi_thread_spill=*/false, shredding_context, pool_));
+            /*enable_multi_thread_spill=*/false, pool_));
 
     // Each batch contains duplicated primary keys. DeduplicateMergeFunction should keep the
     // latest sequence number for each key across and within batches.
@@ -488,10 +482,12 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMapDataFileMetaInfo) {
     ASSERT_TRUE(expected_data_file_meta->TEST_Equal(*actual_meta));
 }
 
-TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation) {
+TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsAdaptAcrossRollingFiles) {
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
                          CoreOptions::FromMap({
                              {Options::FILE_FORMAT, "orc"},
+                             {Options::TARGET_FILE_ROW_NUM, "2"},
+                             {Options::WRITE_BATCH_SIZE, "2"},
                              {"fields.tags.map.storage-layout", "shared-shredding"},
                              {"fields.tags.map.shared-shredding.max-columns", "8"},
                              {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -517,9 +513,6 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation)
     ASSERT_OK_AND_ASSIGN(std::shared_ptr key_comparator,
                          FieldsComparator::Create({value_fields[0]},
                                                   /*is_ascending_order=*/true));
-    ASSERT_OK_AND_ASSIGN(auto shredding_context,
-                         MapSharedShreddingUtils::CreateShreddingContext(write_schema, options));
-
     ASSERT_OK_AND_ASSIGN(
         auto merge_writer,
         MergeTreeWriter::Create(
@@ -528,7 +521,7 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation)
             /*user_defined_seq_comparator=*/nullptr, merge_function_wrapper_, /*schema_id=*/0,
             value_schema, options, noop_compact_manager_,
             GetParam() ? std::make_shared(dir->Str() + "/tmp", file_system_) : nullptr,
-            /*enable_multi_thread_spill=*/false, shredding_context, pool_));
+            /*enable_multi_thread_spill=*/false, pool_));
 
     auto array1 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
       [1, [["a", 10], ["b", 20]], [["x", "v1"]]],
@@ -536,11 +529,24 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation)
     ])")
                       .ValueOrDie();
     WriteBatch(array1, /*row_kinds=*/{}, merge_writer.get());
-    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment1,
+
+    auto array2 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
+      [3, [["c", 100], ["d", 200], ["e", 300]], [["p", "a1"], ["q", "a2"], ["r", "a3"]]]
+    ])")
+                      .ValueOrDie();
+    WriteBatch(array2, /*row_kinds=*/{}, merge_writer.get());
+
+    auto array3 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
+      [4, [["f", 400], ["g", 500]], [["s", "b1"], ["t", "b2"]]]
+    ])")
+                      .ValueOrDie();
+    WriteBatch(array3, /*row_kinds=*/{}, merge_writer.get());
+    ASSERT_OK_AND_ASSIGN(CommitIncrement increment,
                          merge_writer->PrepareCommit(/*wait_compaction=*/false));
-    ASSERT_EQ(1, commit_increment1.GetNewFilesIncrement().NewFiles().size());
-    std::string file1_path =
-        path_factory->ToPath(commit_increment1.GetNewFilesIncrement().NewFiles()[0]->file_name);
+    const auto& files = increment.GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+    std::string file1_path = path_factory->ToPath(files[0]->file_name);
+    std::string file2_path = path_factory->ToPath(files[1]->file_name);
 
     std::map column_to_k_file1 = {{"tags", 8}, {"attrs", 4}};
     ASSERT_OK_AND_ASSIGN(auto physical_schema1, MapSharedShreddingUtils::LogicalToPhysicalSchema(
@@ -551,7 +557,6 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation)
     tags_meta1.num_columns = 8;
     tags_meta1.max_row_width = 2;
     CheckShreddingFileSchema(file1_path, physical_schema1, /*field_index=*/3, tags_meta1);
-
     MapSharedShreddingFieldMeta attrs_meta1;
     attrs_meta1.name_to_id = {{"x", 0}};
     attrs_meta1.field_to_columns = {{0, {0}}};
@@ -559,64 +564,24 @@ TEST_P(MergeTreeWriterTest, TestSharedShreddingMultipleMapFieldsWithKAdaptation)
     attrs_meta1.max_row_width = 1;
     CheckShreddingFileSchema(file1_path, physical_schema1, /*field_index=*/4, attrs_meta1);
 
-    auto array2 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
-      [3, [["c", 100], ["d", 200], ["e", 300]], [["p", "a1"], ["q", "a2"], ["r", "a3"]]]
-    ])")
-                      .ValueOrDie();
-    WriteBatch(array2, /*row_kinds=*/{}, merge_writer.get());
-    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment2,
-                         merge_writer->PrepareCommit(/*wait_compaction=*/false));
-    ASSERT_EQ(1, commit_increment2.GetNewFilesIncrement().NewFiles().size());
-    std::string file2_path =
-        path_factory->ToPath(commit_increment2.GetNewFilesIncrement().NewFiles()[0]->file_name);
-
     std::map column_to_k_file2 = {{"tags", 2}, {"attrs", 1}};
     ASSERT_OK_AND_ASSIGN(auto physical_schema2, MapSharedShreddingUtils::LogicalToPhysicalSchema(
                                                     write_schema, column_to_k_file2));
     MapSharedShreddingFieldMeta tags_meta2;
-    tags_meta2.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}};
-    tags_meta2.field_to_columns = {{0, {0}}, {1, {1}}};
+    tags_meta2.name_to_id = {{"c", 0}, {"d", 1}, {"e", 2}, {"f", 3}, {"g", 4}};
+    tags_meta2.field_to_columns = {{0, {0}}, {1, {1}}, {3, {0}}, {4, {1}}};
     tags_meta2.overflow_field_set = {2};
     tags_meta2.num_columns = 2;
     tags_meta2.max_row_width = 3;
     CheckShreddingFileSchema(file2_path, physical_schema2, /*field_index=*/3, tags_meta2);
-
     MapSharedShreddingFieldMeta attrs_meta2;
-    attrs_meta2.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}};
-    attrs_meta2.field_to_columns = {{0, {0}}};
-    attrs_meta2.overflow_field_set = {1, 2};
+    attrs_meta2.name_to_id = {{"p", 0}, {"q", 1}, {"r", 2}, {"s", 3}, {"t", 4}};
+    attrs_meta2.field_to_columns = {{0, {0}}, {3, {0}}};
+    attrs_meta2.overflow_field_set = {1, 2, 4};
     attrs_meta2.num_columns = 1;
     attrs_meta2.max_row_width = 3;
     CheckShreddingFileSchema(file2_path, physical_schema2, /*field_index=*/4, attrs_meta2);
 
-    auto array3 = arrow::ipc::internal::json::ArrayFromJSON(value_type, R"([
-      [4, [["f", 400], ["g", 500]], [["s", "b1"], ["t", "b2"]]]
-    ])")
-                      .ValueOrDie();
-    WriteBatch(array3, /*row_kinds=*/{}, merge_writer.get());
-    ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment3,
-                         merge_writer->PrepareCommit(/*wait_compaction=*/false));
-    ASSERT_EQ(1, commit_increment3.GetNewFilesIncrement().NewFiles().size());
-    std::string file3_path =
-        path_factory->ToPath(commit_increment3.GetNewFilesIncrement().NewFiles()[0]->file_name);
-
-    std::map column_to_k_file3 = {{"tags", 3}, {"attrs", 3}};
-    ASSERT_OK_AND_ASSIGN(auto physical_schema3, MapSharedShreddingUtils::LogicalToPhysicalSchema(
-                                                    write_schema, column_to_k_file3));
-    MapSharedShreddingFieldMeta tags_meta3;
-    tags_meta3.name_to_id = {{"f", 0}, {"g", 1}};
-    tags_meta3.field_to_columns = {{0, {0}}, {1, {1}}};
-    tags_meta3.num_columns = 3;
-    tags_meta3.max_row_width = 2;
-    CheckShreddingFileSchema(file3_path, physical_schema3, /*field_index=*/3, tags_meta3);
-
-    MapSharedShreddingFieldMeta attrs_meta3;
-    attrs_meta3.name_to_id = {{"s", 0}, {"t", 1}};
-    attrs_meta3.field_to_columns = {{0, {0}}, {1, {1}}};
-    attrs_meta3.num_columns = 3;
-    attrs_meta3.max_row_width = 2;
-    CheckShreddingFileSchema(file3_path, physical_schema3, /*field_index=*/4, attrs_meta3);
-
     ASSERT_OK(merge_writer->Close());
 }
 
@@ -1368,8 +1333,7 @@ TEST_F(MergeTreeWriterTest, TestSpillWithSameKeyDeduplicate) {
                                 key_comparator_, /*user_defined_seq_comparator=*/nullptr,
                                 merge_function_wrapper_, /*schema_id=*/0, value_schema_, options,
                                 noop_compact_manager_, io_manager,
-                                /*enable_multi_thread_spill=*/false,
-                                /*shredding_context=*/nullptr, pool_));
+                                /*enable_multi_thread_spill=*/false, pool_));
 
     std::shared_ptr batch1 =
         arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
@@ -1437,8 +1401,7 @@ TEST_F(MergeTreeWriterTest, TestIntermediateMergeSpillFileBound) {
                                 key_comparator_, /*user_defined_seq_comparator=*/nullptr,
                                 merge_function_wrapper_, /*schema_id=*/0, value_schema_, options,
                                 noop_compact_manager_, io_manager,
-                                /*enable_multi_thread_spill=*/false,
-                                /*shredding_context=*/nullptr, pool_));
+                                /*enable_multi_thread_spill=*/false, pool_));
 
     std::shared_ptr batch1 =
         arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
@@ -1504,8 +1467,7 @@ TEST_F(MergeTreeWriterTest, TestDiskQuotaExhaustedFallsBackToFlushWriteBuffer) {
                                 key_comparator_, /*user_defined_seq_comparator=*/nullptr,
                                 merge_function_wrapper_, /*schema_id=*/0, value_schema_, options,
                                 noop_compact_manager_, io_manager,
-                                /*enable_multi_thread_spill=*/false,
-                                /*shredding_context=*/nullptr, pool_));
+                                /*enable_multi_thread_spill=*/false, pool_));
 
     // Phase 1: Manual FlushMemory path — disk quota exhausted causes fallback.
     std::shared_ptr array1 =
@@ -1583,8 +1545,7 @@ TEST_F(MergeTreeWriterTest, TestFlushMemoryQuotaExhaustedFallsBackToFlushWriteBu
                                 key_comparator_, /*user_defined_seq_comparator=*/nullptr,
                                 merge_function_wrapper_, /*schema_id=*/0, value_schema_, options,
                                 noop_compact_manager_, io_manager,
-                                /*enable_multi_thread_spill=*/false,
-                                /*shredding_context=*/nullptr, pool_));
+                                /*enable_multi_thread_spill=*/false, pool_));
 
     std::shared_ptr array =
         arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
@@ -1628,8 +1589,7 @@ TEST_F(MergeTreeWriterTest, TestCloseDeletesSpillTempFiles) {
                                 key_comparator_, /*user_defined_seq_comparator=*/nullptr,
                                 merge_function_wrapper_, /*schema_id=*/0, value_schema_, options,
                                 noop_compact_manager_, io_manager,
-                                /*enable_multi_thread_spill=*/false,
-                                /*shredding_context=*/nullptr, pool_));
+                                /*enable_multi_thread_spill=*/false, pool_));
 
     std::shared_ptr array =
         arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
@@ -1662,8 +1622,7 @@ TEST_F(MergeTreeWriterTest, TestMultiplePrepareCommitWithSpill) {
                                 key_comparator_, /*user_defined_seq_comparator=*/nullptr,
                                 merge_function_wrapper_, /*schema_id=*/0, value_schema_, options,
                                 noop_compact_manager_, io_manager,
-                                /*enable_multi_thread_spill=*/false,
-                                /*shredding_context=*/nullptr, pool_));
+                                /*enable_multi_thread_spill=*/false, pool_));
 
     std::shared_ptr array1 =
         arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
@@ -1741,8 +1700,7 @@ TEST_F(MergeTreeWriterTest, TestSpillWithIOException) {
                                     key_comparator_, /*user_defined_seq_comparator=*/nullptr,
                                     merge_function_wrapper_, /*schema_id=*/0, value_schema_,
                                     options, noop_compact_manager_, io_manager,
-                                    /*enable_multi_thread_spill=*/false,
-                                    /*shredding_context=*/nullptr, pool_));
+                                    /*enable_multi_thread_spill=*/false, pool_));
 
         ScopeGuard guard([&io_hook]() { io_hook->Clear(); });
         io_hook->Reset(i, IOHook::Mode::RETURN_ERROR);
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index 92e521bf..2296013a 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -256,8 +256,7 @@ AbstractSplitRead::ApplySharedShreddingReaderIfNeeded(
         }
         // get meta
         PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingFieldMeta meta,
-                               MapSharedShreddingUtils::DeserializeMetadata(
-                                   metadata, MapSharedShreddingDefine::kDefaultDictCompression));
+                               MapSharedShreddingUtils::DeserializeMetadata(metadata));
         // get selected_keys
         PAIMON_ASSIGN_OR_RAISE(std::vector selected_keys,
                                NestedProjectionUtils::GetMapSelectedKeys(read_field));
diff --git a/src/paimon/core/operation/append_only_file_store_write.cpp b/src/paimon/core/operation/append_only_file_store_write.cpp
index b021e64b..135c49dc 100644
--- a/src/paimon/core/operation/append_only_file_store_write.cpp
+++ b/src/paimon/core/operation/append_only_file_store_write.cpp
@@ -19,6 +19,7 @@
 #include "paimon/core/operation/append_only_file_store_write.h"
 
 #include 
+#include 
 #include 
 
 #include "arrow/c/bridge.h"
@@ -34,7 +35,6 @@
 #include "paimon/core/io/append_data_file_writer_factory.h"
 #include "paimon/core/io/data_file_meta.h"
 #include "paimon/core/io/data_file_path_factory.h"
-#include "paimon/core/io/map_shared_shredding_core_utils.h"
 #include "paimon/core/io/rolling_file_writer.h"
 #include "paimon/core/io/shredding_append_data_file_writer_factory.h"
 #include "paimon/core/manifest/manifest_file.h"
@@ -122,14 +122,13 @@ Result>> AppendOnlyFileStoreWrite::Com
                            CreateFilesReader(partition, bucket, dv_factory, to_compact));
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_file_path_factory,
                            file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
-    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr shredding_context,
-                           MapSharedShreddingCoreUtils::CreateAndRestoreContext(
-                               write_schema_, to_compact, data_file_path_factory, options_, pool_));
+    PAIMON_ASSIGN_OR_RAISE(
+        WriterFactory writer_factory,
+        GetDataFileWriterFactory(data_file_path_factory, write_schema_, write_cols_, to_compact));
     auto rewriter =
         std::make_unique>>(
             options_.GetTargetFileSize(/*has_primary_key=*/false),
-            GetDataFileWriterFactory(data_file_path_factory, write_schema_, write_cols_, to_compact,
-                                     shredding_context));
+            /*target_file_row_num=*/std::numeric_limits::max(), writer_factory);
 
     ScopeGuard reader_guard([&]() {
         if (reader) {
@@ -213,25 +212,21 @@ Result> AppendOnlyFileStoreWrite::CreateWriter(
             compaction_metrics_->CreateReporter(partition, bucket), cancellation_controller);
     }
 
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr shredding_context,
-        MapSharedShreddingCoreUtils::CreateAndRestoreContext(
-            write_schema_, restore_data_files, data_file_path_factory, options_, pool_));
     auto writer = std::make_unique(
         options_, table_schema_->Id(), write_schema_, write_cols_, restore_max_seq_number,
-        data_file_path_factory, compact_manager, shredding_context, pool_);
+        data_file_path_factory, compact_manager, pool_);
     return std::shared_ptr(std::move(writer));
 }
 
-AppendOnlyFileStoreWrite::WriterFactory AppendOnlyFileStoreWrite::GetDataFileWriterFactory(
+Result AppendOnlyFileStoreWrite::GetDataFileWriterFactory(
     const std::shared_ptr& data_file_path_factory,
     const std::shared_ptr& schema,
     const std::optional>& write_cols,
-    const std::vector>& to_compact,
-    const std::shared_ptr& shredding_context) const {
+    const std::vector>& to_compact) const {
     auto seq_num_counter = std::make_shared(to_compact[0]->min_sequence_number);
-    if (auto plan_factory =
-            ShreddingWritePlanFactories::SelectActive(options_, schema, shredding_context, pool_)) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan_factory,
+                           ShreddingWritePlanFactories::SelectActive(options_, schema, pool_));
+    if (plan_factory != nullptr) {
         return std::make_shared(
             options_, table_schema_->Id(), schema, write_cols, seq_num_counter,
             FileSource::Compact(), data_file_path_factory, plan_factory, pool_);
diff --git a/src/paimon/core/operation/append_only_file_store_write.h b/src/paimon/core/operation/append_only_file_store_write.h
index e79bd9b9..905a4180 100644
--- a/src/paimon/core/operation/append_only_file_store_write.h
+++ b/src/paimon/core/operation/append_only_file_store_write.h
@@ -63,7 +63,6 @@ class BinaryRow;
 class CoreOptions;
 class Executor;
 class Logger;
-class MapSharedShreddingContext;
 class MemoryPool;
 class SchemaManager;
 class TableSchema;
@@ -111,12 +110,11 @@ class AppendOnlyFileStoreWrite : public AbstractFileStoreWrite {
     Result> CreateFileStoreScan(
         const std::shared_ptr& filter) const override;
 
-    WriterFactory GetDataFileWriterFactory(
+    Result GetDataFileWriterFactory(
         const std::shared_ptr& data_file_path_factory,
         const std::shared_ptr& schema,
         const std::optional>& write_cols,
-        const std::vector>& to_compact,
-        const std::shared_ptr& shredding_context) const;
+        const std::vector>& to_compact) const;
 
     Result> CreateFilesReader(
         const BinaryRow& partition, int32_t bucket, DeletionVector::Factory dv_factory,
diff --git a/src/paimon/core/operation/append_only_file_store_write_test.cpp b/src/paimon/core/operation/append_only_file_store_write_test.cpp
index 4f330985..df1e1bd2 100644
--- a/src/paimon/core/operation/append_only_file_store_write_test.cpp
+++ b/src/paimon/core/operation/append_only_file_store_write_test.cpp
@@ -173,9 +173,7 @@ class AppendOnlyFileStoreWriteTest : public testing::Test {
                                               int32_t field_index) const {
         auto metadata = file_schema->field(field_index)->metadata();
         EXPECT_NE(nullptr, metadata);
-        return MapSharedShreddingUtils::DeserializeMetadata(
-                   metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression)
-            .value();
+        return MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()).value();
     }
 
  private:
@@ -284,9 +282,10 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestGetMaxSequenceNumberFromMultiPartition)
     }
 }
 
-TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNextWriter) {
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFiles) {
     std::map options = {
         {"file.format", "parquet"},
+        {"target-file-row-num", "1"},
         {"fields.tags.map.storage-layout", "shared-shredding"},
         {"fields.tags.map.shared-shredding.max-columns", "10"},
         {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -305,25 +304,37 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNex
 
     std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
 
-    auto first_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
+    WriteContextBuilder builder(table_path, commit_user_);
+    builder.SetOptions(options);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context)));
+    ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([
         [1, [["a", 1], ["b", 2]]]
-    ])",
-                                             /*commit_identifier=*/0);
-    auto first_file_schema =
-        ReadDataFileSchema(table_path, OnlyNewFile(first_commit_msgs), options);
+    ])")));
+    ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([
+        [2, [["c", 3], ["d", 4], ["e", 5]]]
+    ])")));
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit(
+                                               /*wait_compaction=*/false, /*commit_identifier=*/0));
+    ASSERT_OK(file_store_write->Close());
+
+    ASSERT_EQ(1, commit_msgs.size());
+    auto commit_msg = std::dynamic_pointer_cast(commit_msgs[0]);
+    ASSERT_NE(nullptr, commit_msg);
+    const auto& files = commit_msg->GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+
+    auto first_file_schema = ReadDataFileSchema(table_path, files[0], options);
     auto first_meta = ShreddingMeta(first_file_schema, /*field_index=*/1);
+    ASSERT_OK_AND_ASSIGN(
+        auto expected_first_schema,
+        MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 10}}));
+    ASSERT_TRUE(first_file_schema->Equals(*expected_first_schema, /*check_metadata=*/false));
     ASSERT_EQ(10, first_meta.num_columns);
     ASSERT_EQ(2, first_meta.max_row_width);
-    Commit(table_path, options, first_commit_msgs);
 
-    auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
-        [2, [["c", 3], ["d", 4], ["e", 5]]]
-    ])",
-                                              /*commit_identifier=*/1);
-    auto second_file_schema =
-        ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), options);
+    auto second_file_schema = ReadDataFileSchema(table_path, files[1], options);
     auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/1);
-
     ASSERT_OK_AND_ASSIGN(
         auto expected_second_schema,
         MapSharedShreddingUtils::LogicalToPhysicalSchema(logical_schema, {{"tags", 2}}));
@@ -332,7 +343,7 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNex
     ASSERT_EQ(3, second_meta.max_row_width);
 }
 
-TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreIgnoresAvroFileWithoutMetadata) {
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterIgnoresExistingAvroFile) {
     auto logical_schema = arrow::schema({
         arrow::field("id", arrow::int32()),
         arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
@@ -376,9 +387,10 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreIgnoresAvroFileWi
     ASSERT_EQ(3, second_meta.max_row_width);
 }
 
-TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreMultipleMapColumns) {
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingMultipleMapColumnsAdaptAcrossRollingFiles) {
     std::map options = {
         {"file.format", "parquet"},
+        {"target-file-row-num", "1"},
         {"fields.tags.map.storage-layout", "shared-shredding"},
         {"fields.tags.map.shared-shredding.max-columns", "10"},
         {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -394,52 +406,58 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreMultipleMapColumn
         arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
         arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64())),
     });
-    auto tags_schema = arrow::schema({
-        logical_schema->field(0),
-        logical_schema->field(1),
-    });
-    auto attrs_schema = arrow::schema({
-        logical_schema->field(0),
-        logical_schema->field(2),
-    });
 
     auto dir = UniqueTestDirectory::Create();
     ASSERT_TRUE(dir);
     CreateTable(dir->Str(), logical_schema, options);
     std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
 
-    auto tags_commit_msgs =
-        WriteAndPrepareWithWriteSchema(table_path, tags_schema, options, {"id", "tags"}, R"([
-            [1, [["a", 1], ["b", 2]]]
-        ])",
-                                       /*commit_identifier=*/0);
-    Commit(table_path, options, tags_commit_msgs);
-
-    auto attrs_commit_msgs =
-        WriteAndPrepareWithWriteSchema(table_path, attrs_schema, options, {"id", "attrs"}, R"([
-            [2, [["c", 3], ["d", 4], ["e", 5], ["f", 6]]]
-        ])",
-                                       /*commit_identifier=*/1);
-    Commit(table_path, options, attrs_commit_msgs);
-
-    auto full_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
-        [3, [["g", 7], ["h", 8], ["i", 9]], [["j", 10], ["k", 11], ["l", 12], ["m", 13], ["n", 14]]]
-    ])",
-                                            /*commit_identifier=*/2);
-    auto full_file_schema = ReadDataFileSchema(table_path, OnlyNewFile(full_commit_msgs), options);
-    auto tags_meta = ShreddingMeta(full_file_schema, /*field_index=*/1);
-    auto attrs_meta = ShreddingMeta(full_file_schema, /*field_index=*/2);
-
-    ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
-                                                   logical_schema, {{"tags", 2}, {"attrs", 4}}));
-    ASSERT_TRUE(full_file_schema->Equals(*expected_schema, /*check_metadata=*/false));
-    ASSERT_EQ(2, tags_meta.num_columns);
-    ASSERT_EQ(3, tags_meta.max_row_width);
-    ASSERT_EQ(4, attrs_meta.num_columns);
-    ASSERT_EQ(5, attrs_meta.max_row_width);
+    WriteContextBuilder builder(table_path, commit_user_);
+    builder.SetOptions(options);
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context, builder.Finish());
+    ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context)));
+    ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([
+        [1, [["a", 1], ["b", 2]], [["c", 3], ["d", 4], ["e", 5], ["f", 6]]]
+    ])")));
+    ASSERT_OK(file_store_write->Write(MakeBatch(logical_schema, R"([
+        [2, [["g", 7], ["h", 8], ["i", 9]], [["j", 10], ["k", 11], ["l", 12], ["m", 13], ["n", 14]]]
+    ])")));
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs, file_store_write->PrepareCommit(
+                                               /*wait_compaction=*/false, /*commit_identifier=*/0));
+    ASSERT_OK(file_store_write->Close());
+
+    ASSERT_EQ(1, commit_msgs.size());
+    auto commit_msg = std::dynamic_pointer_cast(commit_msgs[0]);
+    ASSERT_NE(nullptr, commit_msg);
+    const auto& files = commit_msg->GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+
+    auto first_file_schema = ReadDataFileSchema(table_path, files[0], options);
+    auto first_tags_meta = ShreddingMeta(first_file_schema, /*field_index=*/1);
+    auto first_attrs_meta = ShreddingMeta(first_file_schema, /*field_index=*/2);
+    ASSERT_OK_AND_ASSIGN(auto expected_first_schema,
+                         MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                             logical_schema, {{"tags", 10}, {"attrs", 10}}));
+    ASSERT_TRUE(first_file_schema->Equals(*expected_first_schema, /*check_metadata=*/false));
+    ASSERT_EQ(10, first_tags_meta.num_columns);
+    ASSERT_EQ(2, first_tags_meta.max_row_width);
+    ASSERT_EQ(10, first_attrs_meta.num_columns);
+    ASSERT_EQ(4, first_attrs_meta.max_row_width);
+
+    auto second_file_schema = ReadDataFileSchema(table_path, files[1], options);
+    auto second_tags_meta = ShreddingMeta(second_file_schema, /*field_index=*/1);
+    auto second_attrs_meta = ShreddingMeta(second_file_schema, /*field_index=*/2);
+    ASSERT_OK_AND_ASSIGN(auto expected_second_schema,
+                         MapSharedShreddingUtils::LogicalToPhysicalSchema(
+                             logical_schema, {{"tags", 2}, {"attrs", 4}}));
+    ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false));
+    ASSERT_EQ(2, second_tags_meta.num_columns);
+    ASSERT_EQ(3, second_tags_meta.max_row_width);
+    ASSERT_EQ(4, second_attrs_meta.num_columns);
+    ASSERT_EQ(5, second_attrs_meta.max_row_width);
 }
 
-TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreUsesDefaultForMissingMap) {
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingNewWriterUsesMaxForEveryMap) {
     std::map options = {
         {"file.format", "parquet"},
         {"fields.tags.map.storage-layout", "shared-shredding"},
@@ -483,14 +501,65 @@ TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingRestoreUsesDefaultForMis
     auto attrs_meta = ShreddingMeta(full_file_schema, /*field_index=*/2);
 
     ASSERT_OK_AND_ASSIGN(auto expected_schema, MapSharedShreddingUtils::LogicalToPhysicalSchema(
-                                                   logical_schema, {{"tags", 2}, {"attrs", 10}}));
+                                                   logical_schema, {{"tags", 10}, {"attrs", 10}}));
     ASSERT_TRUE(full_file_schema->Equals(*expected_schema, /*check_metadata=*/false));
-    ASSERT_EQ(2, tags_meta.num_columns);
+    ASSERT_EQ(10, tags_meta.num_columns);
     ASSERT_EQ(3, tags_meta.max_row_width);
     ASSERT_EQ(10, attrs_meta.num_columns);
     ASSERT_EQ(4, attrs_meta.max_row_width);
 }
 
+TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingFieldDictCompressionFileRoundTrip) {
+    std::vector file_formats = {"parquet"};
+#ifdef PAIMON_ENABLE_ORC
+    file_formats.emplace_back("orc");
+#endif
+    for (const std::string& file_format : file_formats) {
+        for (const std::string compression : {"none", "zstd"}) {
+            SCOPED_TRACE("format=" + file_format + ", compression=" + compression);
+            std::map options = {
+                {Options::FILE_FORMAT, file_format},
+                {Options::FILE_COMPRESSION, compression},
+                {"fields.tags.map.storage-layout", "shared-shredding"},
+                {"fields.tags.map.shared-shredding.max-columns", "4"},
+                {"write-only", "true"},
+                {"bucket", "1"},
+                {"bucket-key", "id"},
+            };
+            auto logical_schema = arrow::schema({
+                arrow::field("id", arrow::int32()),
+                arrow::field("tags", arrow::map(arrow::utf8(), arrow::int64())),
+            });
+
+            auto dir = UniqueTestDirectory::Create();
+            ASSERT_TRUE(dir);
+            CreateTable(dir->Str(), logical_schema, options);
+            std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
+            auto commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
+                [1, [["a", 1], ["b", 2]]],
+                [2, [["c", 3]]]
+            ])",
+                                               /*commit_identifier=*/0);
+
+            auto file_schema = ReadDataFileSchema(table_path, OnlyNewFile(commit_msgs), options);
+            auto tags_metadata = file_schema->GetFieldByName("tags")->metadata();
+            ASSERT_NE(tags_metadata, nullptr);
+            int32_t compression_index =
+                tags_metadata->FindKey(MapSharedShreddingDefine::kFieldDictCompression);
+            ASSERT_GE(compression_index, 0);
+            ASSERT_EQ(compression, tags_metadata->value(compression_index));
+
+            ASSERT_OK_AND_ASSIGN(
+                MapSharedShreddingFieldMeta field_meta,
+                MapSharedShreddingUtils::DeserializeMetadata(tags_metadata->Copy()));
+            ASSERT_EQ(3, field_meta.name_to_id.size());
+            ASSERT_TRUE(field_meta.name_to_id.count("a"));
+            ASSERT_TRUE(field_meta.name_to_id.count("b"));
+            ASSERT_TRUE(field_meta.name_to_id.count("c"));
+        }
+    }
+}
+
 TEST_F(AppendOnlyFileStoreWriteTest, TestSharedShreddingPartialWriteSkipsMissingMapColumn) {
     std::map options = {
         {"file.format", "parquet"},
diff --git a/src/paimon/core/operation/key_value_file_store_write.cpp b/src/paimon/core/operation/key_value_file_store_write.cpp
index 075d1a96..08c5ea0c 100644
--- a/src/paimon/core/operation/key_value_file_store_write.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write.cpp
@@ -21,10 +21,8 @@
 #include 
 
 #include "paimon/common/data/binary_row.h"
-#include "paimon/common/table/special_fields.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_meta.h"
-#include "paimon/core/io/map_shared_shredding_core_utils.h"
 #include "paimon/core/manifest/manifest_file.h"
 #include "paimon/core/manifest/manifest_list.h"
 #include "paimon/core/mergetree/levels.h"
@@ -108,11 +106,6 @@ Result> KeyValueFileStoreWrite::CreateWriter(
                            file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
     PAIMON_ASSIGN_OR_RAISE(std::vector trimmed_primary_keys,
                            table_schema_->TrimmedPrimaryKeys());
-    auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(schema_);
-    PAIMON_ASSIGN_OR_RAISE(
-        std::shared_ptr shredding_context,
-        MapSharedShreddingCoreUtils::CreateAndRestoreContext(
-            write_schema, restore_data_files, data_file_path_factory, options_, pool_));
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr levels,
         Levels::Create(key_comparator_, restore_data_files, options_.GetNumLevels()));
@@ -124,11 +117,10 @@ Result> KeyValueFileStoreWrite::CreateWriter(
 
     PAIMON_ASSIGN_OR_RAISE(
         std::shared_ptr writer,
-        MergeTreeWriter::Create(restore_max_seq_number, trimmed_primary_keys,
-                                data_file_path_factory, key_comparator_,
-                                user_defined_seq_comparator_, merge_function_wrapper_,
-                                table_schema_->Id(), schema_, options_, compact_manager,
-                                io_manager_, enable_multi_thread_spill_, shredding_context, pool_));
+        MergeTreeWriter::Create(
+            restore_max_seq_number, trimmed_primary_keys, data_file_path_factory, key_comparator_,
+            user_defined_seq_comparator_, merge_function_wrapper_, table_schema_->Id(), schema_,
+            options_, compact_manager, io_manager_, enable_multi_thread_spill_, pool_));
     return writer;
 }
 
diff --git a/src/paimon/core/operation/key_value_file_store_write_test.cpp b/src/paimon/core/operation/key_value_file_store_write_test.cpp
index 705220be..60f2d95a 100644
--- a/src/paimon/core/operation/key_value_file_store_write_test.cpp
+++ b/src/paimon/core/operation/key_value_file_store_write_test.cpp
@@ -188,9 +188,7 @@ class KeyValueFileStoreWriteTest : public ::testing::Test {
                                               int32_t field_index) const {
         auto metadata = file_schema->field(field_index)->metadata();
         EXPECT_NE(nullptr, metadata);
-        return MapSharedShreddingUtils::DeserializeMetadata(
-                   metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression)
-            .value();
+        return MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()).value();
     }
 };
 
@@ -314,9 +312,11 @@ TEST_F(KeyValueFileStoreWriteTest,
     ASSERT_EQ(commit_messages.size(), 1);
 }
 
-TEST_F(KeyValueFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNextWriter) {
+TEST_F(KeyValueFileStoreWriteTest, TestSharedShreddingMapAdaptsAcrossRollingFiles) {
     std::map options = {
         {"file.format", "parquet"},
+        {"target-file-row-num", "2"},
+        {"write.batch-size", "2"},
         {"fields.tags.map.storage-layout", "shared-shredding"},
         {"fields.tags.map.shared-shredding.max-columns", "10"},
         {"fields.tags.map.shared-shredding.column-placement-policy", "plain"},
@@ -334,31 +334,37 @@ TEST_F(KeyValueFileStoreWriteTest, TestSharedShreddingMapRestoreInitializesNextW
     CreateTable(dir->Str(), logical_schema, options);
     std::string table_path = PathUtil::JoinPath(dir->Str(), "foo.db/bar");
 
-    auto first_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
-        [1, [["a", 1], ["b", 2]]]
+    auto commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
+        [1, [["a", 1], ["b", 2]]],
+        [2, [["c", 3], ["d", 4], ["e", 5]]],
+        [3, [["f", 6]]],
+        [4, [["g", 7], ["h", 8]]]
     ])",
-                                             /*commit_identifier=*/0);
-    auto first_file_schema =
-        ReadDataFileSchema(table_path, OnlyNewFile(first_commit_msgs), options);
+                                       /*commit_identifier=*/0);
+
+    ASSERT_EQ(1, commit_msgs.size());
+    auto commit_msg = std::dynamic_pointer_cast(commit_msgs[0]);
+    ASSERT_NE(nullptr, commit_msg);
+    const auto& files = commit_msg->GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+
+    auto first_file_schema = ReadDataFileSchema(table_path, files[0], options);
     auto first_meta = ShreddingMeta(first_file_schema, /*field_index=*/3);
+    ASSERT_OK_AND_ASSIGN(
+        auto expected_first_schema,
+        MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 10}}));
+    ASSERT_TRUE(first_file_schema->Equals(*expected_first_schema, /*check_metadata=*/false));
     ASSERT_EQ(10, first_meta.num_columns);
-    ASSERT_EQ(2, first_meta.max_row_width);
-    Commit(table_path, options, first_commit_msgs);
+    ASSERT_EQ(3, first_meta.max_row_width);
 
-    auto second_commit_msgs = WriteAndPrepare(table_path, logical_schema, options, R"([
-        [2, [["c", 3], ["d", 4], ["e", 5]]]
-    ])",
-                                              /*commit_identifier=*/1);
-    auto second_file_schema =
-        ReadDataFileSchema(table_path, OnlyNewFile(second_commit_msgs), options);
+    auto second_file_schema = ReadDataFileSchema(table_path, files[1], options);
     auto second_meta = ShreddingMeta(second_file_schema, /*field_index=*/3);
-
     ASSERT_OK_AND_ASSIGN(
         auto expected_second_schema,
-        MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 2}}));
+        MapSharedShreddingUtils::LogicalToPhysicalSchema(write_schema, {{"tags", 3}}));
     ASSERT_TRUE(second_file_schema->Equals(*expected_second_schema, /*check_metadata=*/false));
-    ASSERT_EQ(2, second_meta.num_columns);
-    ASSERT_EQ(3, second_meta.max_row_width);
+    ASSERT_EQ(3, second_meta.num_columns);
+    ASSERT_EQ(2, second_meta.max_row_width);
 }
 
 TEST_F(KeyValueFileStoreWriteTest, TestSpillSimple) {
diff --git a/src/paimon/core/postpone/postpone_bucket_file_store_write.h b/src/paimon/core/postpone/postpone_bucket_file_store_write.h
index 567b0267..1ca65d28 100644
--- a/src/paimon/core/postpone/postpone_bucket_file_store_write.h
+++ b/src/paimon/core/postpone/postpone_bucket_file_store_write.h
@@ -25,9 +25,7 @@
 #include 
 #include 
 
-#include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/preconditions.h"
-#include "paimon/core/io/map_shared_shredding_core_utils.h"
 #include "paimon/core/operation/abstract_file_store_write.h"
 #include "paimon/core/operation/file_store_scan.h"
 #include "paimon/core/postpone/postpone_bucket_writer.h"
@@ -129,16 +127,10 @@ class PostponeBucketFileStoreWrite : public AbstractFileStoreWrite {
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr data_file_path_factory,
             file_store_path_factory_->CreateDataFilePathFactory(partition, bucket));
-        auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(schema_);
-        PAIMON_ASSIGN_OR_RAISE(
-            std::shared_ptr shredding_context,
-            MapSharedShreddingCoreUtils::CreateAndRestoreContext(
-                write_schema, restore_data_files, data_file_path_factory, options_, pool_));
         PAIMON_ASSIGN_OR_RAISE(
             std::shared_ptr writer,
             PostponeBucketWriter::Create(trimmed_primary_keys, data_file_path_factory,
-                                         table_schema_->Id(), schema_, options_, shredding_context,
-                                         pool_));
+                                         table_schema_->Id(), schema_, options_, pool_));
         return writer;
     }
 
diff --git a/src/paimon/core/postpone/postpone_bucket_writer.cpp b/src/paimon/core/postpone/postpone_bucket_writer.cpp
index 208a0a29..fb27d001 100644
--- a/src/paimon/core/postpone/postpone_bucket_writer.cpp
+++ b/src/paimon/core/postpone/postpone_bucket_writer.cpp
@@ -74,21 +74,19 @@ Result> PostponeBucketWriter::Create(
     const std::vector& trimmed_primary_keys,
     const std::shared_ptr& path_factory, int64_t schema_id,
     const std::shared_ptr& value_schema, const CoreOptions& options,
-    const std::shared_ptr& shredding_context,
     const std::shared_ptr& pool) {
     auto write_schema = BuildPostponeBucketWriteSchema(value_schema);
-    return std::unique_ptr(
-        new PostponeBucketWriter(trimmed_primary_keys, path_factory, schema_id, value_schema,
-                                 write_schema, options, pool, shredding_context));
+    return std::unique_ptr(new PostponeBucketWriter(
+        trimmed_primary_keys, path_factory, schema_id, value_schema, write_schema, options, pool));
 }
 
-PostponeBucketWriter::PostponeBucketWriter(
-    const std::vector& trimmed_primary_keys,
-    const std::shared_ptr& path_factory, int64_t schema_id,
-    const std::shared_ptr& value_schema,
-    const std::shared_ptr& write_schema, const CoreOptions& options,
-    const std::shared_ptr& pool,
-    const std::shared_ptr& shredding_context)
+PostponeBucketWriter::PostponeBucketWriter(const std::vector& trimmed_primary_keys,
+                                           const std::shared_ptr& path_factory,
+                                           int64_t schema_id,
+                                           const std::shared_ptr& value_schema,
+                                           const std::shared_ptr& write_schema,
+                                           const CoreOptions& options,
+                                           const std::shared_ptr& pool)
     : pool_(pool),
       arrow_pool_(GetArrowPool(pool)),
       trimmed_primary_keys_(trimmed_primary_keys),
@@ -97,7 +95,6 @@ PostponeBucketWriter::PostponeBucketWriter(
       schema_id_(schema_id),
       value_type_(arrow::struct_(value_schema->fields())),
       write_schema_(write_schema),
-      shredding_context_(shredding_context),
       metrics_(std::make_shared()) {}
 
 Status PostponeBucketWriter::Write(std::unique_ptr&& moved_batch) {
@@ -139,7 +136,10 @@ Status PostponeBucketWriter::Write(std::unique_ptr&& moved_batch) {
 
     // write KeyValueBatch to RollingFileWriter
     if (!writer_) {
-        writer_ = CreateRollingRowWriter();
+        std::unique_ptr>>
+            rolling_writer;
+        PAIMON_ASSIGN_OR_RAISE(rolling_writer, CreateRollingRowWriter());
+        writer_ = std::move(rolling_writer);
     }
     PAIMON_RETURN_NOT_OK(writer_->Write(std::move(key_value_batch)));
     return Status::OK();
@@ -261,11 +261,13 @@ PostponeBucketWriter::PrepareMinMaxKey(
                                       value_struct_array->length() - 1));
 }
 
-std::unique_ptr>>
+Result>>>
 PostponeBucketWriter::CreateRollingRowWriter() const {
     std::shared_ptr>> factory;
-    if (auto plan_factory = ShreddingWritePlanFactories::SelectActive(options_, write_schema_,
-                                                                      shredding_context_, pool_)) {
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr plan_factory,
+        ShreddingWritePlanFactories::SelectActive(options_, write_schema_, pool_));
+    if (plan_factory != nullptr) {
         factory = std::make_shared(
             options_, schema_id_, write_schema_, /*level=*/0, FileSource::Append(),
             trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, plan_factory,
@@ -276,7 +278,8 @@ PostponeBucketWriter::CreateRollingRowWriter() const {
             trimmed_primary_keys_, path_factory_, /*create_stats_extractor=*/false, pool_);
     }
     return std::make_unique>>(
-        options_.GetTargetFileSize(/*has_primary_key=*/true), factory);
+        options_.GetTargetFileSize(/*has_primary_key=*/true), options_.GetTargetFileRowNum(),
+        factory);
 }
 
 Status PostponeBucketWriter::Flush() {
diff --git a/src/paimon/core/postpone/postpone_bucket_writer.h b/src/paimon/core/postpone/postpone_bucket_writer.h
index d8c02a2e..f61c9887 100644
--- a/src/paimon/core/postpone/postpone_bucket_writer.h
+++ b/src/paimon/core/postpone/postpone_bucket_writer.h
@@ -46,7 +46,6 @@ struct ArrowArray;
 
 namespace paimon {
 class DataFilePathFactory;
-class MapSharedShreddingContext;
 class MemoryPool;
 class Metrics;
 
@@ -56,7 +55,6 @@ class PostponeBucketWriter : public BatchWriter {
         const std::vector& trimmed_primary_keys,
         const std::shared_ptr& path_factory, int64_t schema_id,
         const std::shared_ptr& value_schema, const CoreOptions& options,
-        const std::shared_ptr& shredding_context,
         const std::shared_ptr& pool);
 
     ~PostponeBucketWriter() override {
@@ -124,15 +122,14 @@ class PostponeBucketWriter : public BatchWriter {
     Status Flush();
     Result DrainIncrement();
 
-    std::unique_ptr>>
+    Result>>>
     CreateRollingRowWriter() const;
 
     PostponeBucketWriter(const std::vector& trimmed_primary_keys,
                          const std::shared_ptr& path_factory,
                          int64_t schema_id, const std::shared_ptr& value_schema,
                          const std::shared_ptr& write_schema,
-                         const CoreOptions& options, const std::shared_ptr& pool,
-                         const std::shared_ptr& shredding_context);
+                         const CoreOptions& options, const std::shared_ptr& pool);
 
  private:
     std::shared_ptr pool_;
@@ -144,7 +141,6 @@ class PostponeBucketWriter : public BatchWriter {
     // write_schema = value_schema + special fields
     std::shared_ptr value_type_;
     std::shared_ptr write_schema_;
-    std::shared_ptr shredding_context_;
     std::shared_ptr metrics_;
     std::vector> new_files_;
     std::unique_ptr>> writer_;
diff --git a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp
index 8b466a3e..a629a6bd 100644
--- a/src/paimon/core/postpone/postpone_bucket_writer_test.cpp
+++ b/src/paimon/core/postpone/postpone_bucket_writer_test.cpp
@@ -133,10 +133,8 @@ class PostponeBucketWriterTest : public ::testing::Test,
 
         auto metadata = file_schema->field(field_index)->metadata();
         ASSERT_NE(nullptr, metadata);
-        ASSERT_OK_AND_ASSIGN(
-            auto actual_meta,
-            MapSharedShreddingUtils::DeserializeMetadata(
-                metadata->Copy(), MapSharedShreddingDefine::kDefaultDictCompression));
+        ASSERT_OK_AND_ASSIGN(auto actual_meta,
+                             MapSharedShreddingUtils::DeserializeMetadata(metadata->Copy()));
         ASSERT_EQ(expected_meta, actual_meta);
     }
 
@@ -175,10 +173,9 @@ TEST_P(PostponeBucketWriterTest, TestSimple) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    ASSERT_OK_AND_ASSIGN(
-        auto postpone_bucket_writer,
-        PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_,
-                                     options, /*shredding_context=*/nullptr, pool_));
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // write batch
     std::shared_ptr array1 =
@@ -256,8 +253,7 @@ TEST_P(PostponeBucketWriterTest, TestNestedType) {
     ASSERT_OK_AND_ASSIGN(
         auto postpone_bucket_writer,
         PostponeBucketWriter::Create(std::vector{"key"}, path_factory, /*schema_id=*/1,
-                                     arrow::schema(fields), options,
-                                     /*shredding_context=*/nullptr, pool_));
+                                     arrow::schema(fields), options, pool_));
 
     // write batch
     auto array1 = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
@@ -336,14 +332,10 @@ TEST_F(PostponeBucketWriterTest, TestSharedShreddingMap) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
     auto value_schema = arrow::schema(fields);
-    auto write_schema = SpecialFields::CompleteSequenceAndValueKindField(value_schema);
-    ASSERT_OK_AND_ASSIGN(auto shredding_context,
-                         MapSharedShreddingUtils::CreateShreddingContext(write_schema, options));
-
     ASSERT_OK_AND_ASSIGN(
         auto postpone_bucket_writer,
         PostponeBucketWriter::Create(std::vector{"key"}, path_factory, /*schema_id=*/1,
-                                     value_schema, options, shredding_context, pool_));
+                                     value_schema, options, pool_));
 
     auto array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
         ["Lucy", [["a", 1], ["b", 2]]],
@@ -402,10 +394,9 @@ TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    ASSERT_OK_AND_ASSIGN(
-        auto postpone_bucket_writer,
-        PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_,
-                                     options, /*shredding_context=*/nullptr, pool_));
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // write batch 1, batch size = 3
     std::shared_ptr array1 =
@@ -489,6 +480,45 @@ TEST_P(PostponeBucketWriterTest, TestWriteMultiBatch) {
     ASSERT_EQ(expected_data_increment, commit_increment.GetNewFilesIncrement());
 }
 
+TEST_P(PostponeBucketWriterTest, TargetFileRowNumRollsOnlyAfterWholeBatch) {
+    auto file_format = GetParam();
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({{Options::FILE_FORMAT, file_format},
+                                               {Options::TARGET_FILE_ROW_NUM, "2"}}));
+
+    auto dir = UniqueTestDirectory::Create();
+    ASSERT_TRUE(dir);
+    auto path_factory = std::make_shared();
+    ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
+
+    // The first batch exceeds the target by itself. It must remain intact in one file instead of
+    // being sliced at two rows.
+    auto first_batch = arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
+      ["David", 120, 11, null],
+      ["Bob", 140, 12, null],
+      ["Alex", 110, 10, null]
+    ])")
+                           .ValueOrDie();
+    WriteBatch(first_batch, /*row_kinds=*/{}, postpone_bucket_writer.get());
+
+    auto second_batch = arrow::ipc::internal::json::ArrayFromJSON(value_type_, R"([
+      ["Lucy", 20, 1, 14.1]
+    ])")
+                            .ValueOrDie();
+    WriteBatch(second_batch, /*row_kinds=*/{}, postpone_bucket_writer.get());
+
+    ASSERT_OK_AND_ASSIGN(CommitIncrement increment,
+                         postpone_bucket_writer->PrepareCommit(/*wait_compaction=*/false));
+    ASSERT_OK(postpone_bucket_writer->Close());
+    const auto& files = increment.GetNewFilesIncrement().NewFiles();
+    ASSERT_EQ(2, files.size());
+    EXPECT_EQ(3, files[0]->row_count);
+    EXPECT_EQ(1, files[1]->row_count);
+}
+
 TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) {
     auto file_format = GetParam();
     ASSERT_OK_AND_ASSIGN(CoreOptions options,
@@ -501,10 +531,9 @@ TEST_P(PostponeBucketWriterTest, TestMultiplePrepareCommit) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    ASSERT_OK_AND_ASSIGN(
-        auto postpone_bucket_writer,
-        PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_,
-                                     options, /*shredding_context=*/nullptr, pool_));
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // write batch 1, batch size = 3
     std::shared_ptr array1 =
@@ -632,10 +661,9 @@ TEST_P(PostponeBucketWriterTest, TestPrepareCommitForEmptyData) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    ASSERT_OK_AND_ASSIGN(
-        auto postpone_bucket_writer,
-        PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_,
-                                     options, /*shredding_context=*/nullptr, pool_));
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // prepare commit, without write
     ASSERT_OK_AND_ASSIGN(CommitIncrement commit_increment,
@@ -674,10 +702,9 @@ TEST_P(PostponeBucketWriterTest, TestCloseBeforePrepareCommit) {
     ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
     std::string uuid = path_factory->uuid_;
 
-    ASSERT_OK_AND_ASSIGN(
-        auto postpone_bucket_writer,
-        PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1, value_schema_,
-                                     options, /*shredding_context=*/nullptr, pool_));
+    ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
+                         PostponeBucketWriter::Create(primary_keys_, path_factory, /*schema_id=*/1,
+                                                      value_schema_, options, pool_));
 
     // write batch
     std::shared_ptr array1 =
@@ -708,10 +735,10 @@ TEST_P(PostponeBucketWriterTest, TestIOException) {
         ASSERT_OK(path_factory->Init(dir->Str(), file_format, options.DataFilePrefix(), nullptr));
         std::string uuid = path_factory->uuid_;
 
-        ASSERT_OK_AND_ASSIGN(auto postpone_bucket_writer,
-                             PostponeBucketWriter::Create(primary_keys_, path_factory,
-                                                          /*schema_id=*/1, value_schema_, options,
-                                                          /*shredding_context=*/nullptr, pool_));
+        ASSERT_OK_AND_ASSIGN(
+            auto postpone_bucket_writer,
+            PostponeBucketWriter::Create(primary_keys_, path_factory,
+                                         /*schema_id=*/1, value_schema_, options, pool_));
 
         // write batch
         std::shared_ptr array =
diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp
index e01afe01..b9ea5da2 100644
--- a/src/paimon/core/schema/arrow_schema_validator.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator.cpp
@@ -231,6 +231,10 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr&
                 arrow::internal::checked_cast(*field->type()).key_field();
             const auto& item_field =
                 arrow::internal::checked_cast(*field->type()).item_field();
+            if (key_field->nullable()) {
+                return Status::Invalid(
+                    fmt::format("Map field '{}' has a nullable key.", field->name()));
+            }
             PAIMON_RETURN_NOT_OK(ValidateField(key_field, /*allow_blob=*/false));
             PAIMON_RETURN_NOT_OK(ValidateField(item_field, /*allow_blob=*/false));
             break;
diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp
index ecb83b61..ed79e566 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -21,6 +21,7 @@
 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -30,6 +31,7 @@
 #include 
 
 #include "arrow/type.h"
+#include "arrow/util/checked_cast.h"
 #include "fmt/format.h"
 #include "fmt/ranges.h"
 #include "paimon/common/data/blob_utils.h"
@@ -52,6 +54,69 @@
 #include "paimon/result.h"
 
 namespace paimon {
+namespace {
+
+bool ContainsBlobField(const std::shared_ptr& field) {
+    if (BlobUtils::IsBlobField(field)) {
+        return true;
+    }
+    const std::shared_ptr& type = field->type();
+    if (type->id() == arrow::Type::STRUCT) {
+        for (const auto& child : type->fields()) {
+            if (ContainsBlobField(child)) {
+                return true;
+            }
+        }
+    } else if (type->id() == arrow::Type::LIST) {
+        return ContainsBlobField(type->fields().front());
+    } else if (type->id() == arrow::Type::MAP) {
+        const auto& map_type = arrow::internal::checked_cast(*type);
+        return ContainsBlobField(map_type.key_field()) || ContainsBlobField(map_type.item_field());
+    }
+    return false;
+}
+
+Status ValidateSharedShreddingCompression(const std::string& option_key,
+                                          const std::string& compression) {
+    std::string normalized = StringUtils::ToLowerCase(compression);
+    if (normalized != "none" && normalized != "lz4" && normalized != "zstd") {
+        return Status::Invalid(fmt::format(
+            "MAP shared-shredding only supports none/lz4/zstd compression, but {} is {}.",
+            option_key, compression));
+    }
+    return Status::OK();
+}
+
+Status ValidateSharedShreddingFileFormat(const std::string& option_key,
+                                         const std::string& file_format) {
+    std::string normalized = StringUtils::ToLowerCase(file_format);
+    if (normalized != "parquet" && normalized != "orc") {
+        return Status::Invalid(fmt::format(
+            "MAP shared-shredding only supports parquet/orc file formats, but {} is {}.",
+            option_key, file_format));
+    }
+    return Status::OK();
+}
+
+Status ValidatePerLevelOption(
+    const std::map& options, const std::string& option_key,
+    const std::function& validator) {
+    auto it = options.find(option_key);
+    if (it == options.end() || it->second.empty()) {
+        return Status::OK();
+    }
+    auto entries = StringUtils::Split(it->second, std::string(","));
+    for (const std::string& entry : entries) {
+        auto level_and_value = StringUtils::Split(entry, std::string(":"));
+        if (level_and_value.size() == 2) {
+            PAIMON_RETURN_NOT_OK(
+                validator(option_key + "." + level_and_value[0], level_and_value[1]));
+        }
+    }
+    return Status::OK();
+}
+
+}  // namespace
 
 bool SchemaValidation::IsComplexType(const std::shared_ptr& field) {
     arrow::Type::type arrow_type_id = field->type()->id();
@@ -500,6 +565,7 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema,
     }
 
     std::string fields_prefix_str = std::string(Options::FIELDS_PREFIX);
+    bool has_shared_shredding = false;
     for (const auto& [key, value] : options_map) {
         if (!StringUtils::StartsWith(key, fields_prefix_str)) {
             continue;
@@ -535,6 +601,7 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema,
         if (layout != MapStorageLayout::SHARED_SHREDDING) {
             continue;
         }
+        has_shared_shredding = true;
         for (const auto& field : schema.Fields()) {
             if (VariantTypeUtils::ContainsVariantField(field.ArrowField())) {
                 return Status::Invalid(
@@ -545,14 +612,42 @@ Status SchemaValidation::ValidateMapStorageLayout(const TableSchema& schema,
         if (!MapSharedShreddingUtils::IsShreddingKeyMap(field_type)) {
             return Status::Invalid(
                 fmt::format("Column '{}' is configured with map.storage-layout=shared-shredding "
-                            "but its type is not MAP.",
+                            "but its type is not MAP.",
+                            field_name));
+        }
+        auto map_type = arrow::internal::checked_pointer_cast(field_type);
+        if (map_type->key_field()->nullable()) {
+            return Status::Invalid(
+                fmt::format("Column '{}' is configured with map.storage-layout=shared-shredding "
+                            "but its map key type is nullable.",
                             field_name));
         }
+        if (ContainsBlobField(map_type->item_field())) {
+            return Status::Invalid("MAP shared-shredding currently cannot contain BLOB fields.");
+        }
         // Validate max-columns config
         PAIMON_RETURN_NOT_OK(options.GetMapSharedShreddingMaxColumns(field_name));
         // Validate placement policy config
         PAIMON_RETURN_NOT_OK(options.GetMapSharedShreddingColumnPlacementPolicy(field_name));
     }
+    if (!has_shared_shredding) {
+        return Status::OK();
+    }
+
+    if (IsPostponeBucketTable(schema, options.GetBucket())) {
+        return Status::Invalid(
+            "MAP shared-shredding currently does not support postpone bucket mode.");
+    }
+
+    PAIMON_RETURN_NOT_OK(ValidateSharedShreddingFileFormat(Options::FILE_FORMAT,
+                                                           options.GetFileFormat()->Identifier()));
+    PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_FORMAT_PER_LEVEL,
+                                                ValidateSharedShreddingFileFormat));
+    PAIMON_RETURN_NOT_OK(ValidateSharedShreddingCompression(Options::FILE_COMPRESSION,
+                                                            options.GetFileCompression()));
+    PAIMON_RETURN_NOT_OK(ValidatePerLevelOption(options_map, Options::FILE_COMPRESSION_PER_LEVEL,
+                                                ValidateSharedShreddingCompression));
+
     return Status::OK();
 }
 
diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp
index 09a31c40..57179e8c 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -872,7 +872,7 @@ TEST(SchemaValidationTest, TestMapStorageLayout) {
                              TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
                                                  /*primary_keys=*/{"f0", "f1"}, options));
         ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
-                            "not MAP");
+                            "not MAP");
     }
     // Invalid: nested MAP paths are not shared-shredding columns; only top-level columns are
     // addressable by fields..map.storage-layout.
@@ -938,4 +938,163 @@ TEST(SchemaValidationTest, TestMapStorageLayout) {
     }
 }
 
+TEST(SchemaValidationTest, TestMapRequiresNonNullableKey) {
+    auto nullable_key_map =
+        std::make_shared(arrow::field("key", arrow::utf8(), /*nullable=*/true),
+                                         arrow::field("value", arrow::int64()));
+    auto schema = arrow::schema({
+        arrow::field("f0", arrow::utf8()),
+        arrow::field("f1", nullable_key_map),
+    });
+    std::map options = {
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "f0"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+    ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                            /*primary_keys=*/{}, options),
+                        "Map field 'f1' has a nullable key.");
+}
+
+TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) {
+    auto direct_blob_map =
+        arrow::map(arrow::utf8(), BlobUtils::ToArrowField("value", /*nullable=*/true));
+    auto nested_blob_map = arrow::map(
+        arrow::utf8(), arrow::field("value", arrow::struct_({BlobUtils::ToArrowField("blob")})));
+    std::map options = {
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "f0"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+
+    for (const auto& map_type : {direct_blob_map, nested_blob_map}) {
+        auto schema = arrow::schema({
+            arrow::field("f0", arrow::utf8()),
+            arrow::field("f1", map_type),
+        });
+        ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                /*primary_keys=*/{}, options),
+                            "Blob field must be a top-level field.");
+    }
+}
+
+TEST(SchemaValidationTest, TestMapSharedShreddingCompression) {
+    auto schema = arrow::schema({
+        arrow::field("f0", arrow::utf8()),
+        arrow::field("f1", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+    std::map base_options = {
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "f0"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+
+    for (const std::string compression : {"none", "lz4", "zstd", "ZSTD"}) {
+        auto options = base_options;
+        options[Options::FILE_COMPRESSION] = compression;
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_COMPRESSION] = "snappy";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports none/lz4/zstd compression, but "
+                            "file.compression is snappy.");
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_COMPRESSION] = "";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports none/lz4/zstd compression, but "
+                            "file.compression is .");
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_COMPRESSION_PER_LEVEL] = "0:lz4,1:snappy";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports none/lz4/zstd compression, but "
+                            "file.compression.per.level.1 is snappy.");
+    }
+    {
+        auto options = base_options;
+        options.erase("fields.f1.map.storage-layout");
+        options[Options::FILE_COMPRESSION] = "snappy";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    }
+}
+
+TEST(SchemaValidationTest, TestMapSharedShreddingFileFormat) {
+    auto schema = arrow::schema({
+        arrow::field("f0", arrow::utf8()),
+        arrow::field("f1", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+    std::map base_options = {
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "f0"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+
+    for (const std::string file_format : {"parquet", "orc"}) {
+        auto options = base_options;
+        options[Options::FILE_FORMAT] = file_format;
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_FORMAT] = "avro";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(
+            SchemaValidation::ValidateTableSchema(*table_schema),
+            "MAP shared-shredding only supports parquet/orc file formats, but file.format is "
+            "avro.");
+    }
+    {
+        auto options = base_options;
+        options[Options::FILE_FORMAT_PER_LEVEL] = "0:parquet,1:avro";
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                             TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                                 /*primary_keys=*/{}, options));
+        ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                            "MAP shared-shredding only supports parquet/orc file formats, but "
+                            "file.format.per.level.1 is avro.");
+    }
+}
+
+TEST(SchemaValidationTest, TestMapSharedShreddingRejectsPostponeBucketMode) {
+    auto schema = arrow::schema({
+        arrow::field("f0", arrow::utf8()),
+        arrow::field("f1", arrow::map(arrow::utf8(), arrow::int64())),
+    });
+    std::map options = {
+        {Options::BUCKET, "-2"},
+        {Options::WRITE_ONLY, "true"},
+        {"fields.f1.map.storage-layout", "shared-shredding"},
+    };
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema,
+                         TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{},
+                                             /*primary_keys=*/{"f0"}, options));
+    ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+                        "MAP shared-shredding currently does not support postpone bucket mode.");
+}
+
 }  // namespace paimon::test
diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp
index ec434261..2beff595 100644
--- a/src/paimon/core/schema/table_schema.cpp
+++ b/src/paimon/core/schema/table_schema.cpp
@@ -125,8 +125,11 @@ Result> TableSchema::AssignFieldIdsRecursively(
             key_field, AssignFieldIdsRecursively(key_field, /*set_field_id=*/false, field_id));
         PAIMON_ASSIGN_OR_RAISE(
             value_field, AssignFieldIdsRecursively(value_field, /*set_field_id=*/false, field_id));
-        return arrow::field(field->name(), arrow::map(key_field->type(), value_field),
-                            field->nullable(), metadata);
+        // Paimon MAP does not expose Arrow's keys_sorted property. Normalize it so an
+        // in-memory schema and the same schema reloaded from JSON remain equivalent.
+        auto new_map_type =
+            std::make_shared(key_field, value_field, /*keys_sorted=*/false);
+        return arrow::field(field->name(), new_map_type, field->nullable(), metadata);
     }
     return metadata ? field->WithMergedMetadata(metadata) : field;
 }
diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp
index 0dcc74fe..988f17f9 100644
--- a/src/paimon/core/schema/table_schema_test.cpp
+++ b/src/paimon/core/schema/table_schema_test.cpp
@@ -1279,6 +1279,34 @@ TEST_F(TableSchemaTest, MapKeyMustBeNotNull) {
     })";
     ASSERT_NOK_WITH_MSG(TableSchema::CreateFromJson(table_schema_str),
                         "Map field 'f0' has a nullable key.");
+
+    auto nullable_key_map =
+        std::make_shared(arrow::field("key", arrow::int8(), /*nullable=*/true),
+                                         arrow::field("value", arrow::int16()));
+    ASSERT_NOK_WITH_MSG(
+        TableSchema::Create(/*schema_id=*/0, arrow::schema({arrow::field("f0", nullable_key_map)}),
+                            /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{}),
+        "Map field 'f0' has a nullable key.");
+}
+
+TEST_F(TableSchemaTest, MapKeysSortedIsNormalized) {
+    auto sorted_map =
+        std::make_shared(arrow::field("key", arrow::utf8(), /*nullable=*/false),
+                                         arrow::field("value", arrow::int64()),
+                                         /*keys_sorted=*/true);
+    ASSERT_OK_AND_ASSIGN(
+        auto table_schema,
+        TableSchema::Create(/*schema_id=*/0, arrow::schema({arrow::field("f0", sorted_map)}),
+                            /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{}));
+
+    auto map_type = std::static_pointer_cast(table_schema->Fields()[0].Type());
+    ASSERT_FALSE(map_type->keys_sorted());
+
+    ASSERT_OK_AND_ASSIGN(std::string json, table_schema->ToJsonString());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr restored, TableSchema::CreateFromJson(json));
+    auto restored_map_type = std::static_pointer_cast(restored->Fields()[0].Type());
+    ASSERT_FALSE(restored_map_type->keys_sorted());
+    ASSERT_TRUE(map_type->Equals(*restored_map_type));
 }
 
 TEST_F(TableSchemaTest, CrossPartitionUpdate) {
diff --git a/src/paimon/format/parquet/variant_parquet_test.cpp b/src/paimon/format/parquet/variant_parquet_test.cpp
index b2bf1693..b2235b2a 100644
--- a/src/paimon/format/parquet/variant_parquet_test.cpp
+++ b/src/paimon/format/parquet/variant_parquet_test.cpp
@@ -34,11 +34,13 @@
 #include "paimon/common/data/variant/variant_shredding_read_plan_factory.h"
 #include "paimon/common/data/variant/variant_shredding_utils.h"
 #include "paimon/common/data/variant/variant_shredding_write_plan.h"
+#include "paimon/common/data/variant/variant_shredding_write_plan_factory.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/types/data_field.h"
 #include "paimon/common/utils/arrow/arrow_input_stream_adapter.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/path_util.h"
+#include "paimon/core/core_options.h"
 #include "paimon/data/variant.h"
 #include "paimon/format/parquet/parquet_field_id_converter.h"
 #include "paimon/format/parquet/parquet_file_batch_reader.h"
@@ -144,12 +146,8 @@ class VariantParquetTest : public ::testing::Test {
         WriteFile(paimon_schema_, arrow_array.get());
     }
 
-    // Writes `jsons` shredded according to the configured ROW-type shredding schema JSON.
     void WriteShreddedFile(const std::vector& jsons,
-                           const char* shredding_schema_json) {
-        ASSERT_OK_AND_ASSIGN(
-            std::shared_ptr plan,
-            VariantShreddingWritePlan::FromConfiguredSchema(paimon_schema_, shredding_schema_json));
+                           const std::shared_ptr& plan) {
         ASSERT_NE(plan, nullptr);
         ASSERT_OK_AND_ASSIGN(std::shared_ptr converter,
                              VariantShreddingBatchConverter::Create(plan, pool_));
@@ -161,6 +159,24 @@ class VariantParquetTest : public ::testing::Test {
         WriteFile(converter->GetPhysicalSchema(), c_physical.get());
     }
 
+    // Writes `jsons` shredded according to the configured ROW-type shredding schema JSON.
+    void WriteShreddedFile(const std::vector& jsons,
+                           const char* shredding_schema_json) {
+        ASSERT_OK_AND_ASSIGN(
+            std::shared_ptr plan,
+            VariantShreddingWritePlan::FromConfiguredSchema(paimon_schema_, shredding_schema_json));
+        WriteShreddedFile(jsons, plan);
+    }
+
+    // Writes `jsons` using the given inferred shredding type for the top-level Variant column.
+    void WriteShreddedFile(const std::vector& jsons,
+                           const std::shared_ptr& shredding_type) {
+        ASSERT_OK_AND_ASSIGN(
+            std::shared_ptr plan,
+            VariantShreddingWritePlan::Create(paimon_schema_, {{"v", shredding_type}}));
+        WriteShreddedFile(jsons, plan);
+    }
+
     static std::string NestedSiblingValue(size_t row) {
         return "t" + std::to_string(row);
     }
@@ -639,6 +655,106 @@ TEST_F(VariantParquetTest, ShreddedWriteAndReadRoundTrip) {
     }
 }
 
+TEST_F(VariantParquetTest, UntypedPhysicalVariantWriteAndReadRoundTrip) {
+    std::vector jsons = {
+        R"({"a": 1, "b": "hello"})",
+        nullptr,
+        "[1,2,3]",
+    };
+    WriteShreddedFile(jsons, arrow::null());
+
+    {
+        std::unique_ptr file_reader;
+        std::shared_ptr file_schema;
+        OpenFile(&file_reader, &file_schema);
+        auto file_variant_field = file_schema->GetFieldByName("v");
+        ASSERT_NE(file_variant_field, nullptr);
+        const auto& physical_type =
+            static_cast(*file_variant_field->type());
+        ASSERT_EQ(physical_type.num_fields(), 2);
+        ASSERT_EQ(physical_type.field(0)->name(), VariantDefs::kMetadataFieldName);
+        ASSERT_EQ(physical_type.field(1)->name(), VariantDefs::kValueFieldName);
+        ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(file_variant_field->type()));
+        ASSERT_TRUE(
+            VariantShreddingUtils::IsUntypedPhysicalVariantType(file_variant_field->type()));
+        file_reader->Close();
+    }
+
+    std::shared_ptr variant_column;
+    ReadVariantColumn(paimon_schema_, &variant_column);
+    ASSERT_EQ(variant_column->length(), static_cast(jsons.size()));
+    auto value_column = std::static_pointer_cast(variant_column->field(0));
+    auto metadata_column = std::static_pointer_cast(variant_column->field(1));
+    for (size_t i = 0; i < jsons.size(); ++i) {
+        SCOPED_TRACE("row " + std::to_string(i));
+        if (jsons[i] == nullptr) {
+            ASSERT_TRUE(variant_column->IsNull(i));
+            continue;
+        }
+        ASSERT_FALSE(variant_column->IsNull(i));
+        ASSERT_OK_AND_ASSIGN(
+            std::shared_ptr variant,
+            GenericVariant::Create(value_column->GetView(i), metadata_column->GetView(i), pool_));
+        ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson());
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr expected,
+                             GenericVariant::FromJson(jsons[i], pool_));
+        ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson());
+        ASSERT_EQ(actual_json, expected_json);
+    }
+}
+
+TEST_F(VariantParquetTest, AdaptiveInferenceUntypedPhysicalWriteAndReadRoundTrip) {
+    std::vector jsons = {
+        R"({"a": 1})",
+        "[1,2,3]",
+        nullptr,
+    };
+    auto logical = BuildArray(jsons);
+    ASSERT_OK_AND_ASSIGN(CoreOptions options,
+                         CoreOptions::FromMap({
+                             {Options::MANIFEST_FORMAT, "parquet"},
+                             {Options::VARIANT_INFER_SHREDDING_SCHEMA, "true"},
+                             {Options::VARIANT_SHREDDING_INFERENCE_MODE, "adaptive"},
+                         }));
+    auto factory = VariantShreddingWritePlanFactory::Create(options, paimon_schema_, pool_);
+    std::vector> samples = {logical};
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr converter,
+                         factory->CreateConverter("parquet", samples));
+    auto physical_variant = converter->GetPhysicalSchema()->GetFieldByName("v");
+    ASSERT_NE(physical_variant, nullptr);
+    ASSERT_FALSE(VariantShreddingUtils::IsShreddedFileType(physical_variant->type()));
+    ASSERT_TRUE(VariantShreddingUtils::IsUntypedPhysicalVariantType(physical_variant->type()));
+
+    auto c_logical = std::make_unique();
+    ASSERT_TRUE(arrow::ExportArray(*logical, c_logical.get()).ok());
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr c_physical,
+                         converter->Convert(c_logical.get()));
+    WriteFile(converter->GetPhysicalSchema(), c_physical.get());
+    ASSERT_OK(factory->OnFileCompleted(converter));
+
+    std::shared_ptr variant_column;
+    ReadVariantColumn(paimon_schema_, &variant_column);
+    ASSERT_EQ(variant_column->length(), static_cast(jsons.size()));
+    auto value_column = std::static_pointer_cast(variant_column->field(0));
+    auto metadata_column = std::static_pointer_cast(variant_column->field(1));
+    for (size_t i = 0; i < jsons.size(); ++i) {
+        SCOPED_TRACE("row " + std::to_string(i));
+        if (jsons[i] == nullptr) {
+            ASSERT_TRUE(variant_column->IsNull(i));
+            continue;
+        }
+        ASSERT_FALSE(variant_column->IsNull(i));
+        ASSERT_OK_AND_ASSIGN(
+            std::shared_ptr variant,
+            GenericVariant::Create(value_column->GetView(i), metadata_column->GetView(i), pool_));
+        ASSERT_OK_AND_ASSIGN(std::string actual_json, variant->ToJson());
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr expected,
+                             GenericVariant::FromJson(jsons[i], pool_));
+        ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson());
+        ASSERT_EQ(actual_json, expected_json);
+    }
+}
+
 TEST_F(VariantParquetTest, VariantAccessReadMixedTypedAndBinary) {
     std::vector jsons = {R"({"age": 35, "city": "Chicago"})",
                                       R"({"age": 25, "other": "Hello"})", nullptr};
diff --git a/test/inte/append_compaction_inte_test.cpp b/test/inte/append_compaction_inte_test.cpp
index 0b5158dc..b8a2c87a 100644
--- a/test/inte/append_compaction_inte_test.cpp
+++ b/test/inte/append_compaction_inte_test.cpp
@@ -254,7 +254,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompaction) {
 
 TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMapSharedShredding) {
     auto file_format = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -269,6 +269,7 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMap
 
     std::map options = {
         {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_ROW_NUM, "1"},
         {Options::BUCKET, "1"},
         {Options::BUCKET_KEY, "id"},
         {Options::FILE_SYSTEM, "local"},
@@ -324,7 +325,8 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMap
                          helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt));
     ASSERT_EQ(data_splits.size(), 1);
     {
-        // check adaptive k
+        // Compaction ignores target-file-row-num and creates one five-row output file. It also
+        // creates a fresh shared-shredding writer, so the file starts from K_max.
         auto data_split = std::dynamic_pointer_cast(data_splits[0]);
         ASSERT_TRUE(data_split);
         ASSERT_EQ(data_split->DataFiles().size(), 1);
@@ -342,11 +344,9 @@ TEST_P(AppendCompactionInteTest, TestAppendTableStreamWriteFullCompactionWithMap
         auto tags_field = file_schema->GetFieldByName("tags");
         ASSERT_TRUE(tags_field);
         ASSERT_TRUE(tags_field->metadata());
-        ASSERT_OK_AND_ASSIGN(
-            auto tags_meta,
-            MapSharedShreddingUtils::DeserializeMetadata(
-                tags_field->metadata()->Copy(), MapSharedShreddingDefine::kDefaultDictCompression));
-        ASSERT_EQ(4, tags_meta.num_columns);
+        ASSERT_OK_AND_ASSIGN(auto tags_meta, MapSharedShreddingUtils::DeserializeMetadata(
+                                                 tags_field->metadata()->Copy()));
+        ASSERT_EQ(64, tags_meta.num_columns);
         ASSERT_EQ(4, tags_meta.max_row_width);
     }
     {
diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp
index 6ff31288..85c73ef7 100644
--- a/test/inte/blob_table_inte_test.cpp
+++ b/test/inte/blob_table_inte_test.cpp
@@ -2303,7 +2303,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorMultiCommitAndShuffledReadSchema) {
 // The shared-shredding map is read from one main data file while the blob payload is read from a
 // separate blob file with the same row-id range.
 TEST_P(BlobTableInteTest, TestSharedShreddingWithBlobDataEvolution) {
-    if (GetParam() != "parquet" && GetParam() != "orc") {
+    if (GetParam() == "avro") {
         return;
     }
 
@@ -2362,7 +2362,7 @@ TEST_P(BlobTableInteTest, TestSharedShreddingWithBlobDataEvolution) {
 
 // Two independent shared-shredding map columns are written into different main files.
 TEST_P(BlobTableInteTest, TestMultipleSharedShreddingMapsWithBlobDataEvolution) {
-    if (GetParam() != "parquet" && GetParam() != "orc") {
+    if (GetParam() == "avro") {
         return;
     }
 
@@ -2424,7 +2424,7 @@ TEST_P(BlobTableInteTest, TestMultipleSharedShreddingMapsWithBlobDataEvolution)
 
 // A newer partial data file rewrites only the shared-shredding map for the same row-id range.
 TEST_P(BlobTableInteTest, TestSharedShreddingMapOverrideWithBlobDataEvolution) {
-    if (GetParam() != "parquet" && GetParam() != "orc") {
+    if (GetParam() == "avro") {
         return;
     }
 
@@ -2699,7 +2699,7 @@ TEST_P(BlobTableInteTest, TestBlobDescriptorFieldWriteRawBytesDirectly) {
 
 TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) {
     auto file_format = GetParam();
-    if (file_format != "orc" && file_format != "parquet") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2863,7 +2863,7 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamTable) {
 
 TEST_P(BlobTableInteTest, TestForwardBlobViewReference) {
     auto file_format = GetParam();
-    if (file_format != "orc" && file_format != "parquet") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -3147,7 +3147,7 @@ TEST_P(BlobTableInteTest, TestBlobViewFieldWithUpstreamDescriptorBlob) {
 
 TEST_P(BlobTableInteTest, TestBlobViewFieldWithMultipleUpstreamTables) {
     auto file_format = GetParam();
-    if (file_format != "orc" && file_format != "parquet") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -3470,7 +3470,7 @@ TEST_P(BlobTableInteTest, TestBlobViewWithFallbackPath) {
 
 TEST_P(BlobTableInteTest, TestReadBlobDescriptorFieldFromJava) {
     auto file_format = GetParam();
-    if (file_format != "orc" && file_format != "parquet") {
+    if (file_format == "avro") {
         return;
     }
     std::string table_path =
diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp
index 69276442..c47f8a3f 100644
--- a/test/inte/data_evolution_table_test.cpp
+++ b/test/inte/data_evolution_table_test.cpp
@@ -600,7 +600,7 @@ TEST_P(DataEvolutionTableTest, TestOnlySomeColumns) {
 }
 
 TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) {
-    if (FileFormat() != "parquet" && FileFormat() != "orc") {
+    if (FileFormat() == "avro") {
         return;
     }
 
diff --git a/test/inte/nested_column_pruning_inte_test.cpp b/test/inte/nested_column_pruning_inte_test.cpp
index 468e87ce..b77d6bda 100644
--- a/test/inte/nested_column_pruning_inte_test.cpp
+++ b/test/inte/nested_column_pruning_inte_test.cpp
@@ -1135,7 +1135,7 @@ TEST_P(NestedColumnPruningInteTest, MapSelectedKeysPreserveOrder) {
 }
 
 TEST_P(NestedColumnPruningInteTest, NestedStructMapSelectedKeysWithPredicate) {
-    if (file_format_ != "parquet" && file_format_ != "orc") {
+    if (file_format_ == "avro") {
         return;
     }
 
diff --git a/test/inte/pk_compaction_inte_test.cpp b/test/inte/pk_compaction_inte_test.cpp
index 0d7a5c2e..47e0e203 100644
--- a/test/inte/pk_compaction_inte_test.cpp
+++ b/test/inte/pk_compaction_inte_test.cpp
@@ -357,7 +357,7 @@ class PkCompactionInteTest : public ::testing::Test,
 // Verify shared-shredding MAP can be read correctly after PK full compaction.
 TEST_P(PkCompactionInteTest, TestKeyValueTableFullCompactionWithMapSharedShredding) {
     auto file_format = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -429,7 +429,7 @@ TEST_P(PkCompactionInteTest, TestKeyValueTableFullCompactionWithMapSharedShreddi
 
 TEST_P(PkCompactionInteTest, TestKeyValueTableDvCompactionWithMapSharedShredding) {
     auto file_format = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
diff --git a/test/inte/variant_table_inte_test.cpp b/test/inte/variant_table_inte_test.cpp
index ac9d157c..3cedac21 100644
--- a/test/inte/variant_table_inte_test.cpp
+++ b/test/inte/variant_table_inte_test.cpp
@@ -17,6 +17,8 @@
  * under the License.
  */
 
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -28,12 +30,20 @@
 #include "fmt/format.h"
 #include "gtest/gtest.h"
 #include "paimon/common/data/variant/generic_variant.h"
+#include "paimon/common/data/variant/variant_shredding_utils.h"
 #include "paimon/common/data/variant/variant_type_utils.h"
 #include "paimon/common/factories/io_hook.h"
+#include "paimon/common/utils/path_util.h"
 #include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/table/source/data_split_impl.h"
 #include "paimon/data/variant.h"
 #include "paimon/defs.h"
+#include "paimon/format/file_format_factory.h"
+#include "paimon/format/reader_builder.h"
+#include "paimon/fs/file_system.h"
 #include "paimon/memory/memory_pool.h"
+#include "paimon/reader/file_batch_reader.h"
 #include "paimon/record_batch.h"
 #include "paimon/table/source/startup_mode.h"
 #include "paimon/testing/utils/io_exception_helper.h"
@@ -144,6 +154,49 @@ class VariantTableInteTest : public ::testing::Test {
         *result_struct = std::static_pointer_cast(result->chunk(0));
     }
 
+    Result> ReadDataFileSchema(
+        const std::string& bucket_path, const std::shared_ptr& file,
+        const std::map& options) const {
+        std::string file_path = PathUtil::JoinPath(bucket_path, file->file_name);
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr unique_input_stream,
+                               dir_->GetFileSystem()->Open(file_path));
+        std::shared_ptr input_stream(std::move(unique_input_stream));
+        PAIMON_ASSIGN_OR_RAISE(std::string format, file->FileFormat());
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_format,
+                               FileFormatFactory::Get(format, options));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder,
+                               file_format->CreateReaderBuilder(/*batch_size=*/10));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader,
+                               reader_builder->Build(input_stream));
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr c_file_schema, reader->GetFileSchema());
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema,
+                                          arrow::ImportSchema(c_file_schema.get()));
+        return file_schema;
+    }
+
+    std::map AdaptiveInferenceOptions(int64_t target_file_row_num,
+                                                                int32_t initial_sample_rows,
+                                                                int32_t adaptive_sample_rows,
+                                                                double admission_ratio,
+                                                                double retention_ratio) const {
+        return {
+            {Options::MANIFEST_FORMAT, "avro"},
+            {Options::FILE_FORMAT, "parquet"},
+            {Options::BUCKET, "-1"},
+            {Options::WRITE_ONLY, "true"},
+            {Options::TARGET_FILE_ROW_NUM, std::to_string(target_file_row_num)},
+            {Options::VARIANT_INFER_SHREDDING_SCHEMA, "true"},
+            {Options::VARIANT_SHREDDING_INFERENCE_MODE, "adaptive"},
+            {Options::VARIANT_SHREDDING_MAX_INFER_BUFFER_ROW, std::to_string(initial_sample_rows)},
+            {Options::VARIANT_SHREDDING_ADAPTIVE_MAX_INFER_BUFFER_ROW,
+             std::to_string(adaptive_sample_rows)},
+            {Options::VARIANT_SHREDDING_MIN_FIELD_CARDINALITY_RATIO,
+             fmt::format("{}", admission_ratio)},
+            {Options::VARIANT_SHREDDING_ADAPTIVE_RETENTION_RATIO,
+             fmt::format("{}", retention_ratio)},
+        };
+    }
+
  protected:
     std::string test_dir_;
     std::unique_ptr dir_;
@@ -201,6 +254,348 @@ TEST_F(VariantTableInteTest, TestAppendTable) {
     ReadAndCheck(helper.get(), splits, {0, 1, 2, 3, 4, 5, 6}, jsons);
 }
 
+TEST_F(VariantTableInteTest, TestAdaptiveInferenceAcrossRollingFiles) {
+    std::map options =
+        AdaptiveInferenceOptions(/*target_file_row_num=*/10, /*initial_sample_rows=*/10,
+                                 /*adaptive_sample_rows=*/10, /*admission_ratio=*/0.4,
+                                 /*retention_ratio=*/0.2);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr helper,
+        TestHelper::Create(test_dir_, schema_, /*partition_keys=*/{}, /*primary_keys=*/{}, options,
+                           /*is_streaming_mode=*/false));
+
+    std::vector json_storage;
+    json_storage.reserve(30);
+    for (int32_t id = 0; id < 10; ++id) {
+        json_storage.push_back(id < 5 ? fmt::format(R"({{"legacy":"value","stable":{}}})", id)
+                                      : fmt::format(R"({{"stable":{}}})", id));
+    }
+    for (int32_t id = 10; id < 20; ++id) {
+        json_storage.push_back(id < 19 ? fmt::format(R"({{"emerging":true,"stable":{}}})", id)
+                                       : fmt::format(R"({{"stable":{}}})", id));
+    }
+    for (int32_t id = 20; id < 30; ++id) {
+        json_storage.push_back(fmt::format(R"({{"stable":{}}})", id));
+    }
+    std::vector jsons;
+    jsons.reserve(json_storage.size());
+    for (const std::string& json : json_storage) {
+        jsons.push_back(json.c_str());
+    }
+
+    std::vector> batches;
+    for (int32_t file_index = 0; file_index < 3; ++file_index) {
+        auto begin = jsons.begin() + file_index * 10;
+        std::vector file_jsons(begin, begin + 10);
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr batch,
+                             MakeBatch(BuildArray(file_jsons, /*id_offset=*/file_index * 10)));
+        batches.push_back(std::move(batch));
+    }
+    ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector> splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt,
+                                         /*is_streaming=*/false));
+    ASSERT_EQ(1, splits.size());
+    auto data_split = std::dynamic_pointer_cast(splits[0]);
+    ASSERT_NE(data_split, nullptr);
+    std::vector> files = data_split->DataFiles();
+    std::sort(
+        files.begin(), files.end(),
+        [](const std::shared_ptr& left, const std::shared_ptr& right) {
+            return left->min_sequence_number < right->min_sequence_number;
+        });
+    ASSERT_EQ(3, files.size());
+
+    std::vector> expected_variant_types;
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr first_type,
+        VariantShreddingUtils::VariantShreddingSchema(arrow::struct_(
+            {arrow::field("legacy", arrow::utf8()), arrow::field("stable", arrow::int64())})));
+    expected_variant_types.push_back(first_type);
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr second_type,
+        VariantShreddingUtils::VariantShreddingSchema(arrow::struct_(
+            {arrow::field("emerging", arrow::boolean()), arrow::field("legacy", arrow::utf8()),
+             arrow::field("stable", arrow::int64())})));
+    expected_variant_types.push_back(second_type);
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr third_type,
+        VariantShreddingUtils::VariantShreddingSchema(arrow::struct_(
+            {arrow::field("emerging", arrow::boolean()), arrow::field("stable", arrow::int64())})));
+    expected_variant_types.push_back(third_type);
+
+    for (size_t i = 0; i < files.size(); ++i) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr file_schema,
+                             ReadDataFileSchema(data_split->BucketPath(), files[i], options));
+        std::shared_ptr variant_field = file_schema->GetFieldByName("v");
+        ASSERT_NE(variant_field, nullptr);
+        ASSERT_TRUE(variant_field->type()->Equals(*expected_variant_types[i]))
+            << "file=" << files[i]->file_name << ", actual=" << variant_field->type()->ToString()
+            << ", expected=" << expected_variant_types[i]->ToString();
+    }
+
+    std::vector expected_ids;
+    expected_ids.reserve(30);
+    for (int32_t id = 0; id < 30; ++id) {
+        expected_ids.push_back(id);
+    }
+    ReadAndCheck(helper.get(), splits, expected_ids, jsons);
+}
+
+TEST_F(VariantTableInteTest, TestAdaptiveInferenceWithMultipleVariantFields) {
+    std::shared_ptr left_field = VariantTypeUtils::ToArrowField("left_payload");
+    std::shared_ptr right_field = VariantTypeUtils::ToArrowField("right_payload");
+    std::shared_ptr table_schema =
+        arrow::schema({arrow::field("id", arrow::int32()), left_field, right_field});
+    std::map options =
+        AdaptiveInferenceOptions(/*target_file_row_num=*/2, /*initial_sample_rows=*/2,
+                                 /*adaptive_sample_rows=*/2, /*admission_ratio=*/0.4,
+                                 /*retention_ratio=*/0.2);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr helper,
+        TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{},
+                           /*primary_keys=*/{}, options, /*is_streaming_mode=*/false));
+
+    std::vector left_jsons = {
+        R"({"legacy":"a","stable":1})",
+        R"({"legacy":"b","stable":2})",
+        R"({"emerging":true,"stable":3})",
+        R"({"emerging":false,"stable":4})",
+    };
+    std::vector right_jsons = {
+        R"({"sparse":"x","stable":"a"})",
+        R"({"stable":"b"})",
+        R"({"stable":"c"})",
+        R"({"emerging":true,"stable":"d"})",
+    };
+    std::vector> batches;
+    for (int32_t file_index = 0; file_index < 2; ++file_index) {
+        auto left_begin = left_jsons.begin() + file_index * 2;
+        std::vector file_left_jsons(left_begin, left_begin + 2);
+        auto right_begin = right_jsons.begin() + file_index * 2;
+        std::vector file_right_jsons(right_begin, right_begin + 2);
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr left,
+                             VariantTestData::BuildVariantBatch(table_schema->field(0), left_field,
+                                                                file_left_jsons, pool_,
+                                                                /*id_offset=*/file_index * 2 + 1));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr right,
+                             VariantTestData::BuildVariantBatch(table_schema->field(0), right_field,
+                                                                file_right_jsons, pool_,
+                                                                /*id_offset=*/file_index * 2 + 1));
+        std::shared_ptr rows =
+            arrow::StructArray::Make({left->field(0), left->field(1), right->field(1)},
+                                     table_schema->fields())
+                .ValueOrDie();
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(rows));
+        batches.push_back(std::move(batch));
+    }
+    ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector> splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt,
+                                         /*is_streaming=*/false));
+    ASSERT_EQ(1, splits.size());
+    std::shared_ptr data_split = std::dynamic_pointer_cast(splits[0]);
+    ASSERT_NE(data_split, nullptr);
+    std::vector> files = data_split->DataFiles();
+    std::sort(
+        files.begin(), files.end(),
+        [](const std::shared_ptr& left, const std::shared_ptr& right) {
+            return left->min_sequence_number < right->min_sequence_number;
+        });
+    ASSERT_EQ(2, files.size());
+
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr first_left_type,
+        VariantShreddingUtils::VariantShreddingSchema(arrow::struct_(
+            {arrow::field("legacy", arrow::utf8()), arrow::field("stable", arrow::int64())})));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr second_left_type,
+        VariantShreddingUtils::VariantShreddingSchema(arrow::struct_(
+            {arrow::field("emerging", arrow::boolean()), arrow::field("legacy", arrow::utf8()),
+             arrow::field("stable", arrow::int64())})));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr right_type,
+        VariantShreddingUtils::VariantShreddingSchema(arrow::struct_(
+            {arrow::field("sparse", arrow::utf8()), arrow::field("stable", arrow::utf8())})));
+    std::vector> expected_left_types = {first_left_type,
+                                                                         second_left_type};
+    for (size_t i = 0; i < files.size(); ++i) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr file_schema,
+                             ReadDataFileSchema(data_split->BucketPath(), files[i], options));
+        std::shared_ptr file_left = file_schema->GetFieldByName("left_payload");
+        std::shared_ptr file_right = file_schema->GetFieldByName("right_payload");
+        ASSERT_NE(file_left, nullptr);
+        ASSERT_NE(file_right, nullptr);
+        ASSERT_TRUE(file_left->type()->Equals(*expected_left_types[i]))
+            << "file=" << files[i]->file_name << ", actual=" << file_left->type()->ToString()
+            << ", expected=" << expected_left_types[i]->ToString();
+        ASSERT_TRUE(file_right->type()->Equals(*right_type))
+            << "file=" << files[i]->file_name << ", actual=" << file_right->type()->ToString()
+            << ", expected=" << right_type->ToString();
+    }
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr result, helper->ReadResult(splits));
+    std::map actual;
+    for (const std::shared_ptr& chunk : result->chunks()) {
+        auto rows = std::static_pointer_cast(chunk);
+        auto row_type = std::static_pointer_cast(rows->type());
+        auto ids =
+            std::static_pointer_cast(rows->field(row_type->GetFieldIndex("id")));
+        auto left = std::static_pointer_cast(
+            rows->field(row_type->GetFieldIndex("left_payload")));
+        auto right = std::static_pointer_cast(
+            rows->field(row_type->GetFieldIndex("right_payload")));
+        auto left_values = std::static_pointer_cast(left->field(0));
+        auto left_metadata = std::static_pointer_cast(left->field(1));
+        auto right_values = std::static_pointer_cast(right->field(0));
+        auto right_metadata = std::static_pointer_cast(right->field(1));
+        for (int64_t i = 0; i < rows->length(); ++i) {
+            ASSERT_OK_AND_ASSIGN(
+                std::shared_ptr left_variant,
+                GenericVariant::Create(left_values->GetView(i), left_metadata->GetView(i), pool_));
+            ASSERT_OK_AND_ASSIGN(std::shared_ptr right_variant,
+                                 GenericVariant::Create(right_values->GetView(i),
+                                                        right_metadata->GetView(i), pool_));
+            ASSERT_OK_AND_ASSIGN(std::string left_json, left_variant->ToJson());
+            ASSERT_OK_AND_ASSIGN(std::string right_json, right_variant->ToJson());
+            actual.emplace(ids->Value(i), left_json + "|" + right_json);
+        }
+    }
+    ASSERT_EQ(4, actual.size());
+    for (int32_t id = 1; id <= 4; ++id) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr expected_left,
+                             GenericVariant::FromJson(left_jsons[id - 1], pool_));
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr expected_right,
+                             GenericVariant::FromJson(right_jsons[id - 1], pool_));
+        ASSERT_OK_AND_ASSIGN(std::string expected_left_json, expected_left->ToJson());
+        ASSERT_OK_AND_ASSIGN(std::string expected_right_json, expected_right->ToJson());
+        ASSERT_EQ(expected_left_json + "|" + expected_right_json, actual[id]);
+    }
+}
+
+TEST_F(VariantTableInteTest, TestAdaptiveInferenceWithNestedVariant) {
+    std::shared_ptr nested_variant = VariantTypeUtils::ToArrowField("payload");
+    std::shared_ptr nested_field = arrow::field(
+        "nested", arrow::struct_({arrow::field("label", arrow::utf8()), nested_variant}));
+    std::shared_ptr table_schema =
+        arrow::schema({arrow::field("id", arrow::int32()), nested_field});
+    std::map options =
+        AdaptiveInferenceOptions(/*target_file_row_num=*/2, /*initial_sample_rows=*/2,
+                                 /*adaptive_sample_rows=*/2, /*admission_ratio=*/0.4,
+                                 /*retention_ratio=*/0.2);
+    ASSERT_OK_AND_ASSIGN(
+        std::unique_ptr helper,
+        TestHelper::Create(test_dir_, table_schema, /*partition_keys=*/{},
+                           /*primary_keys=*/{}, options, /*is_streaming_mode=*/false));
+
+    std::vector labels = {"first", "second", "third", "fourth"};
+    std::vector jsons = {
+        R"({"legacy":"a","stable":1})",
+        R"({"legacy":"b","stable":2})",
+        R"({"emerging":true,"stable":3})",
+        R"({"emerging":false,"stable":4})",
+    };
+    std::vector> batches;
+    for (int32_t file_index = 0; file_index < 2; ++file_index) {
+        auto json_begin = jsons.begin() + file_index * 2;
+        std::vector file_jsons(json_begin, json_begin + 2);
+        ASSERT_OK_AND_ASSIGN(
+            std::shared_ptr variants,
+            VariantTestData::BuildVariantBatch(table_schema->field(0), nested_variant, file_jsons,
+                                               pool_, /*id_offset=*/file_index * 2 + 1));
+        arrow::StringBuilder label_builder;
+        for (int32_t i = file_index * 2; i < file_index * 2 + 2; ++i) {
+            arrow::Status status = label_builder.Append(labels[i]);
+            ASSERT_TRUE(status.ok()) << status.ToString();
+        }
+        std::shared_ptr label_array = label_builder.Finish().ValueOrDie();
+        std::shared_ptr nested =
+            arrow::StructArray::Make({label_array, variants->field(1)},
+                                     nested_field->type()->fields())
+                .ValueOrDie();
+        std::shared_ptr rows =
+            arrow::StructArray::Make({variants->field(0), nested}, table_schema->fields())
+                .ValueOrDie();
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr batch, MakeBatch(rows));
+        batches.push_back(std::move(batch));
+    }
+    ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(std::vector> splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt,
+                                         /*is_streaming=*/false));
+    ASSERT_EQ(1, splits.size());
+    std::shared_ptr data_split = std::dynamic_pointer_cast(splits[0]);
+    ASSERT_NE(data_split, nullptr);
+    std::vector> files = data_split->DataFiles();
+    std::sort(
+        files.begin(), files.end(),
+        [](const std::shared_ptr& left, const std::shared_ptr& right) {
+            return left->min_sequence_number < right->min_sequence_number;
+        });
+    ASSERT_EQ(2, files.size());
+
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr first_type,
+        VariantShreddingUtils::VariantShreddingSchema(arrow::struct_(
+            {arrow::field("legacy", arrow::utf8()), arrow::field("stable", arrow::int64())})));
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr second_type,
+        VariantShreddingUtils::VariantShreddingSchema(arrow::struct_(
+            {arrow::field("emerging", arrow::boolean()), arrow::field("legacy", arrow::utf8()),
+             arrow::field("stable", arrow::int64())})));
+    std::vector> expected_types = {first_type, second_type};
+    for (size_t i = 0; i < files.size(); ++i) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr file_schema,
+                             ReadDataFileSchema(data_split->BucketPath(), files[i], options));
+        std::shared_ptr file_nested = file_schema->GetFieldByName("nested");
+        ASSERT_NE(file_nested, nullptr);
+        auto file_nested_type = std::static_pointer_cast(file_nested->type());
+        std::shared_ptr file_variant = file_nested_type->GetFieldByName("payload");
+        ASSERT_NE(file_variant, nullptr);
+        ASSERT_TRUE(file_variant->type()->Equals(*expected_types[i]))
+            << "file=" << files[i]->file_name << ", actual=" << file_variant->type()->ToString()
+            << ", expected=" << expected_types[i]->ToString();
+    }
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr result, helper->ReadResult(splits));
+    std::map actual;
+    for (const std::shared_ptr& chunk : result->chunks()) {
+        auto rows = std::static_pointer_cast(chunk);
+        auto row_type = std::static_pointer_cast(rows->type());
+        auto ids =
+            std::static_pointer_cast(rows->field(row_type->GetFieldIndex("id")));
+        auto nested = std::static_pointer_cast(
+            rows->field(row_type->GetFieldIndex("nested")));
+        auto nested_type = std::static_pointer_cast(nested->type());
+        auto label_column = std::static_pointer_cast(
+            nested->field(nested_type->GetFieldIndex("label")));
+        auto variant = std::static_pointer_cast(
+            nested->field(nested_type->GetFieldIndex("payload")));
+        auto value_column = std::static_pointer_cast(variant->field(0));
+        auto metadata_column = std::static_pointer_cast(variant->field(1));
+        for (int64_t i = 0; i < rows->length(); ++i) {
+            ASSERT_OK_AND_ASSIGN(std::shared_ptr value,
+                                 GenericVariant::Create(value_column->GetView(i),
+                                                        metadata_column->GetView(i), pool_));
+            ASSERT_OK_AND_ASSIGN(std::string json, value->ToJson());
+            actual.emplace(ids->Value(i), label_column->GetString(i) + "|" + json);
+        }
+    }
+    ASSERT_EQ(4, actual.size());
+    for (int32_t id = 1; id <= 4; ++id) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr expected,
+                             GenericVariant::FromJson(jsons[id - 1], pool_));
+        ASSERT_OK_AND_ASSIGN(std::string expected_json, expected->ToJson());
+        ASSERT_EQ(std::string(labels[id - 1]) + "|" + expected_json, actual[id]);
+    }
+}
+
 TEST_F(VariantTableInteTest, TestPrimaryKeyTable) {
     std::map options = {
         {Options::MANIFEST_FORMAT, "avro"},
diff --git a/test/inte/write_and_read_inte_test.cpp b/test/inte/write_and_read_inte_test.cpp
index 04638892..a18fe9d2 100644
--- a/test/inte/write_and_read_inte_test.cpp
+++ b/test/inte/write_and_read_inte_test.cpp
@@ -60,6 +60,7 @@
 #include "paimon/testing/utils/read_result_collector.h"
 #include "paimon/testing/utils/test_helper.h"
 #include "paimon/testing/utils/testharness.h"
+#include "paimon/testing/utils/timezone_guard.h"
 #include "paimon/write_context.h"
 #include "rapidjson/document.h"
 #include "rapidjson/stringbuffer.h"
@@ -245,8 +246,7 @@ class WriteAndReadInteTest
             return Status::Invalid(
                 fmt::format("field {} has no shared-shredding metadata", field_name));
         }
-        return MapSharedShreddingUtils::DeserializeMetadata(
-            metadata_copy, MapSharedShreddingDefine::kDefaultDictCompression);
+        return MapSharedShreddingUtils::DeserializeMetadata(metadata_copy);
     }
 
  private:
@@ -1125,7 +1125,7 @@ TEST_P(WriteAndReadInteTest, TestWriteSamePartitionTwiceWithAllBasicTypesForPk)
 
 TEST_P(WriteAndReadInteTest, TestCharVarcharBinaryVarbinaryTypes) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
     arrow::FieldVector fields = {
@@ -1511,7 +1511,7 @@ TEST_P(WriteAndReadInteTest, TestAppendWithParquetMetadataCache) {
 
 TEST_P(WriteAndReadInteTest, TestAppendSharedShreddingMap) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -1566,9 +1566,93 @@ TEST_P(WriteAndReadInteTest, TestAppendSharedShreddingMap) {
     ASSERT_TRUE(success);
 }
 
+TEST_P(WriteAndReadInteTest, TestMapSharedShreddingColumnPlacementPolicies) {
+    auto [file_format, file_system] = GetParam();
+    if (file_format == "avro") {
+        return;
+    }
+
+    auto map_type = arrow::map(arrow::utf8(), arrow::int64());
+    arrow::FieldVector fields = {
+        arrow::field("id", arrow::int32()),
+        arrow::field("plain_metrics", map_type),
+        arrow::field("sequential_metrics", map_type),
+        arrow::field("lru_metrics", map_type),
+    };
+    std::map options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"},
+        {Options::BUCKET, "-1"},
+        {Options::FILE_SYSTEM, file_system},
+        {"fields.plain_metrics.map.storage-layout", "shared-shredding"},
+        {"fields.plain_metrics.map.shared-shredding.max-columns", "3"},
+        {"fields.plain_metrics.map.shared-shredding.column-placement-policy", "plain"},
+        {"fields.sequential_metrics.map.storage-layout", "shared-shredding"},
+        {"fields.sequential_metrics.map.shared-shredding.max-columns", "3"},
+        {"fields.sequential_metrics.map.shared-shredding.column-placement-policy", "sequential"},
+        {"fields.lru_metrics.map.storage-layout", "shared-shredding"},
+        {"fields.lru_metrics.map.shared-shredding.max-columns", "3"},
+        {"fields.lru_metrics.map.shared-shredding.column-placement-policy", "lru"},
+    };
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(
+        auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{},
+                                        /*primary_keys=*/{}, options, /*is_streaming_mode=*/false));
+
+    ASSERT_OK_AND_ASSIGN(auto batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields),
+                                                     R"([
+                [1, [["a", 10], ["b", 20], ["c", 30]], [["a", 10], ["c", 30], ["d", 60]], [["a", 10], ["b", 20], ["c", 30]]],
+                [2, [["a", 40], ["b", 50]], [["a", 40], ["c", 50]], [["a", 40], ["b", 50]]],
+                [3, [["d", 60]], [["b", 60]], [["d", 60]]],
+                [4, [["a", 70], ["b", 80], ["c", 90], ["d", 100]], [["a", 70], ["b", 80], ["c", 90], ["d", 100]], [["a", 70], ["b", 80], ["c", 90], ["d", 100]]]
+            ])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options));
+    ASSERT_EQ(1, files.size());
+    auto check_meta = [&](const std::string& field_name,
+                          const std::string& overflow_name) -> Status {
+        PAIMON_ASSIGN_OR_RAISE(MapSharedShreddingFieldMeta meta,
+                               ReadShreddingMeta(files[0], field_name, options));
+        if (meta.name_to_id.size() != 4 || meta.num_columns != 3 || meta.max_row_width != 4 ||
+            meta.overflow_field_set.size() != 1) {
+            return Status::Invalid("unexpected shared-shredding metadata for ", field_name);
+        }
+        auto overflow_id = meta.name_to_id.find(overflow_name);
+        if (overflow_id == meta.name_to_id.end() ||
+            meta.overflow_field_set.count(overflow_id->second) != 1) {
+            return Status::Invalid("unexpected overflow field for ", field_name);
+        }
+        return Status::OK();
+    };
+    ASSERT_OK(check_meta("plain_metrics", "d"));
+    ASSERT_OK(check_meta("sequential_metrics", "b"));
+    ASSERT_OK(check_meta("lru_metrics", "c"));
+
+    arrow::FieldVector expected_fields = fields;
+    expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8()));
+    ASSERT_OK_AND_ASSIGN(auto splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(bool success,
+                         helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits,
+                                                    R"([
+                [0, 1, [["a", 10], ["b", 20], ["c", 30]], [["a", 10], ["c", 30], ["d", 60]], [["a", 10], ["b", 20], ["c", 30]]],
+                [0, 2, [["a", 40], ["b", 50]], [["a", 40], ["c", 50]], [["a", 40], ["b", 50]]],
+                [0, 3, [["d", 60]], [["b", 60]], [["d", 60]]],
+                [0, 4, [["a", 70], ["b", 80], ["c", 90], ["d", 100]], [["a", 70], ["b", 80], ["c", 90], ["d", 100]], [["a", 70], ["b", 80], ["c", 90], ["d", 100]]]
+            ])"));
+    ASSERT_TRUE(success);
+}
+
 TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPartitionAndBucket) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -1665,7 +1749,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPartitionAndBucket)
 
     ASSERT_OK_AND_ASSIGN(auto p1_second_meta,
                          ReadShreddingMeta(p1_bucket0_files[1], "tags", options));
-    ASSERT_EQ(1, p1_second_meta.num_columns);
+    ASSERT_EQ(5, p1_second_meta.num_columns);
     ASSERT_EQ(1, p1_second_meta.max_row_width);
 
     ASSERT_OK_AND_ASSIGN(auto p2_first_meta,
@@ -1676,7 +1760,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPartitionAndBucket)
 
 TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -1768,9 +1852,9 @@ TEST_P(WriteAndReadInteTest, TestAppendMapSharedShreddingWithPredicate) {
     ASSERT_TRUE(expected->Equals(actual)) << actual->ToString();
 }
 
-TEST_P(WriteAndReadInteTest, TestMapSharedShreddingRestoreAdaptiveColumnCountFromFileMetadata) {
+TEST_P(WriteAndReadInteTest, TestMapSharedShreddingNewWriterStartsWithMaxColumnCount) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -1819,7 +1903,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingRestoreAdaptiveColumnCountFro
     ASSERT_EQ(8, first_meta.num_columns);
     ASSERT_EQ(1, first_meta.max_row_width);
     ASSERT_OK_AND_ASSIGN(auto second_meta, ReadShreddingMeta(files[1], "metrics", options));
-    ASSERT_EQ(1, second_meta.num_columns);
+    ASSERT_EQ(8, second_meta.num_columns);
     ASSERT_EQ(1, second_meta.max_row_width);
 
     auto expected_type = arrow::struct_({
@@ -1837,9 +1921,82 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingRestoreAdaptiveColumnCountFro
     ASSERT_TRUE(success);
 }
 
+TEST_P(WriteAndReadInteTest, TestMapSharedShreddingAdaptsAcrossRollingFiles) {
+    auto [file_format, file_system] = GetParam();
+    if (file_format == "avro") {
+        return;
+    }
+
+    auto map_type = arrow::map(arrow::utf8(), arrow::int64());
+    arrow::FieldVector fields = {
+        arrow::field("id", arrow::int32()),
+        arrow::field("metrics", map_type),
+    };
+    std::map options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1048576"},
+        {Options::TARGET_FILE_ROW_NUM, "2"},
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "id"},
+        {Options::FILE_SYSTEM, file_system},
+        {Options::WRITE_ONLY, "true"},
+        {"fields.metrics.map.storage-layout", "shared-shredding"},
+        {"fields.metrics.map.shared-shredding.max-columns", "8"},
+    };
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(
+        auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{},
+                                        /*primary_keys=*/{}, options, /*is_streaming_mode=*/false));
+
+    std::vector> batches;
+    ASSERT_OK_AND_ASSIGN(
+        auto first_batch,
+        TestHelper::MakeRecordBatch(
+            arrow::struct_(fields),
+            R"([[1, [["a", 11], ["b", 12]]], [2, [["c", 21], ["d", 22], ["e", 23]]]])",
+            /*partition_map=*/{}, /*bucket=*/0, {}));
+    batches.push_back(std::move(first_batch));
+    ASSERT_OK_AND_ASSIGN(
+        auto second_batch,
+        TestHelper::MakeRecordBatch(arrow::struct_(fields),
+                                    R"([[3, [["f", 31]]], [4, [["g", 41], ["h", 42]]]])",
+                                    /*partition_map=*/{}, /*bucket=*/0, {}));
+    batches.push_back(std::move(second_batch));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batches), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options));
+    ASSERT_EQ(2, files.size());
+    ASSERT_OK_AND_ASSIGN(auto first_meta, ReadShreddingMeta(files[0], "metrics", options));
+    ASSERT_EQ(8, first_meta.num_columns);
+    ASSERT_EQ(3, first_meta.max_row_width);
+    ASSERT_OK_AND_ASSIGN(auto second_meta, ReadShreddingMeta(files[1], "metrics", options));
+    ASSERT_EQ(3, second_meta.num_columns);
+    ASSERT_EQ(2, second_meta.max_row_width);
+
+    auto expected_type = arrow::struct_({
+        arrow::field("_VALUE_KIND", arrow::int8()),
+        arrow::field("id", arrow::int32()),
+        arrow::field("metrics", map_type),
+    });
+    ASSERT_OK_AND_ASSIGN(auto splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(bool success, helper->ReadAndCheckResult(expected_type, splits,
+                                                                  R"([
+        [0, 1, [["a", 11], ["b", 12]]],
+        [0, 2, [["c", 21], ["d", 22], ["e", 23]]],
+        [0, 3, [["f", 31]]],
+        [0, 4, [["g", 41], ["h", 42]]]
+    ])"));
+    ASSERT_TRUE(success);
+}
+
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingSwitchMapLayoutAndUseMaxColumnsWithoutMetadata) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -1922,7 +2079,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingSwitchMapLayoutAndUseMaxColum
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingReadAfterRenameColumn) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -1991,7 +2148,7 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingReadAfterRenameColumn) {
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingWithSchemaEvolution) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2081,7 +2238,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingWithSchemaEvolution) {
 // Verify storage-layout evolution: default->shared-shredding.
 TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShredding) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2142,7 +2299,7 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShredding) {
 // Verify storage-layout evolution: shared-shredding->default.
 TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefault) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2206,7 +2363,7 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefault) {
 
 TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultCompaction) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2310,7 +2467,7 @@ TEST_P(WriteAndReadInteTest, TestAppendMapStorageLayoutSharedShreddingToDefaultC
 // Nested map values through both selected physical columns and overflow.
 TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2365,7 +2522,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingWithStructValue) {
 
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithComplexValue) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2457,9 +2614,137 @@ TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithComplexValue) {
     ASSERT_TRUE(selected_success);
 }
 
+TEST_P(WriteAndReadInteTest, TestMapSharedShreddingWithAllSupportedComplexValueTypes) {
+    auto [file_format, file_system] = GetParam();
+    if (file_format == "avro") {
+        return;
+    }
+
+    TimezoneGuard timezone_guard("Asia/Shanghai");
+    // Nested TIMESTAMP_LTZ needs a timezone projection inside MAP. The Parquet reader currently
+    // rejects that projection, so only ORC exercises LTZ here; Parquet still covers both timestamp
+    // precisions as timezone-free timestamps.
+    std::string timezone =
+        file_format == "orc" ? DateTimeUtils::GetLocalTimezoneName() : std::string();
+    auto value_type = arrow::struct_({
+        arrow::field("bool", arrow::boolean()),
+        arrow::field("tiny", arrow::int8()),
+        arrow::field("small", arrow::int16()),
+        arrow::field("i", arrow::int32()),
+        arrow::field("big", arrow::int64()),
+        arrow::field("f", arrow::float32()),
+        arrow::field("d", arrow::float64()),
+        arrow::field("s", arrow::utf8()),
+        arrow::field("bin", arrow::binary()),
+        arrow::field("compact_decimal", arrow::decimal128(10, 2)),
+        arrow::field("large_decimal", arrow::decimal128(23, 5)),
+        arrow::field("date", arrow::date32()),
+        arrow::field("ts3", arrow::timestamp(arrow::TimeUnit::MILLI)),
+        arrow::field("ts9", arrow::timestamp(arrow::TimeUnit::NANO)),
+        arrow::field("ts_ltz3", arrow::timestamp(arrow::TimeUnit::MILLI, timezone)),
+        arrow::field("ts_ltz6", arrow::timestamp(arrow::TimeUnit::MICRO, timezone)),
+        arrow::field("ints", arrow::list(arrow::int32())),
+        arrow::field("attrs", arrow::map(arrow::utf8(), arrow::int64())),
+        arrow::field("nested", arrow::struct_({
+                                   arrow::field("name", arrow::utf8()),
+                                   arrow::field("score", arrow::int32()),
+                               })),
+    });
+    auto map_type = arrow::map(arrow::utf8(), value_type);
+    arrow::FieldVector fields = {
+        arrow::field("id", arrow::int32()),
+        arrow::field("metrics", map_type),
+    };
+    std::map options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::TARGET_FILE_SIZE, "1024"},
+        {Options::BUCKET, "-1"},
+        {Options::FILE_SYSTEM, file_system},
+        {Options::WRITE_ONLY, "true"},
+        {"orc.timestamp-ltz.legacy.type", "false"},
+        {"fields.metrics.map.storage-layout", "shared-shredding"},
+        {"fields.metrics.map.shared-shredding.max-columns", "2"},
+    };
+    if (file_system == "jindo") {
+        options = AddOptionsForJindo(options);
+    }
+    ASSERT_OK_AND_ASSIGN(
+        auto helper, TestHelper::Create(test_dir_, arrow::schema(fields), /*partition_keys=*/{},
+                                        /*primary_keys=*/{}, options, /*is_streaming_mode=*/false));
+
+    ASSERT_OK_AND_ASSIGN(auto batch,
+                         TestHelper::MakeRecordBatch(arrow::struct_(fields),
+                                                     R"([
+                [1, [
+                    ["fixed-a", [
+                        true, 1, 2, 3, 4, 5.5, 6.25, "str", "bin",
+                        "12345678.90", "123456789012345678.12345", 19500,
+                        "2023-11-14 22:13:20.123", "2023-11-14 22:13:20.123456789",
+                        "2023-11-14 22:13:20.123", "2023-11-14 22:13:20.123456",
+                        [7, null, 8], [["m1", 10], ["m2", null]], ["nested", 99]
+                    ]],
+                    ["fixed-b", [
+                        null, null, null, null, null, null, null, null, null,
+                        null, null, null, null, null, null, null, null, null, null
+                    ]],
+                    ["overflow", [
+                        false, 9, 10, 11, 12, 13.5, 14.25, "overflow", "raw",
+                        "-1.23", "-123456789012345678.12345", 19600,
+                        "2027-01-15 08:00:00.321", "2027-01-15 08:00:00.321987654",
+                        "2027-01-15 08:00:00.321", "2027-01-15 08:00:00.321987",
+                        [42], [["om", 100]], ["overflow-nested", -7]
+                    ]]
+                ]],
+                [2, null],
+                [3, []]
+            ])",
+                                                     /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0,
+                                     /*expected_commit_messages=*/std::nullopt));
+
+    ASSERT_OK_AND_ASSIGN(auto files, CurrentDataFiles(options));
+    ASSERT_EQ(1, files.size());
+    ASSERT_OK_AND_ASSIGN(auto meta, ReadShreddingMeta(files[0], "metrics", options));
+    ASSERT_EQ(2, meta.num_columns);
+    ASSERT_EQ(3, meta.max_row_width);
+
+    arrow::FieldVector expected_fields = fields;
+    expected_fields.insert(expected_fields.begin(), arrow::field("_VALUE_KIND", arrow::int8()));
+    ASSERT_OK_AND_ASSIGN(auto splits,
+                         helper->NewScan(StartupMode::LatestFull(), /*snapshot_id=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(bool success,
+                         helper->ReadAndCheckResult(arrow::struct_(expected_fields), splits,
+                                                    R"([
+                [0, 1, [
+                    ["fixed-a", [
+                        true, 1, 2, 3, 4, 5.5, 6.25, "str", "bin",
+                        "12345678.90", "123456789012345678.12345", 19500,
+                        "2023-11-14 22:13:20.123", "2023-11-14 22:13:20.123456789",
+                        "2023-11-14 22:13:20.123", "2023-11-14 22:13:20.123456",
+                        [7, null, 8], [["m1", 10], ["m2", null]], ["nested", 99]
+                    ]],
+                    ["fixed-b", [
+                        null, null, null, null, null, null, null, null, null,
+                        null, null, null, null, null, null, null, null, null, null
+                    ]],
+                    ["overflow", [
+                        false, 9, 10, 11, 12, 13.5, 14.25, "overflow", "raw",
+                        "-1.23", "-123456789012345678.12345", 19600,
+                        "2027-01-15 08:00:00.321", "2027-01-15 08:00:00.321987654",
+                        "2027-01-15 08:00:00.321", "2027-01-15 08:00:00.321987",
+                        [42], [["om", 100]], ["overflow-nested", -7]
+                    ]]
+                ]],
+                [0, 2, null],
+                [0, 3, []]
+            ])"));
+    ASSERT_TRUE(success);
+}
+
 TEST_P(WriteAndReadInteTest, TestMapSharedShreddingStructValueSchemaEvolutionReadFails) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2656,7 +2941,7 @@ TEST_P(WriteAndReadInteTest, TestOrcDictionaryLazyDecodingWithSharedShredding) {
 // Verify shared-shredding in the PK read path.
 TEST_P(WriteAndReadInteTest, TestPkSharedShreddingMap) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2710,7 +2995,7 @@ TEST_P(WriteAndReadInteTest, TestPkSharedShreddingMap) {
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithOverflow) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2823,7 +3108,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithOverflow) {
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithNullOrMissingKey) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -2914,7 +3199,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallWithNullOrMissin
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallMultipleColumns) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -3003,7 +3288,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingPartialKeyRecallMultipleColumns)
 
 TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShreddingPartialKeyRecall) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -3076,7 +3361,7 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutDefaultToSharedShreddingPartial
 
 TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefaultPartialKeyRecall) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -3153,7 +3438,7 @@ TEST_P(WriteAndReadInteTest, TestMapStorageLayoutSharedShreddingToDefaultPartial
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingDuplicateSelectedKeys) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
@@ -3201,7 +3486,7 @@ TEST_P(WriteAndReadInteTest, TestSharedShreddingDuplicateSelectedKeys) {
 
 TEST_P(WriteAndReadInteTest, TestSharedShreddingAllNullMapColumn) {
     auto [file_format, file_system] = GetParam();
-    if (file_format != "parquet" && file_format != "orc") {
+    if (file_format == "avro") {
         return;
     }
 
diff --git a/test/inte/write_inte_test.cpp b/test/inte/write_inte_test.cpp
index 114dc9e4..11564b93 100644
--- a/test/inte/write_inte_test.cpp
+++ b/test/inte/write_inte_test.cpp
@@ -4003,6 +4003,77 @@ TEST_P(WriteInteTest, TestNullabilityCheck) {
                                                 /*expected_commit_messages=*/std::nullopt));
 }
 
+TEST_P(WriteInteTest, TestPkSpillableMapSharedShreddingReadWrite) {
+    auto file_format = GetParam();
+    if (file_format == "avro") {
+        return;
+    }
+
+    auto dir = UniqueTestDirectory::Create();
+    auto map_type = arrow::map(arrow::utf8(), arrow::int64());
+    arrow::FieldVector fields = {
+        arrow::field("id", arrow::int32()),
+        arrow::field("metrics", map_type),
+    };
+    std::map options = {
+        {Options::MANIFEST_FORMAT, "avro"},
+        {Options::FILE_FORMAT, file_format},
+        {Options::BUCKET, "1"},
+        {Options::BUCKET_KEY, "id"},
+        {Options::FILE_SYSTEM, "local"},
+        {Options::WRITE_BUFFER_SIZE, "1"},
+        {Options::WRITE_BUFFER_SPILLABLE, "true"},
+        {Options::WRITE_ONLY, "true"},
+        {"fields.metrics.map.storage-layout", "shared-shredding"},
+        {"fields.metrics.map.shared-shredding.max-columns", "2"},
+    };
+    auto schema = arrow::schema(fields);
+    ::ArrowSchema c_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+    ASSERT_OK_AND_ASSIGN(auto table_path, CreateTestTable(dir->Str(), "db", "tbl", &c_schema,
+                                                          /*partition_keys=*/{},
+                                                          /*primary_keys=*/{"id"}, options));
+
+    std::string tmp_dir = PathUtil::JoinPath(dir->Str(), "tmp");
+    WriteContextBuilder write_builder(table_path, "commit_user_1");
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr write_context,
+                         write_builder.SetOptions(options)
+                             .WithStreamingMode(true)
+                             .WithTempDirectory(tmp_dir)
+                             .Finish());
+    ASSERT_OK_AND_ASSIGN(auto file_store_write, FileStoreWrite::Create(std::move(write_context)));
+
+    ASSERT_OK_AND_ASSIGN(
+        auto batch_0, TestHelper::MakeRecordBatch(
+                          arrow::struct_(fields),
+                          R"([[1, [["a", 10], ["b", 11], ["overflow", -10]]], [2, [["x", 20]]]])",
+                          /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(file_store_write->Write(std::move(batch_0)));
+    ASSERT_GT(TestHelper::CountChannelFiles(file_system_, tmp_dir), 0);
+
+    ASSERT_OK_AND_ASSIGN(auto batch_1,
+                         TestHelper::MakeRecordBatch(
+                             arrow::struct_(fields),
+                             R"([[1, [["a", 100], ["b", 101], ["overflow", -100]]], [3, null]])",
+                             /*partition_map=*/{}, /*bucket=*/0, {}));
+    ASSERT_OK(file_store_write->Write(std::move(batch_1)));
+    ASSERT_GT(TestHelper::CountChannelFiles(file_system_, tmp_dir), 0);
+
+    ASSERT_OK_AND_ASSIGN(auto commit_messages,
+                         file_store_write->PrepareCommit(/*wait_compaction=*/false,
+                                                         /*commit_identifier=*/0));
+    ASSERT_EQ(0, TestHelper::CountChannelFiles(file_system_, tmp_dir));
+    ASSERT_OK(CommitMessages(table_path, commit_messages));
+    ASSERT_OK(file_store_write->Close());
+
+    ASSERT_OK(ScanAndVerifyResult(table_path, fields,
+                                  R"([
+        [0, 1, [["a", 100], ["b", 101], ["overflow", -100]]],
+        [0, 2, [["x", 20]]],
+        [0, 3, null]
+    ])"));
+}
+
 TEST_P(WriteInteTest, TestPkSpillableDiskQuotaExhaustedFallsBackToFlush) {
     auto dir = UniqueTestDirectory::Create();
     arrow::FieldVector fields = {

From 77b904212a2ddd00be72ea570787e22c349ad9a2 Mon Sep 17 00:00:00 2001
From: lszskye <57179283+lszskye@users.noreply.github.com>
Date: Wed, 29 Jul 2026 23:14:36 -0700
Subject: [PATCH 129/138] fix(lucene): normalize prefix and wildcard queries

---
 crates/tantivy_ffi/src/reader.rs              | 113 +++++++++++++++---
 crates/tantivy_ffi/src/tokenizer.rs           |  14 ++-
 docs/source/user_guide/global_index.rst       |  11 +-
 include/paimon/predicate/full_text_search.h   |   8 +-
 .../global_index/lucene/jieba_analyzer.cpp    |  29 +++--
 .../global_index/lucene/jieba_analyzer.h      |   2 +
 .../lucene/jieba_analyzer_test.cpp            |  18 +++
 .../lucene/lucene_global_index_reader.cpp     |  56 ++++++++-
 .../lucene/lucene_global_index_reader.h       |   2 +
 .../lucene/lucene_global_index_test.cpp       |  61 ++++++++++
 .../tantivy/tantivy_equivalence_test.cpp      |  71 +++++++++--
 test/inte/global_index_test.cpp               |  16 +++
 12 files changed, 351 insertions(+), 50 deletions(-)

diff --git a/crates/tantivy_ffi/src/reader.rs b/crates/tantivy_ffi/src/reader.rs
index 460e4447..033b7f25 100644
--- a/crates/tantivy_ffi/src/reader.rs
+++ b/crates/tantivy_ffi/src/reader.rs
@@ -23,8 +23,9 @@
 //!   1 MATCH_ALL — tokenize query, BooleanQuery (Must)
 //!   2 MATCH_ANY — tokenize query, BooleanQuery (Should)
 //!   3 PHRASE    — tokenize query, PhraseQuery
-//!   4 PREFIX    — RegexQuery `.*` (no tokenization, mirrors lucene-fts)
-//!   5 WILDCARD  — RegexQuery from glob pattern (`*` → `.*`, `?` → `.`, others escaped)
+//!   4 PREFIX    — original + case-normalized RegexQuery `.*` (no tokenization)
+//!   5 WILDCARD  — original + case-normalized RegexQuery from glob pattern
+//!                 (`*` → `.*`, `?` → `.`, others escaped)
 //!
 //! For paimon-java compatibility, row_id is stored as an explicit u64 field
 //! (`fast` for O(1) retrieval). Reader translates tantivy DocAddress → row_id
@@ -39,7 +40,9 @@ use std::path::Path;
 use croaring::{Portable, Treemap};
 use tantivy::collector::{Collector, SegmentCollector};
 use tantivy::columnar::Column;
-use tantivy::query::{BooleanQuery, Occur, PhraseQuery, Query, RegexQuery, TermQuery};
+use tantivy::query::{
+    BooleanQuery, DisjunctionMaxQuery, Occur, PhraseQuery, Query, RegexQuery, TermQuery,
+};
 use tantivy::schema::{Field, IndexRecordOption};
 use tantivy::{DocAddress, DocId, Index, IndexReader, ReloadPolicy, Score, SegmentOrdinal,
                SegmentReader, Term};
@@ -48,7 +51,7 @@ use crate::buffer::PaimonTantivyBuffer;
 use crate::callback_directory::{PaimonCallbackDirectory, PaimonStreamCallbacks};
 use crate::error::{set_last_error, PaimonTantivyStatus};
 use crate::handle::{borrow_handle_mut, free_handle, into_handle};
-use crate::tokenizer::{PaimonJiebaTokenizer, TokenizeMode};
+use crate::tokenizer::{normalize_case, PaimonJiebaTokenizer, TokenizeMode};
 use crate::writer::{PAIMON_ROW_ID_FIELD_NAME, PAIMON_TEXT_FIELD_NAME, PAIMON_TOKENIZER_NAME};
 
 /// Numeric encoding of `paimon::FullTextSearch::SearchType`. Kept in sync
@@ -231,22 +234,44 @@ impl PaimonTantivyReader {
         if query.is_empty() {
             return Err("prefix query is empty".into());
         }
-        // Mirror lucene-fts: don't tokenize prefix; match indexed term bytes
-        // starting with the given prefix verbatim.
-        let pattern = format!("{}.*", regex_escape(query));
-        RegexQuery::from_pattern(&pattern, self.text_field)
-            .map(|q| Box::new(q) as Box)
-            .map_err(|e| format!("RegexQuery from prefix {query:?}: {e}"))
+        // Mirror lucene-fts: don't tokenize the prefix. Retain the original form for mixed
+        // ASCII/CJK indexed terms, and add the normalized form for pure ASCII terms.
+        let normalized_query = normalize_case(query);
+        let create_query = |value: &str| -> Result, String> {
+            let pattern = format!("{}.*", regex_escape(value));
+            RegexQuery::from_pattern(&pattern, self.text_field)
+                .map(|q| Box::new(q) as Box)
+                .map_err(|e| format!("RegexQuery from prefix {value:?}: {e}"))
+        };
+        let original = create_query(query)?;
+        if normalized_query == query {
+            return Ok(original);
+        }
+        let normalized = create_query(&normalized_query)?;
+        Ok(Box::new(DisjunctionMaxQuery::new(vec![
+            original, normalized,
+        ])))
     }
 
     fn build_wildcard_query(&self, query: &str) -> Result, String> {
         if query.is_empty() {
             return Err("wildcard query is empty".into());
         }
-        let pattern = wildcard_to_regex(query);
-        RegexQuery::from_pattern(&pattern, self.text_field)
-            .map(|q| Box::new(q) as Box)
-            .map_err(|e| format!("RegexQuery from wildcard {query:?} (pattern {pattern}): {e}"))
+        let normalized_query = normalize_wildcard_query(query);
+        let create_query = |value: &str| -> Result, String> {
+            let pattern = wildcard_to_regex(value);
+            RegexQuery::from_pattern(&pattern, self.text_field)
+                .map(|q| Box::new(q) as Box)
+                .map_err(|e| format!("RegexQuery from wildcard {value:?} (pattern {pattern}): {e}"))
+        };
+        let original = create_query(query)?;
+        if normalized_query == query {
+            return Ok(original);
+        }
+        let normalized = create_query(&normalized_query)?;
+        Ok(Box::new(DisjunctionMaxQuery::new(vec![
+            original, normalized,
+        ])))
     }
 
     fn build_query(&self, search_type: SearchType, query: &str) -> Result, String> {
@@ -473,6 +498,23 @@ fn regex_escape(input: &str) -> String {
     out
 }
 
+/// Normalize each literal segment independently while preserving wildcard
+/// operators. This matches lucene-fts, where `*` and `?` are not analyzed.
+fn normalize_wildcard_query(input: &str) -> String {
+    let mut out = String::with_capacity(input.len());
+    let mut segment_start = 0;
+    for (index, ch) in input.char_indices() {
+        if ch != '*' && ch != '?' {
+            continue;
+        }
+        out.push_str(&normalize_case(&input[segment_start..index]));
+        out.push(ch);
+        segment_start = index + ch.len_utf8();
+    }
+    out.push_str(&normalize_case(&input[segment_start..]));
+    out
+}
+
 /// Translate a glob-style wildcard ('*' = any, '?' = single char) into a
 /// regex pattern, escaping all other regex metacharacters.
 fn wildcard_to_regex(input: &str) -> String {
@@ -1026,6 +1068,14 @@ mod tests {
         assert_eq!(ids, vec![0u64]);
     }
 
+    #[test]
+    fn prefix_normalizes_ascii_alphanumeric_case() {
+        let bytes = build(&["This is a test document", "another document"]);
+        let r = open(&bytes);
+        let ids = r.search_all(SearchType::Prefix, "THIS").unwrap();
+        assert_eq!(ids, vec![0u64]);
+    }
+
     #[test]
     fn wildcard_with_star() {
         let bytes = build(&["unordered", "ordered", "border"]);
@@ -1034,6 +1084,34 @@ mod tests {
         assert_eq!(ids, vec![0u64, 1, 2]);
     }
 
+    #[test]
+    fn wildcard_normalizes_ascii_alphanumeric_segments() {
+        let bytes = build(&["This is a test document", "another document"]);
+        let r = open(&bytes);
+        assert_eq!(
+            r.search_all(SearchType::Wildcard, "*THIS*").unwrap(),
+            vec![0u64]
+        );
+        assert_eq!(
+            r.search_all(SearchType::Wildcard, "*?HIS*").unwrap(),
+            vec![0u64]
+        );
+    }
+
+    #[test]
+    fn prefix_and_wildcard_preserve_mixed_ascii_cjk_terms() {
+        let bytes = build(&["B超检查", "T恤"]);
+        let r = open(&bytes);
+        assert_eq!(
+            r.search_all(SearchType::Prefix, "B").unwrap(),
+            vec![0u64]
+        );
+        assert_eq!(
+            r.search_all(SearchType::Wildcard, "*T*").unwrap(),
+            vec![1u64]
+        );
+    }
+
     #[test]
     fn empty_query_for_match_returns_query_parse_error() {
         let bytes = build(&["hello"]);
@@ -1050,6 +1128,13 @@ mod tests {
         assert_eq!(wildcard_to_regex("*a*"), ".*a.*");
     }
 
+    #[test]
+    fn wildcard_normalization_preserves_non_alphanumeric_segments() {
+        assert_eq!(normalize_wildcard_query("*?HIS*"), "*?his*");
+        assert_eq!(normalize_wildcard_query("*THIS_IS*"), "*THIS_IS*");
+        assert_eq!(normalize_wildcard_query("*机器*"), "*机器*");
+    }
+
     // ----- limit + pre_filter + scoring (row_id-based) -----
 
     #[test]
diff --git a/crates/tantivy_ffi/src/tokenizer.rs b/crates/tantivy_ffi/src/tokenizer.rs
index b8f8b5bb..19139fcb 100644
--- a/crates/tantivy_ffi/src/tokenizer.rs
+++ b/crates/tantivy_ffi/src/tokenizer.rs
@@ -122,11 +122,7 @@ impl PaimonJiebaTokenizer {
             let start = piece.as_ptr() as usize - text_start;
             let end = start + piece.len();
             // lowercase only if pure ASCII alphanumeric (match cppjieba Normalize behavior)
-            let token_text = if is_ascii_alnum(piece) {
-                piece.to_ascii_lowercase()
-            } else {
-                piece.to_string()
-            };
+            let token_text = normalize_case(piece);
             out.push((start, end, token_text));
         }
         out
@@ -137,6 +133,14 @@ fn is_ascii_alnum(s: &str) -> bool {
     !s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric())
 }
 
+pub(crate) fn normalize_case(s: &str) -> String {
+    if is_ascii_alnum(s) {
+        s.to_ascii_lowercase()
+    } else {
+        s.to_string()
+    }
+}
+
 fn load_jieba(dict_dir: &Path) -> Result {
     let main_dict = dict_dir.join("jieba.dict.utf8");
     let mut jieba = if main_dict.exists() {
diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst
index be585329..bc12c3ae 100644
--- a/docs/source/user_guide/global_index.rst
+++ b/docs/source/user_guide/global_index.rst
@@ -74,8 +74,15 @@ search modes including match-all, match-any, phrase, prefix, and wildcard querie
 - ``MATCH_ALL``: All terms in the query must be present (AND semantics).
 - ``MATCH_ANY``: Any term in the query can match (OR semantics).
 - ``PHRASE``: Matches the exact sequence of words (with proximity).
-- ``PREFIX``: Matches terms starting with the given string (e.g., "run*" → running, runner).
-- ``WILDCARD``: Supports wildcards ``*`` and ``?`` (e.g., "ap*e", "app?e" → "apple").
+- ``PREFIX``: Matches terms starting with the given string (e.g., "run*" → running, runner). The
+  query is not tokenized. The original prefix is retained, and a pure ASCII alphanumeric prefix
+  is also matched using the lowercase case-normalization applied to pure ASCII terms at indexing
+  time. This preserves matches for mixed terms such as ``B超`` while allowing ``THIS`` to match
+  terms indexed as ``this...``.
+- ``WILDCARD``: Supports wildcards ``*`` and ``?`` (e.g., "ap*e", "app?e" → "apple"). The query
+  is not tokenized, and wildcard operators are preserved. Both the original pattern and an
+  alternative with each ASCII alphanumeric fragment lowercased are matched, covering pure ASCII
+  and mixed ASCII/non-ASCII terms.
 
 **Special Configuration:**
 
diff --git a/include/paimon/predicate/full_text_search.h b/include/paimon/predicate/full_text_search.h
index e31cc3a6..3e0676c5 100644
--- a/include/paimon/predicate/full_text_search.h
+++ b/include/paimon/predicate/full_text_search.h
@@ -79,10 +79,14 @@ struct PAIMON_EXPORT FullTextSearch {
     /// - For PHRASE: matches the exact word sequence (with optional slop). Also be analyzed.
     ///
     /// - For PREFIX: matches terms starting with the given string (e.g., "run" → running, runner).
-    ///   Only the prefix part is considered; analysis will not be applied.
+    ///   The query is not tokenized or filtered for stop words. The original prefix is retained,
+    ///   and a prefix consisting entirely of ASCII letters and digits is also matched using the
+    ///   lowercase case-normalization applied to pure ASCII terms at indexing time.
     ///
     /// - For WILDCARD: supports wildcards * and ? (e.g., "ap*e", "app?e").
-    ///   Not passed through analyzer — matched directly against indexed terms.
+    ///   The query is not tokenized or filtered for stop words. The wildcard operators are
+    ///   preserved. Both the original pattern and an alternative with each ASCII alphanumeric
+    ///   fragment lowercased are matched, covering pure ASCII and mixed ASCII/non-ASCII terms.
     ///
     /// @note Analyzer consistency between indexing and querying is critical for correctness.
     std::string query;
diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp
index e5081386..39cecec2 100644
--- a/src/paimon/global_index/lucene/jieba_analyzer.cpp
+++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp
@@ -85,6 +85,21 @@ void JiebaTokenizer::CutWithMode(const std::string& tokenize_mode, const cppjieb
     }
 }
 
+void JiebaTokenizer::NormalizeCase(std::string* term) {
+    bool is_alphanumeric = true;
+    for (const auto& c : *term) {
+        if (!std::isalnum(static_cast(c))) {
+            is_alphanumeric = false;
+            break;
+        }
+    }
+    if (is_alphanumeric && !term->empty()) {
+        std::transform(term->begin(), term->end(), term->begin(), [](char ch) {
+            return static_cast(std::tolower(static_cast(ch)));
+        });
+    }
+}
+
 void JiebaTokenizer::Normalize(const std::unordered_set& stop_words,
                                std::vector* input_ptr,
                                std::vector* output_ptr) {
@@ -100,19 +115,7 @@ void JiebaTokenizer::Normalize(const std::unordered_set& stop_words
         if (stop_words.find(term) != stop_words.end()) {
             continue;
         }
-        // to lower case
-        bool is_alphanumeric = true;
-        for (const auto& c : term) {
-            if (!std::isalnum(static_cast(c))) {
-                is_alphanumeric = false;
-                break;
-            }
-        }
-        if (is_alphanumeric && !term.empty()) {
-            std::transform(term.begin(), term.end(), term.begin(), [](char ch) {
-                return static_cast(std::tolower(static_cast(ch)));
-            });
-        }
+        NormalizeCase(&term);
         output.emplace_back(term.data(), term.length());
     }
 }
diff --git a/src/paimon/global_index/lucene/jieba_analyzer.h b/src/paimon/global_index/lucene/jieba_analyzer.h
index 6a179df5..2af534ac 100644
--- a/src/paimon/global_index/lucene/jieba_analyzer.h
+++ b/src/paimon/global_index/lucene/jieba_analyzer.h
@@ -55,6 +55,8 @@ class JiebaTokenizer : public Lucene::Tokenizer {
     static void CutWithMode(const std::string& tokenize_mode, const cppjieba::Jieba* jieba,
                             const std::string& str, std::vector* terms_ptr);
 
+    static void NormalizeCase(std::string* term);
+
     // In-place converts each string in `input` to lowercase to avoid data copying.
     static void Normalize(const std::unordered_set& stop_words,
                           std::vector* input, std::vector* output);
diff --git a/src/paimon/global_index/lucene/jieba_analyzer_test.cpp b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp
index c84f4b50..fcdbcf7d 100644
--- a/src/paimon/global_index/lucene/jieba_analyzer_test.cpp
+++ b/src/paimon/global_index/lucene/jieba_analyzer_test.cpp
@@ -109,6 +109,24 @@ TEST_P(JiebaAnalyzerTest, TestNormalize) {
     ASSERT_EQ(expected, results);
 }
 
+TEST(JiebaTokenizerTest, TestNormalizeCase) {
+    std::string alphanumeric_term = "THIS123";
+    JiebaTokenizer::NormalizeCase(&alphanumeric_term);
+    ASSERT_EQ(alphanumeric_term, "this123");
+
+    std::string mixed_ascii_cjk_term = "B超";
+    JiebaTokenizer::NormalizeCase(&mixed_ascii_cjk_term);
+    ASSERT_EQ(mixed_ascii_cjk_term, "B超");
+
+    std::string term_with_underscore = "THIS_IS";
+    JiebaTokenizer::NormalizeCase(&term_with_underscore);
+    ASSERT_EQ(term_with_underscore, "THIS_IS");
+
+    std::string chinese_term = "机器";
+    JiebaTokenizer::NormalizeCase(&chinese_term);
+    ASSERT_EQ(chinese_term, "机器");
+}
+
 INSTANTIATE_TEST_SUITE_P(ReadBufferSize, JiebaAnalyzerTest,
                          ::testing::ValuesIn(std::vector({2, 5, 10, 100})));
 
diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp
index 2d99dee4..c037f6a0 100644
--- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp
+++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp
@@ -18,6 +18,7 @@
 #include "paimon/global_index/lucene/lucene_global_index_reader.h"
 
 #include "arrow/c/bridge.h"
+#include "lucene++/DisjunctionMaxQuery.h"
 #include "lucene++/FileUtils.h"
 #include "paimon/common/utils/options_utils.h"
 #include "paimon/common/utils/path_util.h"
@@ -118,6 +119,25 @@ std::vector LuceneGlobalIndexReader::TokenizeQuery(const std::stri
     return wterms;
 }
 
+std::string LuceneGlobalIndexReader::NormalizeWildcardQuery(const std::string& query) {
+    std::string normalized_query;
+    normalized_query.reserve(query.size());
+    size_t term_begin = 0;
+    for (size_t i = 0; i <= query.size(); i++) {
+        if (i != query.size() && query[i] != '*' && query[i] != '?') {
+            continue;
+        }
+        std::string term = query.substr(term_begin, i - term_begin);
+        JiebaTokenizer::NormalizeCase(&term);
+        normalized_query.append(term);
+        if (i != query.size()) {
+            normalized_query.push_back(query[i]);
+        }
+        term_begin = i + 1;
+    }
+    return normalized_query;
+}
+
 Lucene::QueryPtr LuceneGlobalIndexReader::ConstructMatchQuery(
     const std::shared_ptr& full_text_search) const noexcept(false) {
     assert(full_text_search->search_type == FullTextSearch::SearchType::MATCH_ALL ||
@@ -155,15 +175,43 @@ Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPhraseQuery(
 Lucene::QueryPtr LuceneGlobalIndexReader::ConstructPrefixQuery(
     const std::shared_ptr& full_text_search) const noexcept(false) {
     assert(full_text_search->search_type == FullTextSearch::SearchType::PREFIX);
-    return Lucene::newLucene(Lucene::newLucene(
-        wfield_name_, LuceneUtils::StringToWstring(full_text_search->query)));
+    auto create_query = [this](const std::string& query) -> Lucene::QueryPtr {
+        return Lucene::newLucene(
+            Lucene::newLucene(wfield_name_, LuceneUtils::StringToWstring(query)));
+    };
+    std::string normalized_query = full_text_search->query;
+    JiebaTokenizer::NormalizeCase(&normalized_query);
+    Lucene::QueryPtr query = create_query(full_text_search->query);
+    if (normalized_query == full_text_search->query) {
+        return query;
+    }
+
+    // Preserve the original query for mixed ASCII/CJK indexed terms such as "B超", whose
+    // complete token is not lowercased by the index analyzer. The normalized alternative still
+    // matches pure ASCII terms. DisjunctionMax avoids double-counting scores if both match.
+    auto disjunction = Lucene::newLucene(0.0);
+    disjunction->add(query);
+    disjunction->add(create_query(normalized_query));
+    return disjunction;
 }
 
 Lucene::QueryPtr LuceneGlobalIndexReader::ConstructWildCardQuery(
     const std::shared_ptr& full_text_search) const noexcept(false) {
     assert(full_text_search->search_type == FullTextSearch::SearchType::WILDCARD);
-    return Lucene::newLucene(Lucene::newLucene(
-        wfield_name_, LuceneUtils::StringToWstring(full_text_search->query)));
+    auto create_query = [this](const std::string& query) -> Lucene::QueryPtr {
+        return Lucene::newLucene(
+            Lucene::newLucene(wfield_name_, LuceneUtils::StringToWstring(query)));
+    };
+    std::string normalized_query = NormalizeWildcardQuery(full_text_search->query);
+    Lucene::QueryPtr query = create_query(full_text_search->query);
+    if (normalized_query == full_text_search->query) {
+        return query;
+    }
+
+    auto disjunction = Lucene::newLucene(0.0);
+    disjunction->add(query);
+    disjunction->add(create_query(normalized_query));
+    return disjunction;
 }
 
 Result> LuceneGlobalIndexReader::SearchWithLimit(
diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.h b/src/paimon/global_index/lucene/lucene_global_index_reader.h
index b86c332f..4fd8051e 100644
--- a/src/paimon/global_index/lucene/lucene_global_index_reader.h
+++ b/src/paimon/global_index/lucene/lucene_global_index_reader.h
@@ -128,6 +128,8 @@ class LuceneGlobalIndexReader : public GlobalIndexReader {
 
     std::vector TokenizeQuery(const std::string& query) const;
 
+    static std::string NormalizeWildcardQuery(const std::string& query);
+
     std::shared_ptr CreateAllResult() const {
         return nullptr;
     }
diff --git a/src/paimon/global_index/lucene/lucene_global_index_test.cpp b/src/paimon/global_index/lucene/lucene_global_index_test.cpp
index cb4bffcf..4fac71a3 100644
--- a/src/paimon/global_index/lucene/lucene_global_index_test.cpp
+++ b/src/paimon/global_index/lucene/lucene_global_index_test.cpp
@@ -236,6 +236,14 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) {
                                  /*pre_filter=*/std::nullopt)));
         CheckResult(result, {3l});
     }
+    {
+        ASSERT_OK_AND_ASSIGN(auto result,
+                             lucene_reader->VisitFullTextSearch(std::make_shared(
+                                 "f0",
+                                 /*limit=*/10, "THIS", FullTextSearch::SearchType::PREFIX,
+                                 /*pre_filter=*/std::nullopt)));
+        CheckResult(result, {1l, 0l});
+    }
     // test wildcard query
     {
         ASSERT_OK_AND_ASSIGN(auto result,
@@ -253,6 +261,22 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) {
                                  /*pre_filter=*/std::nullopt)));
         CheckResult(result, {3l});
     }
+    {
+        ASSERT_OK_AND_ASSIGN(auto result,
+                             lucene_reader->VisitFullTextSearch(std::make_shared(
+                                 "f0",
+                                 /*limit=*/10, "*THIS*", FullTextSearch::SearchType::WILDCARD,
+                                 /*pre_filter=*/std::nullopt)));
+        CheckResult(result, {1l, 0l});
+    }
+    {
+        ASSERT_OK_AND_ASSIGN(auto result,
+                             lucene_reader->VisitFullTextSearch(std::make_shared(
+                                 "f0",
+                                 /*limit=*/10, "*?HIS*", FullTextSearch::SearchType::WILDCARD,
+                                 /*pre_filter=*/std::nullopt)));
+        CheckResult(result, {1l, 0l});
+    }
     // test filter
     {
         ASSERT_OK_AND_ASSIGN(auto result,
@@ -441,6 +465,43 @@ TEST_P(LuceneGlobalIndexTest, TestSimpleChinese) {
     }
 }
 
+TEST_P(LuceneGlobalIndexTest, TestMixedAsciiCjkPrefixAndWildcard) {
+    auto test_root_dir = paimon::test::UniqueTestDirectory::Create();
+    ASSERT_TRUE(test_root_dir);
+    auto tmp_dir = paimon::test::UniqueTestDirectory::Create();
+    ASSERT_TRUE(tmp_dir);
+
+    std::map options = {
+        {"lucene-fts.write.omit-term-freq-and-position", "false"},
+        {"lucene-fts.read.buffer-size", std::to_string(GetParam())},
+        {"lucene-fts.jieba.tokenize-mode", "query"},
+        {"lucene-fts.write.tmp.directory", tmp_dir->Str()}};
+    std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([
+            ["B超检查"],
+            ["T恤"]
+        ])")
+                                              .ValueOrDie();
+
+    ASSERT_OK_AND_ASSIGN(auto meta, WriteGlobalIndex(test_root_dir->Str(), data_type_, options,
+                                                     array, Range(0, 1), tmp_dir->Str()));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         CreateGlobalIndexReader(test_root_dir->Str(), data_type_, options, meta));
+    auto lucene_reader = std::dynamic_pointer_cast(reader);
+    ASSERT_TRUE(lucene_reader);
+
+    ASSERT_OK_AND_ASSIGN(auto prefix_result,
+                         lucene_reader->VisitFullTextSearch(std::make_shared(
+                             "f0", /*limit=*/10, "B", FullTextSearch::SearchType::PREFIX,
+                             /*pre_filter=*/std::nullopt)));
+    CheckResult(prefix_result, {0l});
+
+    ASSERT_OK_AND_ASSIGN(auto wildcard_result,
+                         lucene_reader->VisitFullTextSearch(std::make_shared(
+                             "f0", /*limit=*/10, "*T*", FullTextSearch::SearchType::WILDCARD,
+                             /*pre_filter=*/std::nullopt)));
+    CheckResult(wildcard_result, {1l});
+}
+
 TEST_F(LuceneGlobalIndexTest, TestInvalidWithoutTmpDir) {
     auto test_root_dir = paimon::test::UniqueTestDirectory::Create();
     ASSERT_TRUE(test_root_dir);
diff --git a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp
index 33f1e596..44f780ac 100644
--- a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp
+++ b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp
@@ -17,6 +17,23 @@
  * under the License.
  */
 
+/*
+ * Equivalence + benchmark.
+ *
+ * EQUIVALENCE: a parametric corpus × query battery that compares lucene-fts
+ * and tantivy-fulltext result *sets* (doc_id only — not score order, not score
+ * values). Coverage targets:
+ *   - English bag-of-words: MATCH_ALL / MATCH_ANY / PHRASE / PREFIX / WILDCARD
+ *   - Chinese (jieba "query" mode): MATCH_ALL / MATCH_ANY / PHRASE
+ *   - Pre_filter intersection (no scoring)
+ * PREFIX and WILDCARD equivalence covers ASCII token patterns, including case
+ * normalization. Engine-specific regex edge cases are outside this test.
+ *
+ * BENCHMARK: build a 200-doc index per backend and time write + 100 queries.
+ * Prints to stderr; never fails on perf — guarding against perf regressions
+ * is out of scope here. Numbers are a reportable baseline.
+ */
+
 #include 
 #include 
 #include 
@@ -208,22 +225,30 @@ TEST_F(TantivyEquivalenceTest, EnglishBagOfWordsBattery) {
     struct Case {
         std::string query;
         FullTextSearch::SearchType type;
+        std::set expected_ids;
     };
     std::vector cases = {
-        {"alpha", FullTextSearch::SearchType::MATCH_ALL},
-        {"alpha", FullTextSearch::SearchType::MATCH_ANY},
-        {"alpha beta", FullTextSearch::SearchType::MATCH_ALL},
-        {"alpha beta", FullTextSearch::SearchType::MATCH_ANY},
-        {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ALL},
-        {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ANY},
-        {"epsilon iota", FullTextSearch::SearchType::MATCH_ALL},
-        {"alpha beta gamma", FullTextSearch::SearchType::PHRASE},
-        {"beta gamma delta", FullTextSearch::SearchType::PHRASE},
-        {"delta epsilon", FullTextSearch::SearchType::PHRASE},
+        {"alpha", FullTextSearch::SearchType::MATCH_ALL, {}},
+        {"alpha", FullTextSearch::SearchType::MATCH_ANY, {}},
+        {"alpha beta", FullTextSearch::SearchType::MATCH_ALL, {}},
+        {"alpha beta", FullTextSearch::SearchType::MATCH_ANY, {}},
+        {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ALL, {}},
+        {"alpha gamma delta", FullTextSearch::SearchType::MATCH_ANY, {}},
+        {"epsilon iota", FullTextSearch::SearchType::MATCH_ALL, {}},
+        {"alpha beta gamma", FullTextSearch::SearchType::PHRASE, {}},
+        {"beta gamma delta", FullTextSearch::SearchType::PHRASE, {}},
+        {"delta epsilon", FullTextSearch::SearchType::PHRASE, {}},
+        {"ALP", FullTextSearch::SearchType::PREFIX, {0, 1, 4, 6, 9}},
+        {"*ALPHA*", FullTextSearch::SearchType::WILDCARD, {0, 1, 4, 6, 9}},
+        {"*ALP?A*", FullTextSearch::SearchType::WILDCARD, {0, 1, 4, 6, 9}},
     };
     for (const auto& c : cases) {
         auto [l, t] = RunPair(pair, c.query, c.type);
         ASSERT_EQ(l, t) << "diverge: query=" << c.query << " type=" << static_cast(c.type);
+        if (!c.expected_ids.empty()) {
+            ASSERT_EQ(l, c.expected_ids) << "unexpected matches: query=" << c.query
+                                         << " type=" << static_cast(c.type);
+        }
     }
 }
 
@@ -263,6 +288,32 @@ TEST_F(TantivyEquivalenceTest, ChineseQueryModeBattery) {
     }
 }
 
+TEST_F(TantivyEquivalenceTest, MixedAsciiCjkPrefixAndWildcard) {
+    auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())});
+    auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([
+        ["B超检查"],
+        ["T恤"]
+    ])")
+                     .ValueOrDie();
+    std::map lopts = {{"lucene-fts.jieba.tokenize-mode", "query"}};
+    std::map topts = {
+        {"tantivy-fulltext.tantivy.write.tokenizer", "paimon_jieba"},
+        {"tantivy-fulltext.jieba.tokenize-mode", "query"},
+    };
+    auto pair = WriteAndOpenBoth(data_type, array, lopts, topts);
+
+    {
+        auto [l, t] = RunPair(pair, "B", FullTextSearch::SearchType::PREFIX);
+        ASSERT_EQ(l, (std::set{0}));
+        ASSERT_EQ(t, l);
+    }
+    {
+        auto [l, t] = RunPair(pair, "*T*", FullTextSearch::SearchType::WILDCARD);
+        ASSERT_EQ(l, (std::set{1}));
+        ASSERT_EQ(t, l);
+    }
+}
+
 TEST_F(TantivyEquivalenceTest, PreFilterIntersectionEquivalent) {
     auto data_type = arrow::struct_({arrow::field("f0", arrow::utf8())});
     auto array = arrow::ipc::internal::json::ArrayFromJSON(data_type, R"([
diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp
index 5ea50d9d..ccb9ce32 100644
--- a/test/inte/global_index_test.cpp
+++ b/test/inte/global_index_test.cpp
@@ -2370,6 +2370,22 @@ TEST_P(GlobalIndexTest, TestLuceneWriteCommitScanReadIndexWithScore) {
                                  /*pre_filter=*/std::nullopt)));
         ASSERT_TRUE(index_result->ToString().find("row ids: {3}") != std::string::npos);
     }
+    {
+        ASSERT_OK_AND_ASSIGN(auto index_result,
+                             index_reader->VisitFullTextSearch(std::make_shared(
+                                 "f0",
+                                 /*limit=*/10, "THIS", FullTextSearch::SearchType::PREFIX,
+                                 /*pre_filter=*/std::nullopt)));
+        ASSERT_TRUE(index_result->ToString().find("row ids: {0,1}") != std::string::npos);
+    }
+    {
+        ASSERT_OK_AND_ASSIGN(auto index_result,
+                             index_reader->VisitFullTextSearch(std::make_shared(
+                                 "f0",
+                                 /*limit=*/10, "*THIS*", FullTextSearch::SearchType::WILDCARD,
+                                 /*pre_filter=*/std::nullopt)));
+        ASSERT_TRUE(index_result->ToString().find("row ids: {0,1}") != std::string::npos);
+    }
 }
 
 TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) {

From b58d2386ae61dba2808bcd24d4ac0ca724ee6b75 Mon Sep 17 00:00:00 2001
From: lszskye <57179283+lszskye@users.noreply.github.com>
Date: Thu, 30 Jul 2026 06:31:39 -0700
Subject: [PATCH 130/138] chore: update version to 0.3.0

---
 CMakeLists.txt      | 2 +-
 docs/source/conf.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2e640713..52053d4e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -41,7 +41,7 @@ if(NOT CMAKE_BUILD_TYPE)
 endif()
 
 project(paimon
-        VERSION 0.2.3
+        VERSION 0.3.0
         DESCRIPTION "Paimon C++ Project")
 
 string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE)
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 989ea078..92dfa7c1 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -114,7 +114,7 @@
 # The master toctree document.
 master_doc = "index"
 
-version = "0.2.3"
+version = "0.3.0"
 
 html_theme_options = {
     "show_toc_level": 2,

From e543f76335672302703afa9f4b847c5d840ae2bf Mon Sep 17 00:00:00 2001
From: Nicholas Jiang 
Date: Fri, 31 Jul 2026 09:11:45 +0800
Subject: [PATCH 131/138] feat(blob): support placeholder fallback for partial
 updates

---
 src/paimon/CMakeLists.txt                     |   3 +
 src/paimon/common/data/blob_defs.h            |  62 ++
 .../reader/blob_fallback_batch_reader.cpp     | 406 ++++++++++++
 .../reader/blob_fallback_batch_reader.h       | 162 +++++
 .../blob_fallback_batch_reader_test.cpp       | 361 +++++++++++
 .../reader/data_evolution_file_reader.cpp     |   9 +-
 .../reader/data_evolution_file_reader.h       |   6 +-
 .../data_evolution_file_reader_test.cpp       |  16 +-
 src/paimon/core/append/append_only_writer.cpp |  16 +-
 src/paimon/core/append/append_only_writer.h   |   2 +-
 .../core/io/blob_data_file_writer_factory.cpp |  17 +-
 .../core/io/blob_data_file_writer_factory.h   |   6 +-
 .../complete_row_tracking_fields_reader.cpp   |  28 +-
 .../io/complete_row_tracking_fields_reader.h  |  12 +-
 ...mplete_row_tracking_fields_reader_test.cpp |  48 +-
 src/paimon/core/mergetree/lookup_levels.cpp   |   3 +-
 .../core/operation/abstract_split_read.cpp    |  27 +-
 .../core/operation/abstract_split_read.h      |  10 +-
 .../operation/data_evolution_split_read.cpp   | 234 +++++--
 .../operation/data_evolution_split_read.h     |  46 +-
 .../data_evolution_split_read_test.cpp        | 141 +++--
 .../core/operation/merge_file_split_read.cpp  |   6 +-
 .../core/operation/raw_file_split_read.cpp    |   3 +-
 .../source/data_evolution_batch_scan.cpp      |  25 +-
 .../table/source/data_evolution_batch_scan.h  |  11 +-
 .../source/data_evolution_batch_scan_test.cpp | 116 ++++
 .../format/blob/blob_file_batch_reader.cpp    |  50 +-
 .../format/blob/blob_file_batch_reader.h      |  57 +-
 .../blob/blob_file_batch_reader_test.cpp      |  98 ++-
 src/paimon/format/blob/blob_format_writer.cpp |  23 +-
 src/paimon/format/blob/blob_format_writer.h   |  11 +-
 .../format/blob/blob_format_writer_test.cpp   | 342 ++++++++--
 src/paimon/format/blob/blob_reader_builder.h  |   6 +-
 .../format/blob/blob_stats_extractor.cpp      |   5 +-
 src/paimon/format/blob/blob_writer_builder.h  |   6 +-
 test/inte/blob_table_inte_test.cpp            | 590 ++++++++++++++++++
 36 files changed, 2684 insertions(+), 280 deletions(-)
 create mode 100644 src/paimon/common/reader/blob_fallback_batch_reader.cpp
 create mode 100644 src/paimon/common/reader/blob_fallback_batch_reader.h
 create mode 100644 src/paimon/common/reader/blob_fallback_batch_reader_test.cpp
 create mode 100644 src/paimon/core/table/source/data_evolution_batch_scan_test.cpp

diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt
index 1dfc2a5d..f80c3ee2 100644
--- a/src/paimon/CMakeLists.txt
+++ b/src/paimon/CMakeLists.txt
@@ -133,6 +133,7 @@ set(PAIMON_COMMON_SRCS
     common/reader/predicate_batch_reader.cpp
     common/reader/prefetch_file_batch_reader_impl.cpp
     common/reader/reader_utils.cpp
+    common/reader/blob_fallback_batch_reader.cpp
     common/reader/blob_view_resolving_batch_reader.cpp
     common/reader/complete_row_kind_batch_reader.cpp
     common/reader/data_evolution_file_reader.cpp
@@ -544,6 +545,7 @@ if(PAIMON_BUILD_TESTS)
                     common/reader/prefetch_file_batch_reader_impl_test.cpp
                     common/reader/reader_utils_test.cpp
                     common/reader/complete_row_kind_batch_reader_test.cpp
+                    common/reader/blob_fallback_batch_reader_test.cpp
                     common/reader/blob_view_resolving_batch_reader_test.cpp
                     common/reader/data_evolution_file_reader_test.cpp
                     common/reader/data_evolution_array_test.cpp
@@ -805,6 +807,7 @@ if(PAIMON_BUILD_TESTS)
                     core/table/source/table_read_test.cpp
                     core/table/source/append_count_reader_test.cpp
                     core/table/source/pk_count_reader_test.cpp
+                    core/table/source/data_evolution_batch_scan_test.cpp
                     core/table/source/data_split_test.cpp
                     core/table/source/deletion_file_test.cpp
                     core/table/source/split_generator_test.cpp
diff --git a/src/paimon/common/data/blob_defs.h b/src/paimon/common/data/blob_defs.h
index 35b52e2a..deacd1d0 100644
--- a/src/paimon/common/data/blob_defs.h
+++ b/src/paimon/common/data/blob_defs.h
@@ -20,6 +20,10 @@
 #pragma once
 
 #include 
+#include 
+#include 
+#include 
+#include 
 
 namespace paimon {
 
@@ -48,6 +52,64 @@ class BlobDefs {
 
     /// A bin_length value of -1 in the index indicates a null blob entry.
     static constexpr int64_t kNullBinLength = -1;
+    /// A bin_length value of -2 in the index indicates a placeholder blob entry, written by
+    /// data-evolution partial updates for rows whose blob value is not updated. A placeholder
+    /// entry occupies no file space; readers must fall back to an older blob file covering the
+    /// same row to resolve the value. Aligned with Java's BlobFormatWriter.PLACE_HOLDER_LENGTH.
+    static constexpr int64_t kPlaceholderBinLength = -2;
+    /// Sentinel bytes standing for a placeholder blob value in two internal channels:
+    ///
+    /// - Write channel: a data-evolution partial update (a blob-only column write, see
+    ///   kWritePlaceholderKey) marks a not-updated row with these bytes, and the blob format
+    ///   writer persists it as a bin_length -2 entry. Use PlaceholderSentinelView() to build
+    ///   such write arrays. Outside that mode the writer never interprets values, so arbitrary
+    ///   user bytes can never be turned into a placeholder entry.
+    /// - Read channel: a placeholder-aware reader (see kEmitPlaceholderSentinelKey) emits these
+    ///   bytes for -2 entries so the fallback merge can identify placeholders after the batch
+    ///   has passed through schema-mapping readers.
+    ///
+    /// Both channels identify a placeholder by exact byte equality with this internal reserved
+    /// value (IsPlaceholderSentinel), and the fallback merge byte-compares every layer of a
+    /// bunch — including files written outside the write channel. A user blob whose bytes
+    /// exactly equal the marker therefore collides with it in two ways: written through the
+    /// partial-update channel it is persisted as a placeholder entry, which a single-layer read
+    /// rejects loudly (no older layer can resolve it); left untouched in an older layer under a
+    /// later partial update it reads as a placeholder in every layer and silently degrades to a
+    /// null blob. The marker is distinctive enough that these collisions are accepted as
+    /// negligibly improbable. Sentinel bytes are never stored in blob files.
+    static constexpr char kPlaceholderSentinel[] = "_PAIMON_BLOB_PLACEHOLDER";
+    /// Byte length of kPlaceholderSentinel, excluding the literal's terminating NUL.
+    static constexpr int32_t kPlaceholderSentinelLength = sizeof(kPlaceholderSentinel) - 1;
+    /// Internal (non user-facing) format option, "false" by default: when "true", the blob
+    /// reader emits kPlaceholderSentinel for placeholder entries instead of failing on them.
+    /// Only the data-evolution blob fallback read path sets this.
+    static constexpr char kEmitPlaceholderSentinelKey[] = "blob.internal.emit-placeholder-sentinel";
+    /// Internal (non user-facing) format option, "false" by default: when "true", the blob
+    /// format writer persists a value exactly equal to kPlaceholderSentinel as a bin_length -2
+    /// entry. Only set for data-evolution partial updates, i.e. blob-only column writes of a
+    /// table with data evolution enabled; all other writes store bytes verbatim.
+    static constexpr char kWritePlaceholderKey[] = "blob.internal.write-placeholder";
+
+    /// The sentinel bytes for building a data-evolution partial-update write array: a row equal
+    /// to this view is persisted as a placeholder entry (see kWritePlaceholderKey).
+    static std::string_view PlaceholderSentinelView() {
+        return {kPlaceholderSentinel, static_cast(kPlaceholderSentinelLength)};
+    }
+
+    /// True when the bytes are exactly the placeholder sentinel.
+    static bool IsPlaceholderSentinel(const char* data, size_t size) {
+        return size == static_cast(kPlaceholderSentinelLength) &&
+               memcmp(data, kPlaceholderSentinel, kPlaceholderSentinelLength) == 0;
+    }
+
+    /// Removes the internal placeholder option keys from a format options map. The placeholder
+    /// channels must only ever be enabled by the internal data-evolution write and read paths,
+    /// so every consumer building format options from user-supplied table options strips these
+    /// keys before applying its own decision.
+    static void EraseInternalPlaceholderOptions(std::map* options) {
+        options->erase(kEmitPlaceholderSentinelKey);
+        options->erase(kWritePlaceholderKey);
+    }
     /// Blob file format version.
     static constexpr int8_t kFileVersion = 1;
     /// Magic number identifying the start of each blob bin.
diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.cpp b/src/paimon/common/reader/blob_fallback_batch_reader.cpp
new file mode 100644
index 00000000..68d45c11
--- /dev/null
+++ b/src/paimon/common/reader/blob_fallback_batch_reader.cpp
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/reader/blob_fallback_batch_reader.h"
+
+#include 
+#include 
+#include 
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "fmt/format.h"
+#include "paimon/common/data/blob_defs.h"
+#include "paimon/common/data/blob_utils.h"
+#include "paimon/common/metrics/metrics_impl.h"
+#include "paimon/common/reader/reader_utils.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+
+namespace paimon {
+
+Result> BlobFallbackBatchReader::Create(
+    std::vector>&& sequence_groups,
+    const std::shared_ptr& read_schema, int32_t read_batch_size,
+    const std::shared_ptr& pool) {
+    if (sequence_groups.size() < 2) {
+        return Status::Invalid(
+            "Blob fallback needs at least two sequence groups; a single group should be read "
+            "sequentially.");
+    }
+    if (read_schema == nullptr) {
+        return Status::Invalid("Blob fallback read schema cannot be nullptr.");
+    }
+    if (read_batch_size <= 0) {
+        return Status::Invalid(fmt::format(
+            "Blob fallback read batch size '{}' should be larger than zero", read_batch_size));
+    }
+    int32_t blob_field_idx = -1;
+    for (int32_t i = 0; i < read_schema->num_fields(); i++) {
+        if (BlobUtils::IsBlobField(read_schema->field(i))) {
+            if (blob_field_idx != -1) {
+                return Status::Invalid(
+                    "Blob fallback read schema should contain exactly one blob field.");
+            }
+            blob_field_idx = i;
+        }
+    }
+    if (blob_field_idx == -1) {
+        return Status::Invalid("Blob fallback read schema should contain a blob field.");
+    }
+    int32_t row_id_field_idx = read_schema->GetFieldIndex(SpecialFields::RowId().Name());
+    int32_t seq_num_field_idx = read_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name());
+    std::vector groups;
+    groups.reserve(sequence_groups.size());
+    for (auto& segments : sequence_groups) {
+        if (segments.empty()) {
+            return Status::Invalid("Blob fallback sequence group should not be empty.");
+        }
+        for (const auto& segment : segments) {
+            if (segment.reader == nullptr && segment.gap_selected_ranges.empty()) {
+                return Status::Invalid(
+                    "Blob fallback gap segment should cover at least one selected row id.");
+            }
+        }
+        GroupCursor cursor;
+        cursor.segments = std::move(segments);
+        groups.push_back(std::move(cursor));
+    }
+    return std::unique_ptr(
+        new BlobFallbackBatchReader(std::move(groups), read_schema, blob_field_idx,
+                                    row_id_field_idx, seq_num_field_idx, read_batch_size, pool));
+}
+
+BlobFallbackBatchReader::BlobFallbackBatchReader(std::vector&& groups,
+                                                 const std::shared_ptr& read_schema,
+                                                 int32_t blob_field_idx, int32_t row_id_field_idx,
+                                                 int32_t seq_num_field_idx, int32_t read_batch_size,
+                                                 const std::shared_ptr& pool)
+    : groups_(std::move(groups)),
+      read_schema_(read_schema),
+      blob_field_idx_(blob_field_idx),
+      row_id_field_idx_(row_id_field_idx),
+      seq_num_field_idx_(seq_num_field_idx),
+      read_batch_size_(read_batch_size),
+      arrow_pool_(GetArrowPool(pool)) {}
+
+Result BlobFallbackBatchReader::FillWindow(size_t group_idx, int64_t want,
+                                                    std::vector* chunks) {
+    GroupCursor& cursor = groups_[group_idx];
+    int64_t collected = 0;
+    while (collected < want) {
+        if (!cursor.pending.empty()) {
+            const std::shared_ptr& front = cursor.pending.front();
+            int64_t available = front->length() - cursor.pending_pos;
+            int64_t take = std::min(available, want - collected);
+            chunks->push_back(Chunk{front, cursor.pending_pos, take, {}});
+            cursor.pending_pos += take;
+            collected += take;
+            if (cursor.pending_pos == front->length()) {
+                cursor.pending.pop_front();
+                cursor.pending_pos = 0;
+            }
+            continue;
+        }
+        if (cursor.segment_idx >= cursor.segments.size()) {
+            // group exhausted; only the first group may define a shorter window
+            break;
+        }
+        Segment& segment = cursor.segments[cursor.segment_idx];
+        if (segment.reader == nullptr) {
+            // gap segment: all rows are placeholders, stepped range by range so the row ids
+            // stay available for all-placeholder rows
+            if (cursor.gap_range_idx >= segment.gap_selected_ranges.size()) {
+                cursor.segment_idx++;
+                cursor.gap_range_idx = 0;
+                cursor.gap_range_pos = 0;
+                continue;
+            }
+            const Range& range = segment.gap_selected_ranges[cursor.gap_range_idx];
+            int64_t remaining = range.Count() - cursor.gap_range_pos;
+            int64_t take = std::min(remaining, want - collected);
+            Chunk chunk{nullptr, 0, take, {}};
+            if (row_id_field_idx_ >= 0) {
+                chunk.gap_row_ids.reserve(take);
+                for (int64_t k = 0; k < take; k++) {
+                    chunk.gap_row_ids.push_back(range.from + cursor.gap_range_pos + k);
+                }
+            }
+            chunks->push_back(std::move(chunk));
+            cursor.gap_range_pos += take;
+            collected += take;
+            if (cursor.gap_range_pos == range.Count()) {
+                cursor.gap_range_idx++;
+                cursor.gap_range_pos = 0;
+            }
+            continue;
+        }
+        PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch_with_bitmap,
+                               segment.reader->NextBatchWithBitmap());
+        if (BatchReader::IsEofBatch(batch_with_bitmap)) {
+            cursor.segment_idx++;
+            continue;
+        }
+        auto& [read_batch, bitmap] = batch_with_bitmap;
+        auto& [c_array, c_schema] = read_batch;
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr src_array,
+                                          arrow::ImportArray(c_array.get(), c_schema.get()));
+        PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector selected_array_vec,
+                               ReaderUtils::GenerateFilteredArrayVector(src_array, bitmap));
+        for (const auto& selected_array : selected_array_vec) {
+            if (selected_array->length() == 0) {
+                continue;
+            }
+            auto struct_array = std::dynamic_pointer_cast(selected_array);
+            if (struct_array == nullptr) {
+                return Status::Invalid("Blob fallback expects file readers to emit struct arrays.");
+            }
+            cursor.pending.push_back(std::move(struct_array));
+        }
+    }
+    return collected;
+}
+
+Result> BlobFallbackBatchReader::ComputePlaceholderFlags(
+    const std::vector& chunks, int64_t row_count) const {
+    std::vector flags(row_count, false);
+    int64_t pos = 0;
+    for (const auto& chunk : chunks) {
+        if (chunk.array == nullptr) {
+            // gap rows stand for placeholders
+            std::fill(flags.begin() + pos, flags.begin() + pos + chunk.length, true);
+        } else {
+            std::shared_ptr blob_col = chunk.array->field(blob_field_idx_);
+            auto binary_col = std::dynamic_pointer_cast(blob_col);
+            if (binary_col == nullptr) {
+                return Status::Invalid(fmt::format(
+                    "Blob fallback expects the blob column to be large binary, but got {}",
+                    blob_col->type()->ToString()));
+            }
+            for (int64_t k = 0; k < chunk.length; k++) {
+                int64_t idx = chunk.offset + k;
+                if (binary_col->IsNull(idx)) {
+                    continue;
+                }
+                std::string_view value = binary_col->GetView(idx);
+                if (BlobDefs::IsPlaceholderSentinel(value.data(), value.size())) {
+                    flags[pos + k] = true;
+                }
+            }
+        }
+        pos += chunk.length;
+    }
+    return flags;
+}
+
+Result> BlobFallbackBatchReader::AssembleRowIdRun(
+    const std::vector& chunks, int64_t run_start, int64_t run_end) const {
+    arrow::ArrayVector pieces;
+    int64_t pos = 0;
+    for (const auto& chunk : chunks) {
+        int64_t overlap_start = std::max(run_start, pos);
+        int64_t overlap_end = std::min(run_end, pos + chunk.length);
+        if (overlap_start < overlap_end) {
+            if (chunk.array != nullptr) {
+                std::shared_ptr column = chunk.array->field(row_id_field_idx_);
+                pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos),
+                                               overlap_end - overlap_start));
+            } else {
+                arrow::Int64Builder builder(arrow_pool_.get());
+                for (int64_t r = overlap_start; r < overlap_end; r++) {
+                    PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(chunk.gap_row_ids[r - pos]));
+                }
+                std::shared_ptr piece;
+                PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&piece));
+                pieces.push_back(std::move(piece));
+            }
+        }
+        pos += chunk.length;
+        if (pos >= run_end) {
+            break;
+        }
+    }
+    if (pieces.size() == 1 && pieces[0]->offset() == 0) {
+        return pieces[0];
+    }
+    // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the
+    // zero-offset BatchReader contract.
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array,
+                                      arrow::Concatenate(pieces, arrow_pool_.get()));
+    return concat_array;
+}
+
+Result> BlobFallbackBatchReader::AssembleColumn(
+    int32_t field_idx, const std::vector& group_choice,
+    const std::vector>& group_chunks) const {
+    const auto row_count = static_cast(group_choice.size());
+    arrow::ArrayVector pieces;
+    int64_t run_start = 0;
+    while (run_start < row_count) {
+        const int32_t group = group_choice[run_start];
+        int64_t run_end = run_start + 1;
+        while (run_end < row_count && group_choice[run_end] == group) {
+            run_end++;
+        }
+        if (group < 0) {
+            // placeholder in every layer: the blob degrades to null; the row keeps its row id
+            // (taken from the newest group, which steps in lockstep), reports -1 as its
+            // sequence number, and returns null for every other field
+            if (field_idx == row_id_field_idx_) {
+                PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_id_piece,
+                                       AssembleRowIdRun(group_chunks[0], run_start, run_end));
+                pieces.push_back(std::move(row_id_piece));
+            } else if (field_idx == seq_num_field_idx_) {
+                arrow::Int64Scalar seq_scalar(-1);
+                PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                    std::shared_ptr seq_piece,
+                    arrow::MakeArrayFromScalar(seq_scalar, run_end - run_start, arrow_pool_.get()));
+                pieces.push_back(std::move(seq_piece));
+            } else {
+                PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+                    std::shared_ptr null_piece,
+                    arrow::MakeArrayOfNull(read_schema_->field(field_idx)->type(),
+                                           run_end - run_start, arrow_pool_.get()));
+                pieces.push_back(std::move(null_piece));
+            }
+        } else {
+            int64_t pos = 0;
+            for (const auto& chunk : group_chunks[group]) {
+                int64_t overlap_start = std::max(run_start, pos);
+                int64_t overlap_end = std::min(run_end, pos + chunk.length);
+                if (overlap_start < overlap_end) {
+                    if (chunk.array == nullptr) {
+                        return Status::Invalid(
+                            "Unexpected: a gap row was chosen as a blob fallback result.");
+                    }
+                    std::shared_ptr column = chunk.array->field(field_idx);
+                    pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos),
+                                                   overlap_end - overlap_start));
+                }
+                pos += chunk.length;
+                if (pos >= run_end) {
+                    break;
+                }
+            }
+        }
+        run_start = run_end;
+    }
+    if (pieces.size() == 1 && pieces[0]->offset() == 0) {
+        return pieces[0];
+    }
+    // Concatenate flattens non-zero offsets left by Slice, so the exported batch honors the
+    // zero-offset BatchReader contract.
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array,
+                                      arrow::Concatenate(pieces, arrow_pool_.get()));
+    return concat_array;
+}
+
+Result BlobFallbackBatchReader::NextBatch() {
+    if (closed_) {
+        return Status::Invalid("blob fallback batch reader is closed");
+    }
+    std::vector> group_chunks(groups_.size());
+    // the first (newest) group defines the window; the others must step in lockstep
+    PAIMON_ASSIGN_OR_RAISE(int64_t row_count, FillWindow(0, read_batch_size_, &group_chunks[0]));
+    for (size_t g = 1; g < groups_.size(); g++) {
+        // ask for one row even when the first group is exhausted, so that a longer group is
+        // reported as a misalignment instead of silently truncating the read
+        const int64_t want = std::max(row_count, 1);
+        PAIMON_ASSIGN_OR_RAISE(int64_t got, FillWindow(g, want, &group_chunks[g]));
+        if (got != row_count) {
+            return Status::Invalid(fmt::format(
+                "All sequence groups of a blob fallback read should have the same number of "
+                "rows: group {} yielded {} rows in a window of {}",
+                g, got, row_count));
+        }
+    }
+    if (row_count == 0) {
+        return BatchReader::MakeEofBatch();
+    }
+
+    std::vector> placeholder_flags(groups_.size());
+    for (size_t g = 0; g < groups_.size(); g++) {
+        PAIMON_ASSIGN_OR_RAISE(placeholder_flags[g],
+                               ComputePlaceholderFlags(group_chunks[g], row_count));
+    }
+    // per row, the first group in max-sequence order with a real entry wins; -1 means the row is
+    // a placeholder in every group
+    std::vector group_choice(row_count, -1);
+    for (int64_t r = 0; r < row_count; r++) {
+        for (size_t g = 0; g < groups_.size(); g++) {
+            if (!placeholder_flags[g][r]) {
+                group_choice[r] = static_cast(g);
+                break;
+            }
+        }
+    }
+
+    arrow::ArrayVector columns;
+    columns.reserve(read_schema_->num_fields());
+    for (int32_t field_idx = 0; field_idx < read_schema_->num_fields(); field_idx++) {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr column,
+                               AssembleColumn(field_idx, group_choice, group_chunks));
+        columns.push_back(std::move(column));
+    }
+    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array,
+                                      arrow::StructArray::Make(columns, read_schema_->fields()));
+    std::unique_ptr c_array = std::make_unique();
+    std::unique_ptr c_schema = std::make_unique();
+    PAIMON_RETURN_NOT_OK_FROM_ARROW(
+        arrow::ExportArray(*target_array, c_array.get(), c_schema.get()));
+    return std::make_pair(std::move(c_array), std::move(c_schema));
+}
+
+Result BlobFallbackBatchReader::NextBatchWithBitmap() {
+    PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, NextBatch());
+    if (BatchReader::IsEofBatch(batch)) {
+        return BatchReader::MakeEofBatchWithBitmap();
+    }
+    return ReaderUtils::AddAllValidBitmap(std::move(batch));
+}
+
+void BlobFallbackBatchReader::Close() {
+    for (auto& group : groups_) {
+        group.pending.clear();
+        for (auto& segment : group.segments) {
+            if (segment.reader) {
+                segment.reader->Close();
+            }
+        }
+    }
+    closed_ = true;
+}
+
+std::shared_ptr BlobFallbackBatchReader::GetReaderMetrics() const {
+    auto metrics = std::make_shared();
+    for (const auto& group : groups_) {
+        for (const auto& segment : group.segments) {
+            if (segment.reader) {
+                auto reader_metrics = segment.reader->GetReaderMetrics();
+                if (reader_metrics) {
+                    metrics->Merge(reader_metrics);
+                }
+            }
+        }
+    }
+    return metrics;
+}
+
+}  // namespace paimon
diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.h b/src/paimon/common/reader/blob_fallback_batch_reader.h
new file mode 100644
index 00000000..e265c280
--- /dev/null
+++ b/src/paimon/common/reader/blob_fallback_batch_reader.h
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+#include 
+#include 
+
+#include "arrow/type.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/reader/batch_reader.h"
+#include "paimon/reader/file_batch_reader.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+#include "paimon/utils/range.h"
+
+namespace arrow {
+class Array;
+class StructArray;
+}  // namespace arrow
+
+namespace paimon {
+
+/// Merges the blob files of one data-evolution blob bunch that span multiple max sequence
+/// number layers, resolving placeholder entries row by row. Aligned with Java's
+/// BlobFallbackRecordReader / AllPlaceholdersRecordReader.
+///
+/// A data-evolution partial update rewrites only the touched rows of a blob column; the new blob
+/// file records every untouched row as a placeholder entry. Reading therefore needs, per row, the
+/// value from the newest layer that holds a real (non-placeholder) entry:
+///
+/// 1. The caller groups the blob files by max sequence number (one group per layer, newest
+///    first) and, inside each group, orders them by first row id. Row id ranges the group's
+///    files do not cover are represented by gap segments, which stand for all-placeholder rows.
+/// 2. All groups span the same overall row id range, so with the same row-ranges selection
+///    applied they yield the same number of rows and can be stepped in lockstep.
+/// 3. Each output row takes the first group, in max-sequence order, whose row is not a
+///    placeholder. Placeholder rows are identified by exact equality with the
+///    BlobDefs::kPlaceholderSentinel bytes, emitted by the blob format reader when
+///    BlobDefs::kEmitPlaceholderSentinelKey is set.
+/// 4. A row that is a placeholder in every group degrades to a null blob: it keeps its
+///    _ROW_ID, reports -1 as its _SEQUENCE_NUMBER, and returns null for every other field.
+class BlobFallbackBatchReader : public BatchReader {
+ public:
+    /// One piece of a sequence group: either a reader over a single blob file (already wrapped
+    /// with the usual per-file mapping readers and row selection), or a virtual gap standing for
+    /// row ids the group's files do not cover. Segments must be ordered by ascending row id.
+    struct Segment {
+        /// File segment: emits the rows of one blob file. Null for a gap segment.
+        std::unique_ptr reader;
+        /// Gap segment only: the selected row ids the gap emits (all placeholders), as sorted
+        /// disjoint ranges. Must not be empty for a gap segment.
+        std::vector gap_selected_ranges;
+    };
+
+    /// `sequence_groups` must be ordered by descending max sequence number and contain at least
+    /// two groups (a single group needs no fallback). `read_schema` is the schema every file
+    /// reader emits; it must contain exactly one blob field and may additionally contain the
+    /// row-tracking fields _ROW_ID and _SEQUENCE_NUMBER (completed per file by
+    /// CompleteRowTrackingFieldsBatchReader), which stay correct for rows that are a
+    /// placeholder in every layer.
+    static Result> Create(
+        std::vector>&& sequence_groups,
+        const std::shared_ptr& read_schema, int32_t read_batch_size,
+        const std::shared_ptr& pool);
+
+    Result NextBatch() override;
+
+    Result NextBatchWithBitmap() override;
+
+    void Close() override;
+
+    std::shared_ptr GetReaderMetrics() const override;
+
+ private:
+    /// A run of consecutive rows already fetched from one group: `array` rows
+    /// [offset, offset + length) for a file segment, or `length` placeholder rows for a gap
+    /// segment (array is null).
+    struct Chunk {
+        std::shared_ptr array;
+        int64_t offset = 0;
+        int64_t length = 0;
+        /// Gap chunk only: the row id of each of the `length` rows, filled when the read
+        /// schema contains _ROW_ID so all-placeholder rows can keep their row id.
+        std::vector gap_row_ids;
+    };
+
+    /// Read progress of one sequence group. Move-only, matching Segment's unique_ptr member.
+    struct GroupCursor {
+        GroupCursor() = default;
+        GroupCursor(const GroupCursor&) = delete;
+        GroupCursor& operator=(const GroupCursor&) = delete;
+        GroupCursor(GroupCursor&&) = default;
+        GroupCursor& operator=(GroupCursor&&) = default;
+
+        std::vector segments;
+        size_t segment_idx = 0;
+        /// Position inside the current gap segment: index into gap_selected_ranges and the
+        /// number of rows already emitted from that range.
+        size_t gap_range_idx = 0;
+        int64_t gap_range_pos = 0;
+        /// Rows fetched from the current file segment but not yet consumed.
+        std::deque> pending;
+        int64_t pending_pos = 0;
+    };
+
+    BlobFallbackBatchReader(std::vector&& groups,
+                            const std::shared_ptr& read_schema,
+                            int32_t blob_field_idx, int32_t row_id_field_idx,
+                            int32_t seq_num_field_idx, int32_t read_batch_size,
+                            const std::shared_ptr& pool);
+
+    /// Collects up to `want` rows from the group into chunks. Only the first group may come up
+    /// short (which defines the window size); any later group ending early is a misalignment.
+    Result FillWindow(size_t group_idx, int64_t want, std::vector* chunks);
+
+    /// Flags each of the `row_count` window rows of the given chunks as placeholder or not.
+    Result> ComputePlaceholderFlags(const std::vector& chunks,
+                                                      int64_t row_count) const;
+
+    /// Assembles one output column by stitching, per run of rows choosing the same group,
+    /// slices of that group's chunks. For rows choosing no group (placeholder in every layer),
+    /// _ROW_ID is kept, _SEQUENCE_NUMBER becomes -1, and every other field becomes null.
+    Result> AssembleColumn(
+        int32_t field_idx, const std::vector& group_choice,
+        const std::vector>& group_chunks) const;
+
+    /// Assembles the _ROW_ID values of rows [run_start, run_end) from the given group's chunks;
+    /// gap chunks contribute their synthesized row ids.
+    Result> AssembleRowIdRun(const std::vector& chunks,
+                                                           int64_t run_start,
+                                                           int64_t run_end) const;
+
+    std::vector groups_;
+    std::shared_ptr read_schema_;
+    const int32_t blob_field_idx_;
+    /// Index of _ROW_ID / _SEQUENCE_NUMBER in the read schema, -1 when not read.
+    const int32_t row_id_field_idx_;
+    const int32_t seq_num_field_idx_;
+    const int32_t read_batch_size_;
+    std::shared_ptr arrow_pool_;
+    bool closed_ = false;
+};
+
+}  // namespace paimon
diff --git a/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp
new file mode 100644
index 00000000..29dd754c
--- /dev/null
+++ b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/common/reader/blob_fallback_batch_reader.h"
+
+#include 
+#include 
+#include 
+
+#include "arrow/api.h"
+#include "arrow/util/range.h"
+#include "gtest/gtest.h"
+#include "paimon/common/data/blob_defs.h"
+#include "paimon/common/data/blob_utils.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/memory/memory_pool.h"
+#include "paimon/testing/mock/mock_file_batch_reader.h"
+#include "paimon/testing/utils/read_result_collector.h"
+#include "paimon/testing/utils/testharness.h"
+
+namespace paimon::test {
+
+/// "PH" stands for a placeholder row (the sentinel bytes emitted by the placeholder-aware blob
+/// reader), std::nullopt for null.
+using BlobRows = std::vector>;
+
+class BlobFallbackBatchReaderTest : public ::testing::Test {
+ public:
+    void SetUp() override {
+        pool_ = GetDefaultPool();
+        struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)});
+        read_schema_ = arrow::schema(struct_type_->fields());
+    }
+
+    static std::string Sentinel() {
+        return std::string(BlobDefs::PlaceholderSentinelView());
+    }
+
+    std::shared_ptr MakeBlobStruct(const BlobRows& rows) const {
+        arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(),
+                                            {std::make_shared()});
+        auto blob_builder =
+            static_cast(struct_builder.field_builder(0));
+        for (const auto& row : rows) {
+            EXPECT_TRUE(struct_builder.Append().ok());
+            if (!row) {
+                EXPECT_TRUE(blob_builder->AppendNull().ok());
+            } else if (*row == "PH") {
+                std::string sentinel = Sentinel();
+                EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok());
+            } else {
+                EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok());
+            }
+        }
+        std::shared_ptr array;
+        EXPECT_TRUE(struct_builder.Finish(&array).ok());
+        return array;
+    }
+
+    /// One segment of a group: File(rows) for a file segment, Gap(n) for a placeholder gap
+    /// of n selected rows (with synthetic row ids when the schema does not read them, or the
+    /// given ranges via GapRanges).
+    struct SegmentSpec {
+        std::vector gap_ranges;
+        std::optional file_rows;
+        static SegmentSpec Gap(int64_t rows) {
+            return SegmentSpec{{Range(0, rows - 1)}, std::nullopt};
+        }
+        static SegmentSpec GapRanges(std::vector ranges) {
+            return SegmentSpec{std::move(ranges), std::nullopt};
+        }
+        static SegmentSpec File(BlobRows rows) {
+            return SegmentSpec{{}, std::move(rows)};
+        }
+    };
+
+    std::vector MakeGroup(const std::vector& specs,
+                                                            int32_t file_batch_size) const {
+        std::vector segments;
+        for (const auto& spec : specs) {
+            if (spec.file_rows) {
+                auto reader = std::make_unique(MakeBlobStruct(*spec.file_rows),
+                                                                    struct_type_, file_batch_size);
+                segments.push_back(BlobFallbackBatchReader::Segment{std::move(reader), {}});
+            } else {
+                segments.push_back(BlobFallbackBatchReader::Segment{nullptr, spec.gap_ranges});
+            }
+        }
+        return segments;
+    }
+
+    /// Runs the fallback over the groups with several batch sizes and compares to expected rows.
+    void CheckFallback(const std::vector>& group_specs,
+                       const BlobRows& expected_rows) const {
+        auto expected_array = MakeBlobStruct(expected_rows);
+        for (auto batch_size : arrow::internal::Iota(1, 8)) {
+            for (auto file_batch_size : {1, 3, 1024}) {
+                std::vector> groups;
+                groups.reserve(group_specs.size());
+                for (const auto& specs : group_specs) {
+                    groups.push_back(MakeGroup(specs, file_batch_size));
+                }
+                ASSERT_OK_AND_ASSIGN(
+                    auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_,
+                                                                 batch_size, pool_));
+                ASSERT_OK_AND_ASSIGN(
+                    auto result, paimon::test::ReadResultCollector::CollectResult(reader.get()));
+                reader->Close();
+                auto expected_chunk_array = std::make_shared(expected_array);
+                ASSERT_TRUE(result->Equals(expected_chunk_array))
+                    << "batch_size=" << batch_size << " file_batch_size=" << file_batch_size
+                    << "\nresult: " << result->ToString()
+                    << "\nexpected: " << expected_chunk_array->ToString();
+            }
+        }
+    }
+
+ protected:
+    std::shared_ptr pool_;
+    std::shared_ptr struct_type_;
+    std::shared_ptr read_schema_;
+};
+
+TEST_F(BlobFallbackBatchReaderTest, TestBasicFallback) {
+    // newer layer updates row 1 only; rows 0 and 2 fall back to the older layer
+    CheckFallback(
+        {{SegmentSpec::File({"PH", "u1", "PH"})}, {SegmentSpec::File({"b0", "b1", "b2"})}},
+        {"b0", "u1", "b2"});
+}
+
+TEST_F(BlobFallbackBatchReaderTest, TestGapPadding) {
+    // the newer layer only covers rows 2-3; the gaps stand for placeholders
+    CheckFallback({{SegmentSpec::Gap(2), SegmentSpec::File({"u2", "PH"})},
+                   {SegmentSpec::File({"b0", "b1", "b2", "b3"})}},
+                  {"b0", "b1", "u2", "b3"});
+    // trailing gap
+    CheckFallback({{SegmentSpec::File({"PH", "u1"}), SegmentSpec::Gap(2)},
+                   {SegmentSpec::File({"b0", "b1", "b2", "b3"})}},
+                  {"b0", "u1", "b2", "b3"});
+    // middle gap between two files of one layer
+    CheckFallback({{SegmentSpec::File({"u0"}), SegmentSpec::Gap(2), SegmentSpec::File({"u3"})},
+                   {SegmentSpec::File({"b0", "b1", "b2", "b3"})}},
+                  {"u0", "b1", "b2", "u3"});
+    // a gap segment covering multiple disjoint selected ranges
+    CheckFallback({{SegmentSpec::GapRanges({Range(0, 0), Range(2, 2)}), SegmentSpec::File({"u3"})},
+                   {SegmentSpec::File({"b0", "b2", "b3"})}},
+                  {"b0", "b2", "u3"});
+}
+
+TEST_F(BlobFallbackBatchReaderTest, TestAllPlaceholdersBecomesNull) {
+    // a row that is a placeholder in every layer degrades to null
+    CheckFallback({{SegmentSpec::File({"PH", "PH"})}, {SegmentSpec::File({"b0", "PH"})}},
+                  {"b0", std::nullopt});
+    CheckFallback({{SegmentSpec::Gap(2)}, {SegmentSpec::File({"PH", "PH"})}},
+                  {std::nullopt, std::nullopt});
+}
+
+TEST_F(BlobFallbackBatchReaderTest, TestNullIsNotPlaceholder) {
+    // a real null in a newer layer wins: null means "updated to null", not "not updated"
+    CheckFallback({{SegmentSpec::File({std::nullopt, "u1"})}, {SegmentSpec::File({"b0", "b1"})}},
+                  {std::nullopt, "u1"});
+}
+
+TEST_F(BlobFallbackBatchReaderTest, TestSentinelPrefixedValueIsNotPlaceholder) {
+    // placeholders are identified by exact equality with the sentinel bytes only: a real value
+    // that merely starts with them passes through unchanged, whether it falls back or wins as
+    // the newest layer
+    std::string prefixed = Sentinel() + "suffix";
+    CheckFallback({{SegmentSpec::File({"PH", "u1"})}, {SegmentSpec::File({prefixed, "b1"})}},
+                  {prefixed, "u1"});
+    CheckFallback({{SegmentSpec::File({prefixed, "PH"})}, {SegmentSpec::File({"b0", "b1"})}},
+                  {prefixed, "b1"});
+}
+
+TEST_F(BlobFallbackBatchReaderTest, TestThreeLayers) {
+    CheckFallback({{SegmentSpec::File({"PH", "PH", "u2"})},
+                   {SegmentSpec::File({"PH", "m1", "PH"})},
+                   {SegmentSpec::File({"b0", "b1", "b2"})}},
+                  {"b0", "m1", "u2"});
+}
+
+TEST_F(BlobFallbackBatchReaderTest, TestLayeredFilesAndGaps) {
+    // mirrors the compacted-sequence-groups shape: layers partially cover [0, 9]
+    CheckFallback(
+        {{SegmentSpec::Gap(6), SegmentSpec::File({"u66", "PH"}), SegmentSpec::Gap(1),
+          SegmentSpec::File({"u69"})},
+         {SegmentSpec::File({"u40", "PH", "PH", "PH"}), SegmentSpec::Gap(4),
+          SegmentSpec::File({"u48", "PH"})},
+         {SegmentSpec::File({"b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9"})}},
+        {"u40", "b1", "b2", "b3", "b4", "b5", "u66", "b7", "u48", "u69"});
+}
+
+TEST_F(BlobFallbackBatchReaderTest, TestRowTrackingFieldsPreserved) {
+    // Row-tracking projections: resolved rows keep their layer's row id and sequence number;
+    // an all-placeholder row keeps its row id (here provided by the newest group's gap
+    // segment), reports -1 as its sequence number, and degrades the blob to null. Covers the
+    // schema variants {blob, _ROW_ID, _SEQUENCE_NUMBER}, {blob, _ROW_ID} and
+    // {blob, _SEQUENCE_NUMBER}.
+    struct RowSpec {
+        std::optional blob;
+        int64_t row_id;
+        int64_t seq_num;
+    };
+    for (bool with_row_id : {true, false}) {
+        for (bool with_seq_num : {true, false}) {
+            if (!with_row_id && !with_seq_num) {
+                continue;
+            }
+            arrow::FieldVector fields = {BlobUtils::ToArrowField("blob_col", true)};
+            if (with_row_id) {
+                fields.push_back(SpecialFields::RowId().field_);
+            }
+            if (with_seq_num) {
+                fields.push_back(SpecialFields::SequenceNumber().field_);
+            }
+            auto struct_type = arrow::struct_(fields);
+            auto schema = arrow::schema(fields);
+
+            auto make_rows = [&](const std::vector& rows) {
+                std::vector> field_builders = {
+                    std::make_shared()};
+                for (size_t i = 1; i < fields.size(); i++) {
+                    field_builders.push_back(std::make_shared());
+                }
+                arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(),
+                                                    std::move(field_builders));
+                auto blob_builder =
+                    static_cast(struct_builder.field_builder(0));
+                for (const auto& row : rows) {
+                    EXPECT_TRUE(struct_builder.Append().ok());
+                    if (!row.blob) {
+                        EXPECT_TRUE(blob_builder->AppendNull().ok());
+                    } else if (*row.blob == "PH") {
+                        std::string sentinel = Sentinel();
+                        EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok());
+                    } else {
+                        EXPECT_TRUE(blob_builder->Append(row.blob->data(), row.blob->size()).ok());
+                    }
+                    int32_t next_field = 1;
+                    if (with_row_id) {
+                        auto builder = static_cast(
+                            struct_builder.field_builder(next_field++));
+                        EXPECT_TRUE(builder->Append(row.row_id).ok());
+                    }
+                    if (with_seq_num) {
+                        auto builder = static_cast(
+                            struct_builder.field_builder(next_field));
+                        EXPECT_TRUE(builder->Append(row.seq_num).ok());
+                    }
+                }
+                std::shared_ptr array;
+                EXPECT_TRUE(struct_builder.Finish(&array).ok());
+                return array;
+            };
+
+            for (auto batch_size : arrow::internal::Iota(1, 5)) {
+                for (auto file_batch_size : {1, 1024}) {
+                    // newest layer (seq 20) covers only row 2; rows 0-1 are a gap
+                    std::vector newest;
+                    newest.push_back(BlobFallbackBatchReader::Segment{nullptr, {Range(0, 1)}});
+                    newest.push_back(BlobFallbackBatchReader::Segment{
+                        std::make_unique(make_rows({{"u2", 2, 20}}),
+                                                              struct_type, file_batch_size),
+                        {}});
+                    // oldest layer (seq 10) covers rows 0-2, row 1 is a placeholder there too
+                    std::vector oldest;
+                    oldest.push_back(BlobFallbackBatchReader::Segment{
+                        std::make_unique(
+                            make_rows({{"b0", 0, 10}, {"PH", 1, 10}, {"PH", 2, 10}}), struct_type,
+                            file_batch_size),
+                        {}});
+                    std::vector> groups;
+                    groups.push_back(std::move(newest));
+                    groups.push_back(std::move(oldest));
+
+                    ASSERT_OK_AND_ASSIGN(auto reader,
+                                         BlobFallbackBatchReader::Create(std::move(groups), schema,
+                                                                         batch_size, pool_));
+                    ASSERT_OK_AND_ASSIGN(
+                        auto result,
+                        paimon::test::ReadResultCollector::CollectResult(reader.get()));
+                    reader->Close();
+
+                    // row 0 falls back to seq 10, row 1 is all-placeholder (null blob, row id
+                    // kept, seq -1), row 2 takes seq 20
+                    auto expected_array =
+                        make_rows({{"b0", 0, 10}, {std::nullopt, 1, -1}, {"u2", 2, 20}});
+                    auto expected_chunk_array =
+                        std::make_shared(expected_array);
+                    ASSERT_TRUE(result->Equals(expected_chunk_array))
+                        << "with_row_id=" << with_row_id << " with_seq_num=" << with_seq_num
+                        << " batch_size=" << batch_size << " file_batch_size=" << file_batch_size
+                        << "\nresult: " << result->ToString()
+                        << "\nexpected: " << expected_chunk_array->ToString();
+                }
+            }
+        }
+    }
+}
+
+TEST_F(BlobFallbackBatchReaderTest, TestMisalignedGroupsFail) {
+    std::vector> groups;
+    groups.push_back(MakeGroup({SegmentSpec::File({"PH", "u1", "PH"})}, 1024));
+    groups.push_back(MakeGroup({SegmentSpec::File({"b0", "b1"})}, 1024));
+    ASSERT_OK_AND_ASSIGN(
+        auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, 1024, pool_));
+    ASSERT_NOK_WITH_MSG(reader->NextBatch(), "same number of rows");
+}
+
+TEST_F(BlobFallbackBatchReaderTest, TestCreateValidation) {
+    // a single group needs no fallback
+    std::vector> single_group;
+    single_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024));
+    ASSERT_NOK_WITH_MSG(
+        BlobFallbackBatchReader::Create(std::move(single_group), read_schema_, 1024, pool_),
+        "at least two sequence groups");
+
+    // the read schema must contain a blob field
+    std::vector> groups;
+    groups.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024));
+    groups.push_back(MakeGroup({SegmentSpec::File({"b1"})}, 1024));
+    auto plain_schema =
+        arrow::schema({arrow::field("not_blob", arrow::large_binary(), /*nullable=*/true)});
+    ASSERT_NOK_WITH_MSG(
+        BlobFallbackBatchReader::Create(std::move(groups), plain_schema, 1024, pool_),
+        "should contain a blob field");
+
+    // groups must not be empty
+    std::vector> with_empty_group;
+    with_empty_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024));
+    with_empty_group.emplace_back();
+    ASSERT_NOK_WITH_MSG(
+        BlobFallbackBatchReader::Create(std::move(with_empty_group), read_schema_, 1024, pool_),
+        "should not be empty");
+
+    // a gap segment must cover at least one selected row id
+    std::vector> with_empty_gap;
+    with_empty_gap.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024));
+    with_empty_gap.push_back(MakeGroup({SegmentSpec::GapRanges({})}, 1024));
+    ASSERT_NOK_WITH_MSG(
+        BlobFallbackBatchReader::Create(std::move(with_empty_gap), read_schema_, 1024, pool_),
+        "at least one selected row id");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp
index 0bba204b..296a56c6 100644
--- a/src/paimon/common/reader/data_evolution_file_reader.cpp
+++ b/src/paimon/common/reader/data_evolution_file_reader.cpp
@@ -44,8 +44,13 @@ Result> DataEvolutionFileReader::Create
         return Status::Invalid(
             "read schema, row offsets and field offsets must have the same size");
     }
-    if (readers.size() <= 1) {
-        return Status::Invalid("readers size is supposed to be more than 1");
+    if (readers.empty()) {
+        return Status::Invalid("readers must not be empty");
+    }
+    for (int32_t reader_offset : reader_offsets) {
+        if (reader_offset >= static_cast(readers.size())) {
+            return Status::Invalid("reader offset is out of range of readers");
+        }
     }
     return std::unique_ptr(
         new DataEvolutionFileReader(std::move(readers), read_schema, read_batch_size,
diff --git a/src/paimon/common/reader/data_evolution_file_reader.h b/src/paimon/common/reader/data_evolution_file_reader.h
index e1bea66e..76494358 100644
--- a/src/paimon/common/reader/data_evolution_file_reader.h
+++ b/src/paimon/common/reader/data_evolution_file_reader.h
@@ -29,10 +29,12 @@
 #include "paimon/result.h"
 
 namespace paimon {
-/// This is a union reader which contains multiple inner readers.
+/// This is a union reader which contains one or more inner readers.
 ///
 /// This reader, assembling multiple reader into one big and great reader. The row it produces
-/// also come from the readers it contains.
+/// also come from the readers it contains. A single inner reader is a valid degenerate case
+/// (e.g. a blob-only write): the union still maps its fields into the read schema order and
+/// null fills unmatched read fields.
 ///
 /// For example, the expected schema for this reader is : int, int, string, int, string, int.(Total
 /// 6 fields) It contains three inner readers, we call them reader0, reader1 and reader2.
diff --git a/src/paimon/common/reader/data_evolution_file_reader_test.cpp b/src/paimon/common/reader/data_evolution_file_reader_test.cpp
index 5f7e3a07..9e545c54 100644
--- a/src/paimon/common/reader/data_evolution_file_reader_test.cpp
+++ b/src/paimon/common/reader/data_evolution_file_reader_test.cpp
@@ -166,6 +166,20 @@ TEST_F(DataEvolutionFileReaderTest, TestInvalid) {
                                                             reader_offsets, field_offsets, pool_),
                             "read schema, row offsets and field offsets must have the same size");
     }
+    {
+        arrow::FieldVector read_fields = {
+            arrow::field("f0", arrow::int32()),
+            arrow::field("f1", arrow::int32()),
+            arrow::field("f2", arrow::utf8()),
+            arrow::field("f3", arrow::int32()),
+        };
+        auto read_schema = arrow::schema(read_fields);
+        std::vector reader_offsets = {0, 0, 1, 1};
+        std::vector field_offsets = {0, 1, 1, 0};
+        ASSERT_NOK_WITH_MSG(DataEvolutionFileReader::Create({}, read_schema, /*read_batch_size=*/10,
+                                                            reader_offsets, field_offsets, pool_),
+                            "readers must not be empty");
+    }
     {
         std::vector> readers;
         readers.push_back(nullptr);
@@ -182,7 +196,7 @@ TEST_F(DataEvolutionFileReaderTest, TestInvalid) {
         ASSERT_NOK_WITH_MSG(
             DataEvolutionFileReader::Create(std::move(readers), read_schema, /*read_batch_size=*/10,
                                             reader_offsets, field_offsets, pool_),
-            "readers size is supposed to be more than 1");
+            "reader offset is out of range of readers");
     }
 }
 
diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp
index e6d6de79..feea29c5 100644
--- a/src/paimon/core/append/append_only_writer.cpp
+++ b/src/paimon/core/append/append_only_writer.cpp
@@ -213,11 +213,11 @@ Result AppendOnlyWriter::GetDataFileWriterFacto
 
 AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetBlobFileWriterFactory(
     const std::shared_ptr& single_field_schema,
-    const std::optional>& write_cols) const {
+    const std::optional>& write_cols, bool write_placeholder) const {
     std::shared_ptr path_factory = path_factory_;
     return std::make_shared(options_, schema_id_, single_field_schema,
                                                        write_cols, seq_num_counter_, path_factory,
-                                                       memory_pool_);
+                                                       write_placeholder, memory_pool_);
 }
 
 AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWriter(
@@ -225,8 +225,16 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri
     // Multiple blob fields are supported. Each blob field gets its own rolling file writer
     // via MultipleBlobFileWriter.
     auto blob_schema = schemas.blob_schema;
+    // A data-evolution write touching only blob columns is a partial update: its rows may mark
+    // untouched blobs with the placeholder sentinel (see BlobDefs::kWritePlaceholderKey). Any
+    // write that also carries non-blob columns stores blob bytes verbatim. The gate cannot tell
+    // a first write from an update, so a blob-only first write also runs under the sentinel
+    // channel: a user value equal to the sentinel is then persisted as a placeholder entry with
+    // no older layer to resolve it, and reading fails loudly rather than returning wrong bytes.
+    bool write_placeholder =
+        options_.DataEvolutionEnabled() && schemas.main_schema->num_fields() == 0;
     MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator =
-        [this, blob_schema](const std::string& blob_field_name)
+        [this, blob_schema, write_placeholder](const std::string& blob_field_name)
         -> Result<
             std::unique_ptr>>> {
         // Create a single-field schema for this blob field
@@ -238,7 +246,7 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri
         auto single_field_schema = arrow::schema({field});
         std::vector write_cols = {blob_field_name};
         auto single_blob_file_writer_factory =
-            GetBlobFileWriterFactory(single_field_schema, write_cols);
+            GetBlobFileWriterFactory(single_field_schema, write_cols, write_placeholder);
         return std::make_unique>>(
             options_.GetBlobTargetFileSize(),
             /*target_file_row_num=*/std::numeric_limits::max(),
diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h
index eb4c2de6..2bfb1a53 100644
--- a/src/paimon/core/append/append_only_writer.h
+++ b/src/paimon/core/append/append_only_writer.h
@@ -110,7 +110,7 @@ class AppendOnlyWriter : public BatchWriter {
 
     WriterFactory GetBlobFileWriterFactory(
         const std::shared_ptr& single_field_schema,
-        const std::optional>& write_cols) const;
+        const std::optional>& write_cols, bool write_placeholder) const;
 
     Status TrySyncLatestCompaction(bool blocking);
     Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file);
diff --git a/src/paimon/core/io/blob_data_file_writer_factory.cpp b/src/paimon/core/io/blob_data_file_writer_factory.cpp
index 78d740d7..b6d9346c 100644
--- a/src/paimon/core/io/blob_data_file_writer_factory.cpp
+++ b/src/paimon/core/io/blob_data_file_writer_factory.cpp
@@ -20,8 +20,11 @@
 #include "paimon/core/io/blob_data_file_writer_factory.h"
 
 #include 
+#include 
+#include 
 #include 
 
+#include "paimon/common/data/blob_defs.h"
 #include "paimon/core/core_options.h"
 #include "paimon/core/io/data_file_path_factory.h"
 #include "paimon/core/manifest/file_source.h"
@@ -36,20 +39,28 @@ BlobDataFileWriterFactory::BlobDataFileWriterFactory(
     const std::shared_ptr& file_schema,
     const std::optional>& write_cols,
     const std::shared_ptr& seq_num_counter,
-    const std::shared_ptr& path_factory,
+    const std::shared_ptr& path_factory, bool write_placeholder,
     const std::shared_ptr& pool)
     : DataFileWriterFactory(options, schema_id, pool),
       file_schema_(file_schema),
       write_cols_(write_cols),
       seq_num_counter_(seq_num_counter),
-      path_factory_(path_factory) {}
+      path_factory_(path_factory),
+      write_placeholder_(write_placeholder) {}
 
 Result>>>
 BlobDataFileWriterFactory::CreateWriter() const {
     std::shared_ptr seq_num_counter =
         options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_;
+    std::map format_options = options_.ToMap();
+    // The placeholder channel is internal: strip user-supplied blob.internal.* table options so
+    // only the writer's own decision below can enable it.
+    BlobDefs::EraseInternalPlaceholderOptions(&format_options);
+    if (write_placeholder_) {
+        format_options[BlobDefs::kWritePlaceholderKey] = "true";
+    }
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format,
-                           FileFormatFactory::Get("blob", options_.ToMap()));
+                           FileFormatFactory::Get("blob", format_options));
     PAIMON_ASSIGN_OR_RAISE(WriterResources resources,
                            CreateWriterResources(*format, file_schema_,
                                                  /*create_stats_extractor=*/true));
diff --git a/src/paimon/core/io/blob_data_file_writer_factory.h b/src/paimon/core/io/blob_data_file_writer_factory.h
index 15286020..63c84d72 100644
--- a/src/paimon/core/io/blob_data_file_writer_factory.h
+++ b/src/paimon/core/io/blob_data_file_writer_factory.h
@@ -46,12 +46,15 @@ class BlobDataFileWriterFactory
     : public DataFileWriterFactory,
       public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> {
  public:
+    /// `write_placeholder` marks a data-evolution partial-update write: the blob format
+    /// writer is created with BlobDefs::kWritePlaceholderKey and persists placeholder
+    /// sentinel rows as placeholder entries.
     BlobDataFileWriterFactory(const CoreOptions& options, int64_t schema_id,
                               const std::shared_ptr& file_schema,
                               const std::optional>& write_cols,
                               const std::shared_ptr& seq_num_counter,
                               const std::shared_ptr& path_factory,
-                              const std::shared_ptr& pool);
+                              bool write_placeholder, const std::shared_ptr& pool);
 
     Result>>>
     CreateWriter() const override;
@@ -61,6 +64,7 @@ class BlobDataFileWriterFactory
     std::optional> write_cols_;
     std::shared_ptr seq_num_counter_;
     std::shared_ptr path_factory_;
+    bool write_placeholder_ = false;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp
index 93e93e9b..53749b83 100644
--- a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp
+++ b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp
@@ -28,34 +28,46 @@
 #include "paimon/common/table/special_fields.h"
 #include "paimon/common/utils/arrow/mem_utils.h"
 #include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/object_utils.h"
 
 namespace paimon {
 CompleteRowTrackingFieldsBatchReader::CompleteRowTrackingFieldsBatchReader(
     std::unique_ptr&& reader, const std::optional& first_row_id,
-    int64_t snapshot_id, const std::shared_ptr& pool)
+    int64_t snapshot_id, const std::optional>& file_field_names,
+    const std::shared_ptr& pool)
     : first_row_id_(first_row_id),
       snapshot_id_(snapshot_id),
+      file_field_names_(file_field_names),
       arrow_pool_(GetArrowPool(pool)),
       reader_(std::move(reader)) {}
 
 Status CompleteRowTrackingFieldsBatchReader::SetReadSchema(
     ::ArrowSchema* read_schema, const std::shared_ptr& predicate,
     const std::optional& selection_bitmap) {
-    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader_->GetFileSchema());
-    PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema,
-                                      arrow::ImportSchema(c_file_schema.get()));
+    // The physical fields of the file decide which special fields must be stripped from the
+    // format reader's schema: a format without a self-describing file schema (e.g. blob)
+    // declares them via file_field_names_, self-describing formats are queried directly.
+    std::vector file_field_names;
+    if (file_field_names_) {
+        file_field_names = file_field_names_.value();
+    } else {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema,
+                               reader_->GetFileSchema());
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema,
+                                          arrow::ImportSchema(c_file_schema.get()));
+        file_field_names = file_schema->field_names();
+    }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema,
                                       arrow::ImportSchema(read_schema));
     read_schema_ = arrow_schema;
     int32_t row_id_idx = arrow_schema->GetFieldIndex(SpecialFields::RowId().Name());
-    if (row_id_idx != -1 && file_schema->GetFieldIndex(SpecialFields::RowId().Name()) == -1) {
-        // read special fields but file not exist, remove special fields to format reader
+    if (row_id_idx != -1 &&
+        !ObjectUtils::Contains(file_field_names, SpecialFields::RowId().Name())) {
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow_schema, arrow_schema->RemoveField(row_id_idx));
     }
     int32_t sequence_id_idx = arrow_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name());
     if (sequence_id_idx != -1 &&
-        file_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()) == -1) {
-        // read special fields but file not exist, remove special fields to format reader
+        !ObjectUtils::Contains(file_field_names, SpecialFields::SequenceNumber().Name())) {
         PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(arrow_schema, arrow_schema->RemoveField(sequence_id_idx));
     }
     ArrowSchema c_schema;
diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.h b/src/paimon/core/io/complete_row_tracking_fields_reader.h
index 86b970d6..7f4d7ead 100644
--- a/src/paimon/core/io/complete_row_tracking_fields_reader.h
+++ b/src/paimon/core/io/complete_row_tracking_fields_reader.h
@@ -33,10 +33,13 @@ namespace paimon {
 // Precondition: read_schema has special fields
 class CompleteRowTrackingFieldsBatchReader : public FileBatchReader {
  public:
-    CompleteRowTrackingFieldsBatchReader(std::unique_ptr&& reader,
-                                         const std::optional& first_row_id,
-                                         int64_t snapshot_id,
-                                         const std::shared_ptr& pool);
+    /// `file_field_names` declares the physical fields of a file whose format has no
+    /// self-describing schema (e.g. blob); when nullopt the file schema is queried from the
+    /// inner reader via GetFileSchema().
+    CompleteRowTrackingFieldsBatchReader(
+        std::unique_ptr&& reader, const std::optional& first_row_id,
+        int64_t snapshot_id, const std::optional>& file_field_names,
+        const std::shared_ptr& pool);
 
     Result> GetFileSchema() const override {
         return Status::Invalid(
@@ -82,6 +85,7 @@ class CompleteRowTrackingFieldsBatchReader : public FileBatchReader {
  private:
     std::optional first_row_id_;
     int64_t snapshot_id_ = -1;
+    std::optional> file_field_names_;
     std::shared_ptr arrow_pool_;
     std::shared_ptr read_schema_;
     std::unique_ptr reader_;
diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp
index 19b8e800..b6800c63 100644
--- a/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp
+++ b/src/paimon/core/io/complete_row_tracking_fields_reader_test.cpp
@@ -19,7 +19,10 @@
 #include "paimon/core/io/complete_row_tracking_fields_reader.h"
 
 #include 
+#include 
+#include 
 #include 
+#include 
 
 #include "arrow/api.h"
 #include "arrow/array/array_base.h"
@@ -46,7 +49,8 @@ class CompleteRowTrackingFieldsBatchReaderTest : public testing::Test {
                 std::make_unique(src_array, src_array->type(), batch_size);
             auto complete_row_tracking_fields_reader =
                 std::make_shared(
-                    std::move(file_batch_reader), first_row_id, snapshot_id, pool_);
+                    std::move(file_batch_reader), first_row_id, snapshot_id,
+                    /*file_field_names=*/std::nullopt, pool_);
             ArrowSchema c_read_schema;
             ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok());
             ASSERT_OK(complete_row_tracking_fields_reader->SetReadSchema(
@@ -65,12 +69,14 @@ class CompleteRowTrackingFieldsBatchReaderTest : public testing::Test {
     void CheckSetReadSchema(
         const std::shared_ptr& file_schema,
         const std::shared_ptr& read_schema,
-        const std::shared_ptr& expected_schema_for_inner_reader) const {
+        const std::shared_ptr& expected_schema_for_inner_reader,
+        const std::optional>& file_field_names = std::nullopt) const {
         auto file_batch_reader = std::make_unique(
             /*data=*/nullptr, arrow::struct_(file_schema->fields()), /*read_batch_size=*/1);
         auto complete_row_tracking_fields_reader =
             std::make_shared(
-                std::move(file_batch_reader), /*first_row_id=*/10, /*snapshot_id=*/1, pool_);
+                std::move(file_batch_reader), /*first_row_id=*/10, /*snapshot_id=*/1,
+                file_field_names, pool_);
         ArrowSchema c_read_schema;
         ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok());
         ASSERT_OK(
@@ -132,6 +138,33 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestSetReadSchema) {
     }
 }
 
+TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestSetReadSchemaWithDeclaredFileFields) {
+    arrow::FieldVector fields = {
+        arrow::field("f0", arrow::int32()),
+        arrow::field("f1", arrow::int32()),
+        arrow::field("_ROW_ID", arrow::int64()),
+        arrow::field("_SEQUENCE_NUMBER", arrow::int64()),
+    };
+    {
+        // declared physical fields take precedence over the inner reader's self-described
+        // schema: the special fields are stripped even though the mock's file schema
+        // contains them
+        auto file_schema = arrow::schema(fields);
+        auto read_schema = arrow::schema(fields);
+        auto expected_schema_for_inner_reader = arrow::schema({fields[0], fields[1]});
+        CheckSetReadSchema(file_schema, read_schema, expected_schema_for_inner_reader,
+                           std::vector{"f0", "f1"});
+    }
+    {
+        // declared physical fields containing the special fields keep them for the inner reader
+        auto file_schema = arrow::schema({fields[0], fields[1]});
+        auto read_schema = arrow::schema(fields);
+        auto expected_schema_for_inner_reader = read_schema;
+        CheckSetReadSchema(file_schema, read_schema, expected_schema_for_inner_reader,
+                           std::vector{"f0", "f1", "_ROW_ID", "_SEQUENCE_NUMBER"});
+    }
+}
+
 TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestWithNoRowTrackingFields) {
     arrow::FieldVector fields = {
         arrow::field("f0", arrow::int32()),
@@ -344,7 +377,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidWithReadNonExistFiel
 
     auto complete_row_tracking_fields_reader =
         std::make_shared(
-            std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, pool_);
+            std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8,
+            /*file_field_names=*/std::nullopt, pool_);
     ArrowSchema c_read_schema;
     ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok());
     ASSERT_OK(
@@ -380,7 +414,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidNextBatchBeforeSetRe
 
     auto complete_row_tracking_fields_reader =
         std::make_shared(
-            std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8, pool_);
+            std::move(file_batch_reader), /*first_row_id=*/100, /*snapshot_id=*/8,
+            /*file_field_names=*/std::nullopt, pool_);
     ASSERT_NOK_WITH_MSG(complete_row_tracking_fields_reader->NextBatchWithBitmap(),
                         "in CompleteRowTrackingFieldsBatchReader SetReadSchema is supposed to be "
                         "called before NextBatch");
@@ -410,7 +445,8 @@ TEST_F(CompleteRowTrackingFieldsBatchReaderTest, TestInvalidNullFirstRowId) {
 
     auto complete_row_tracking_fields_reader =
         std::make_shared(
-            std::move(file_batch_reader), /*first_row_id=*/std::nullopt, /*snapshot_id=*/8, pool_);
+            std::move(file_batch_reader), /*first_row_id=*/std::nullopt, /*snapshot_id=*/8,
+            /*file_field_names=*/std::nullopt, pool_);
     ArrowSchema c_read_schema;
     ASSERT_TRUE(arrow::ExportSchema(*read_schema, &c_read_schema).ok());
     ASSERT_OK(
diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp
index 92eca73a..51bb5651 100644
--- a/src/paimon/core/mergetree/lookup_levels.cpp
+++ b/src/paimon/core/mergetree/lookup_levels.cpp
@@ -347,7 +347,8 @@ Status LookupLevels::CreateSstFileFromDataFile(const std::shared_ptr> raw_readers,
         split_read_->CreateRawFileReaders(partition_, {file}, read_schema_,
                                           /*predicate=*/nullptr, dv_factory_,
-                                          /*row_ranges=*/std::nullopt, data_file_path_factory_));
+                                          /*row_ranges=*/std::nullopt, data_file_path_factory_,
+                                          /*extra_format_options=*/{}));
     if (raw_readers.size() != 1) {
         return Status::Invalid("Unexpected, CreateSstFileFromDataFile only create single reader");
     }
diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp
index 2296013a..0850d943 100644
--- a/src/paimon/core/operation/abstract_split_read.cpp
+++ b/src/paimon/core/operation/abstract_split_read.cpp
@@ -26,6 +26,7 @@
 
 #include "arrow/type.h"
 #include "fmt/format.h"
+#include "paimon/common/data/blob_defs.h"
 #include "paimon/common/data/blob_utils.h"
 #include "paimon/common/data/shredding/map_shared_shredding_file_reader.h"
 #include "paimon/common/data/shredding/map_shared_shredding_utils.h"
@@ -77,7 +78,8 @@ Result>> AbstractSplitRead::CreateR
     const BinaryRow& partition, const std::vector>& data_files,
     const std::shared_ptr& read_schema, const std::shared_ptr& predicate,
     DeletionVector::Factory dv_factory, const std::optional>& row_ranges,
-    const std::shared_ptr& data_file_path_factory) const {
+    const std::shared_ptr& data_file_path_factory,
+    const std::map& extra_format_options) const {
     if (data_files.empty()) {
         return std::vector>();
     }
@@ -91,7 +93,7 @@ Result>> AbstractSplitRead::CreateR
         auto data_file_path = data_file_path_factory->ToPath(file);
         PAIMON_ASSIGN_OR_RAISE(std::string data_file_identifier, file->FileFormat());
         PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder,
-                               PrepareReaderBuilder(data_file_identifier));
+                               PrepareReaderBuilder(data_file_identifier, extra_format_options));
         PAIMON_ASSIGN_OR_RAISE(
             std::unique_ptr file_reader,
             CreateFieldMappingReader(data_file_path, file, partition, reader_builder.get(),
@@ -122,9 +124,17 @@ Result> AbstractSplitRead::ApplyPredicateFilterIfNe
 }
 
 Result> AbstractSplitRead::PrepareReaderBuilder(
-    const std::string& format_identifier) const {
+    const std::string& format_identifier,
+    const std::map& extra_format_options) const {
+    std::map format_options = options_.ToMap();
+    // The blob placeholder channels are internal: strip user-supplied blob.internal.* table
+    // options so only the internal read path can enable them through extra_format_options.
+    BlobDefs::EraseInternalPlaceholderOptions(&format_options);
+    for (const auto& [key, value] : extra_format_options) {
+        format_options[key] = value;
+    }
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_format,
-                           FileFormatFactory::Get(format_identifier, options_.ToMap()));
+                           FileFormatFactory::Get(format_identifier, format_options));
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder,
                            file_format->CreateReaderBuilder(options_.GetReadBatchSize()));
     reader_builder->WithMemoryPool(pool_);
@@ -208,8 +218,15 @@ Result> AbstractSplitRead::CreateFieldMappingRe
             file_reader, ApplyVariantShreddingReaderIfNeeded(std::move(file_reader), read_schema));
     }
     if (NeedCompleteRowTrackingFields(options_.RowTrackingEnabled(), read_schema)) {
+        // A blob file has no self-describing schema: its physical fields are declared by the
+        // file meta's write cols instead of queried from the format reader.
+        std::optional> file_field_names;
+        if (file_format_identifier == "blob") {
+            file_field_names = file_meta->write_cols;
+        }
         file_reader = std::make_unique(
-            std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number, pool_);
+            std::move(file_reader), file_meta->first_row_id, file_meta->max_sequence_number,
+            file_field_names, pool_);
     }
     const auto& predicate = field_mapping->non_partition_info.non_partition_filter;
     auto all_data_schema = DataField::ConvertDataFieldsToArrowSchema(data_schema->Fields());
diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h
index d39232a5..ea3f9070 100644
--- a/src/paimon/core/operation/abstract_split_read.h
+++ b/src/paimon/core/operation/abstract_split_read.h
@@ -18,6 +18,7 @@
 
 #pragma once
 
+#include 
 #include 
 #include 
 #include 
@@ -62,12 +63,16 @@ class AbstractSplitRead : public SplitRead {
  public:
     ~AbstractSplitRead() override = default;
 
+    /// `extra_format_options` are merged over the table options when building the format
+    /// reader, e.g. to switch the blob format reader into placeholder-aware mode for the
+    /// data-evolution blob fallback read path.
     Result>> CreateRawFileReaders(
         const BinaryRow& partition, const std::vector>& data_files,
         const std::shared_ptr& read_schema,
         const std::shared_ptr& predicate, DeletionVector::Factory dv_factory,
         const std::optional>& row_ranges,
-        const std::shared_ptr& data_file_path_factory) const;
+        const std::shared_ptr& data_file_path_factory,
+        const std::map& extra_format_options) const;
 
  protected:
     AbstractSplitRead(const std::shared_ptr& path_factory,
@@ -97,7 +102,8 @@ class AbstractSplitRead : public SplitRead {
 
  private:
     Result> PrepareReaderBuilder(
-        const std::string& format_identifier) const;
+        const std::string& format_identifier,
+        const std::map& extra_format_options) const;
 
     Result> CreateFileBatchReader(
         const std::string& file_format_identifier, const std::string& data_file_path,
diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp
index 5ebf6f56..033d3e07 100644
--- a/src/paimon/core/operation/data_evolution_split_read.cpp
+++ b/src/paimon/core/operation/data_evolution_split_read.cpp
@@ -18,6 +18,8 @@
 
 #include "paimon/core/operation/data_evolution_split_read.h"
 
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -30,10 +32,12 @@
 #include "arrow/array/array_nested.h"
 #include "arrow/c/bridge.h"
 #include "paimon/common/catalog/catalog_context.h"
+#include "paimon/common/data/blob_defs.h"
 #include "paimon/common/data/blob_utils.h"
 #include "paimon/common/data/blob_view_struct.h"
 #include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h"
 #include "paimon/common/global_index/complete_index_score_batch_reader.h"
+#include "paimon/common/reader/blob_fallback_batch_reader.h"
 #include "paimon/common/reader/blob_view_resolving_batch_reader.h"
 #include "paimon/common/reader/complete_row_kind_batch_reader.h"
 #include "paimon/common/reader/concat_batch_reader.h"
@@ -47,71 +51,61 @@
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/utils/blob_view_lookup.h"
 namespace paimon {
+int64_t DataEvolutionSplitRead::BlobBunch::RowCount() const {
+    if (files_.empty()) {
+        return 0;
+    }
+    if (!has_row_ids_selection_) {
+        // Add enforces the union range to be contiguous
+        return union_end_row_id_ - union_first_row_id_;
+    }
+    // with a row-ids selection the scan may have pruned files, leaving holes in the union
+    int64_t row_count = 0;
+    for (const auto& range : Range::SortAndMergeOverlap(ranges_, /*adjacent=*/true)) {
+        row_count += range.Count();
+    }
+    return row_count;
+}
+
 Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptr& file) {
     if (!BlobUtils::IsBlobFile(file->file_name)) {
         return Status::Invalid("Only blob file can be added to a blob bunch.");
     }
     PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId());
-    if (first_row_id == latest_first_row_id_) {
-        if (file->max_sequence_number >= latest_max_sequence_number_) {
-            return Status::Invalid(
-                "Blob file with same first row id should have decreasing sequence number.");
-        }
-        // for files with the same first row id, file with larger sequence_number will be chosen,
-        // other files will be skipped
-        return Status::OK();
-    }
     if (!files_.empty()) {
-        if (has_row_ids_selection_) {
-            // for the case:
-            // snapshot 1: blob0 [0, 9]
-            // snapshot 2: blob1 [0, 4] + blob2 [5, 9]
-            // when selected row id is {5}, only blob0 and blob2 is reserved in scan process, as
-            // blob1 has no intersect with {5}
-            // BlobBunch will first add blob0 [0, 9]
-            // then when it comes to blob2 [5, 9], blob0 will be removed as it has smaller sequence
-            // number
-            if (first_row_id < expected_next_first_row_id_) {
-                if (file->max_sequence_number > latest_max_sequence_number_) {
-                    row_count_ -= files_.back()->row_count;
-                    files_.pop_back();
-                } else {
-                    return Status::OK();
-                }
-            }
-        } else {
-            if (first_row_id < expected_next_first_row_id_) {
-                if (file->max_sequence_number >= latest_max_sequence_number_) {
-                    return Status::Invalid(
-                        "Blob file with overlapping row id should have decreasing sequence "
-                        "number.");
-                }
-                // for files with overlapping, if the file with smaller sequence_number is chosen,
-                // there will not exist file with larger sequence_number
-                return Status::OK();
-            } else if (first_row_id > expected_next_first_row_id_) {
-                return Status::Invalid(
-                    fmt::format("Blob file first row id should be continuous, expect {} but got {}",
-                                expected_next_first_row_id_, first_row_id));
-            }
-        }
-        if (!files_.empty()) {
-            // Blob files for the same field may span schema ids.
-            if (file->write_cols != files_[0]->write_cols) {
-                return Status::Invalid(
-                    "All files in a blob bunch should have the same write columns.");
-            }
+        // Blob files for the same field may span schema ids.
+        if (file->write_cols != files_[0]->write_cols) {
+            return Status::Invalid("All files in a blob bunch should have the same write columns.");
         }
     }
-    row_count_ += file->row_count;
-    if (row_count_ > expected_row_count_) {
+    // files sharing a max sequence number form one layer, whose row id ranges must be disjoint;
+    // overlaps across layers are the expected shape of partial updates and are kept for the
+    // row-level placeholder fallback
+    auto [layer_iter, layer_inserted] =
+        sequence_group_end_.try_emplace(file->max_sequence_number, 0);
+    if (!layer_inserted && first_row_id < layer_iter->second) {
+        return Status::Invalid(fmt::format(
+            "Blob files with the same max sequence number should not have overlapping row id "
+            "ranges: file {} (max sequence number {}) starts at row id {} before the previous "
+            "file's end {}",
+            file->file_name, file->max_sequence_number, first_row_id, layer_iter->second));
+    }
+    if (!has_row_ids_selection_ && !files_.empty() && first_row_id > union_end_row_id_) {
+        // a hole no layer covers cannot be aligned with the data files
         return Status::Invalid(
-            fmt::format("Blob files row count exceed the expect {}", expected_row_count_));
+            fmt::format("Blob file first row id should be continuous, expect {} but got {}",
+                        union_end_row_id_, first_row_id));
     }
+    int64_t end_row_id = first_row_id + file->row_count;
+    layer_iter->second = end_row_id;
+    union_first_row_id_ = std::min(union_first_row_id_, first_row_id);
+    union_end_row_id_ = std::max(union_end_row_id_, end_row_id);
+    ranges_.emplace_back(first_row_id, end_row_id - 1);
     files_.push_back(file);
-    latest_max_sequence_number_ = file->max_sequence_number;
-    latest_first_row_id_ = first_row_id;
-    expected_next_first_row_id_ = latest_first_row_id_ + file->row_count;
+    if (!has_row_ids_selection_ && expected_row_count_ >= 0 && RowCount() > expected_row_count_) {
+        return Status::Invalid(
+            fmt::format("Blob files row count exceed the expect {}", expected_row_count_));
+    }
     return Status::OK();
 }
 DataEvolutionSplitRead::DataEvolutionSplitRead(
@@ -235,7 +229,8 @@ Result> DataEvolutionSplitRead::CreateBlobViewReade
         std::vector> raw_file_readers,
         CreateRawFileReaders(split_impl->Partition(), data_files, blob_view_schema,
                              /*predicate=*/nullptr, /*dv_factory=*/nullptr,
-                             /*row_ranges=*/std::nullopt, data_file_path_factory));
+                             /*row_ranges=*/std::nullopt, data_file_path_factory,
+                             /*extra_format_options=*/{}));
 
     auto batch_readers =
         ObjectUtils::MoveVector>(std::move(raw_file_readers));
@@ -314,7 +309,8 @@ Result> DataEvolutionSplitRead::InnerCreateReader(
                 std::vector> raw_file_readers,
                 CreateRawFileReaders(split_impl->Partition(), need_merge_files, raw_read_schema_,
                                      /*predicate=*/nullptr,
-                                     /*dv_factory=*/nullptr, row_ranges, data_file_path_factory));
+                                     /*dv_factory=*/nullptr, row_ranges, data_file_path_factory,
+                                     /*extra_format_options=*/{}));
             assert(raw_file_readers.size() == 1);
             sub_readers.push_back(std::move(raw_file_readers[0]));
         } else {
@@ -490,18 +486,30 @@ Result> DataEvolutionSplitRead::CreateU
         if (!read_fields_in_file.empty()) {
             // create new FieldMappingReader for read partial fields
             auto file_read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields_in_file);
-            PAIMON_ASSIGN_OR_RAISE(std::vector> file_readers,
-                                   CreateRawFileReaders(partition, bunch->Files(), file_read_schema,
-                                                        /*predicate=*/nullptr, /*dv_factory=*/{},
-                                                        row_ranges, data_file_path_factory));
-            if (file_readers.size() == 1) {
-                file_batch_readers[file_idx] = std::move(file_readers[0]);
+            auto blob_bunch = std::dynamic_pointer_cast(bunch);
+            if (blob_bunch && !blob_bunch->SequentialReadOptimize()) {
+                // blob files span multiple max sequence number layers: placeholder entries of
+                // newer layers must fall back row by row to older layers
+                PAIMON_ASSIGN_OR_RAISE(
+                    file_batch_readers[file_idx],
+                    CreateBlobFallbackReader(partition, blob_bunch->Files(), file_read_schema,
+                                             row_ranges, data_file_path_factory));
             } else {
-                auto raw_readers =
-                    ObjectUtils::MoveVector>(std::move(file_readers));
-                // Concat multiple blob files that map to the same data file.
-                file_batch_readers[file_idx] =
-                    std::make_unique(std::move(raw_readers), pool_);
+                PAIMON_ASSIGN_OR_RAISE(
+                    std::vector> file_readers,
+                    CreateRawFileReaders(partition, bunch->Files(), file_read_schema,
+                                         /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges,
+                                         data_file_path_factory,
+                                         /*extra_format_options=*/{}));
+                if (file_readers.size() == 1) {
+                    file_batch_readers[file_idx] = std::move(file_readers[0]);
+                } else {
+                    auto raw_readers = ObjectUtils::MoveVector>(
+                        std::move(file_readers));
+                    // Concat multiple blob files that map to the same data file.
+                    file_batch_readers[file_idx] =
+                        std::make_unique(std::move(raw_readers), pool_);
+                }
             }
         }
     }
@@ -511,6 +519,98 @@ Result> DataEvolutionSplitRead::CreateU
                                            field_offsets, pool_);
 }
 
+namespace {
+/// Selected row ids in [from, to] as sorted disjoint ranges: the whole range without a
+/// selection, otherwise its intersection with the (possibly overlapping) selected ranges.
+std::vector SelectedRangesInRange(int64_t from, int64_t to,
+                                         const std::optional>& row_ranges) {
+    if (!row_ranges) {
+        return {Range(from, to)};
+    }
+    std::vector selected;
+    Range gap_range(from, to);
+    for (const auto& range : Range::SortAndMergeOverlap(row_ranges.value(), /*adjacent=*/true)) {
+        std::optional intersection = Range::Intersection(gap_range, range);
+        if (intersection) {
+            selected.push_back(*intersection);
+        }
+    }
+    return selected;
+}
+}  // namespace
+
+Result> DataEvolutionSplitRead::CreateBlobFallbackReader(
+    const BinaryRow& partition, const std::vector>& files,
+    const std::shared_ptr& file_read_schema,
+    const std::optional>& row_ranges,
+    const std::shared_ptr& data_file_path_factory) const {
+    int64_t union_first_row_id = std::numeric_limits::max();
+    int64_t union_last_row_id = std::numeric_limits::min();
+    using FileWithFirstRowId = std::pair>;
+    std::map, std::greater<>> sequence_groups;
+    for (const auto& file : files) {
+        PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId());
+        union_first_row_id = std::min(union_first_row_id, first_row_id);
+        union_last_row_id = std::max(union_last_row_id, first_row_id + file->row_count - 1);
+        sequence_groups[file->max_sequence_number].emplace_back(first_row_id, file);
+    }
+    // the blob format reader must emit placeholder sentinels instead of failing on them
+    const std::map blob_format_options = {
+        {BlobDefs::kEmitPlaceholderSentinelKey, "true"}};
+    std::vector> groups;
+    groups.reserve(sequence_groups.size());
+    for (auto& [max_sequence_number, group_files] : sequence_groups) {
+        std::stable_sort(group_files.begin(), group_files.end(),
+                         [](const FileWithFirstRowId& f1, const FileWithFirstRowId& f2) {
+                             return f1.first < f2.first;
+                         });
+        std::vector segments;
+        // pad row ids this layer does not cover with placeholder gaps, so that every layer spans
+        // the same union range and the groups can be stepped in lockstep
+        int64_t next_row_id = union_first_row_id;
+        for (const auto& [first_row_id, file] : group_files) {
+            if (first_row_id < next_row_id) {
+                return Status::Invalid(fmt::format(
+                    "Blob files with the same max sequence number should not have overlapping "
+                    "row id ranges: file {} (max sequence number {}) starts at row id {} before "
+                    "the previous file's end {}",
+                    file->file_name, max_sequence_number, first_row_id, next_row_id));
+            }
+            if (first_row_id > next_row_id) {
+                std::vector gap_selected_ranges =
+                    SelectedRangesInRange(next_row_id, first_row_id - 1, row_ranges);
+                if (!gap_selected_ranges.empty()) {
+                    segments.push_back(
+                        BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)});
+                }
+            }
+            PAIMON_ASSIGN_OR_RAISE(
+                std::vector> file_readers,
+                CreateRawFileReaders(partition, {file}, file_read_schema,
+                                     /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges,
+                                     data_file_path_factory, blob_format_options));
+            if (file_readers.size() != 1) {
+                return Status::Invalid("Unexpected: blob fallback file reader was skipped.");
+            }
+            segments.push_back(BlobFallbackBatchReader::Segment{std::move(file_readers[0]), {}});
+            next_row_id = first_row_id + file->row_count;
+        }
+        if (next_row_id <= union_last_row_id) {
+            std::vector gap_selected_ranges =
+                SelectedRangesInRange(next_row_id, union_last_row_id, row_ranges);
+            if (!gap_selected_ranges.empty()) {
+                segments.push_back(
+                    BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)});
+            }
+        }
+        groups.push_back(std::move(segments));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_reader,
+                           BlobFallbackBatchReader::Create(std::move(groups), file_read_schema,
+                                                           options_.GetReadBatchSize(), pool_));
+    return std::move(fallback_reader);
+}
+
 Result DataEvolutionSplitRead::Match(const std::shared_ptr& split,
                                            bool force_keep_delete) const {
     return true;
diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h
index ffb2328c..9c45b32f 100644
--- a/src/paimon/core/operation/data_evolution_split_read.h
+++ b/src/paimon/core/operation/data_evolution_split_read.h
@@ -20,6 +20,8 @@
 
 #include 
 #include 
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -57,7 +59,8 @@ struct DeletionFile;
 /// Readers Overview: (ConcatBatchReader across
 /// splits)->(BlobViewResolvingBatchReader)->(CompleteIndexScoreBatchReader)->
 /// CompleteRowKindBatchReader->(PredicateBatchReader)
-/// ->ConcatBatchReader across files->DataEvolutionFileReader->(ConcatBatchReader across blob files)
+/// ->ConcatBatchReader across files->DataEvolutionFileReader
+/// ->(ConcatBatchReader across blob files | BlobFallbackBatchReader across blob sequence layers)
 /// ->FieldMappingReader->(CompleteRowTrackingFieldsBatchReader)->(ShreddingFileReader)
 /// ->(MapSharedShreddingFileReader)
 /// ->(DelegatingPrefetchReader)->(PrefetchFileBatchReader)->FormatReader
@@ -109,26 +112,43 @@ class DataEvolutionSplitRead : public AbstractSplitRead {
         std::vector> data_files_;
     };
 
+    /// All blob files of one blob field in a merge group. Unlike data files, every file is kept
+    /// (aligned with Java's BlobFileBunch): files whose max sequence numbers differ form layers
+    /// of a data-evolution partial update, where newer layers record untouched rows as
+    /// placeholder entries and reading falls back row by row to older layers.
+    /// Files must be added ordered by first row id ascending (then max sequence number
+    /// descending), as produced by MergeRangesAndSort.
     class BlobBunch : public FieldBunch {
      public:
         explicit BlobBunch(int64_t expected_row_count, bool has_row_ids_selection)
             : expected_row_count_(expected_row_count),
               has_row_ids_selection_(has_row_ids_selection) {}
-        int64_t RowCount() const override {
-            return row_count_;
-        }
+        /// Number of distinct row ids covered by the added files. Without a row-ids selection
+        /// the covered range is contiguous (enforced by Add) and matches the data files.
+        int64_t RowCount() const override;
         const std::vector>& Files() const override {
             return files_;
         }
         Status Add(const std::shared_ptr& file);
+        /// True when every file shares one max sequence number, so the files are read
+        /// sequentially without the fallback merge. A lone layer is expected to hold no
+        /// placeholder entries; if one does (a user value equal to the placeholder sentinel
+        /// written by a blob-only first write), the strict blob reader rejects the read, since
+        /// no older layer exists to resolve it.
+        bool SequentialReadOptimize() const {
+            return sequence_group_end_.size() <= 1;
+        }
 
      private:
         int64_t expected_row_count_ = -1;
-        int64_t latest_first_row_id_ = -1;
-        int64_t expected_next_first_row_id_ = -1;
-        int64_t latest_max_sequence_number_ = -1;
-        int64_t row_count_ = 0;
         bool has_row_ids_selection_ = false;
+        int64_t union_first_row_id_ = std::numeric_limits::max();
+        /// Exclusive end of the union row id range covered so far.
+        int64_t union_end_row_id_ = std::numeric_limits::min();
+        /// Per max sequence number: exclusive end of the last added range, to reject
+        /// overlapping files within one layer.
+        std::map sequence_group_end_;
+        std::vector ranges_;
         std::vector> files_;
     };
 
@@ -166,6 +186,16 @@ class DataEvolutionSplitRead : public AbstractSplitRead {
         const std::vector>& need_merge_files,
         const std::optional>& row_ranges,
         const std::shared_ptr& data_file_path_factory) const;
+
+    /// Builds the row-level fallback reader for a blob bunch spanning multiple max sequence
+    /// number layers: groups the files by max sequence number, pads uncovered row id ranges of
+    /// each layer with placeholder gap segments, and resolves each row to the newest
+    /// non-placeholder layer. See BlobFallbackBatchReader.
+    Result> CreateBlobFallbackReader(
+        const BinaryRow& partition, const std::vector>& files,
+        const std::shared_ptr& file_read_schema,
+        const std::optional>& row_ranges,
+        const std::shared_ptr& data_file_path_factory) const;
 };
 
 }  // namespace paimon
diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp
index d44c3992..597d668d 100644
--- a/src/paimon/core/operation/data_evolution_split_read_test.cpp
+++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp
@@ -147,72 +147,88 @@ TEST_F(DataEvolutionSplitReadTest, TestAddNonBlobFileInvalid) {
 TEST_F(DataEvolutionSplitReadTest, TestAddBlobWithSameFirstRowId) {
     auto blob_entry =
         CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
-                       /*max_sequence_number=*/1,
+                       /*max_sequence_number=*/3,
                        /*write_cols=*/std::optional>({"blob_col"}));
-    auto blob_tail =
-        CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50,
+    auto blob_full_tail =
+        CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/100,
                        /*max_sequence_number=*/2,
                        /*write_cols=*/std::optional>({"blob_col"}));
+    auto blob_short_tail =
+        CreateBlobFile("blob3", /*first_row_id=*/0, /*row_count=*/50,
+                       /*max_sequence_number=*/1,
+                       /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/false);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail),
-                        "Blob file with same first row id should have decreasing sequence number.");
+    // Files with the same first row id and lower sequence numbers are older layers of a
+    // partial update; they are kept for the row-level placeholder fallback, whether they
+    // cover the same range or only a shorter prefix of it.
+    ASSERT_OK(blob_bunch->Add(blob_full_tail));
+    ASSERT_OK(blob_bunch->Add(blob_short_tail));
+
+    ASSERT_EQ(blob_bunch->Files().size(), 3);
+    ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_full_tail);
+    ASSERT_EQ(blob_bunch->Files()[2], blob_short_tail);
+    ASSERT_EQ(blob_bunch->RowCount(), 100);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
-TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithSameFirstRowIdAndLowerSequenceNumber) {
+TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) {
     auto blob_entry =
         CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
                        /*max_sequence_number=*/2,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_tail =
-        CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50,
+        CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150,
                        /*max_sequence_number=*/1,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/false);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    // Adding file with same firstRowId and lower sequence number should be ignored
+    // Overlapping layers with different sequence numbers are kept for the fallback.
     ASSERT_OK(blob_bunch->Add(blob_tail));
 
-    ASSERT_EQ(blob_bunch->Files().size(), 1);
-    ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files().size(), 2);
+    ASSERT_EQ(blob_bunch->RowCount(), 200);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
-TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) {
+TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) {
     auto blob_entry =
         CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
-                       /*max_sequence_number=*/2,
+                       /*max_sequence_number=*/1,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_tail =
         CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150,
-                       /*max_sequence_number=*/1,
+                       /*max_sequence_number=*/2,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/false);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    // Adding file with overlapping row id and lower sequence number should be ignored
     ASSERT_OK(blob_bunch->Add(blob_tail));
 
-    ASSERT_EQ(blob_bunch->Files().size(), 1);
-    ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files().size(), 2);
+    ASSERT_EQ(blob_bunch->RowCount(), 200);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
-TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) {
+TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdInSameLayer) {
     auto blob_entry =
         CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100,
                        /*max_sequence_number=*/1,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_tail =
         CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150,
-                       /*max_sequence_number=*/2,
+                       /*max_sequence_number=*/1,
                        /*write_cols=*/std::optional>({"blob_col"}));
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/false);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    ASSERT_NOK_WITH_MSG(
-        blob_bunch->Add(blob_tail),
-        "Blob file with overlapping row id should have decreasing sequence number.");
+    // Files sharing a max sequence number form one layer and must not overlap.
+    ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail),
+                        "Blob files with the same max sequence number should not have overlapping "
+                        "row id ranges");
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithNonContinuousRowId) {
@@ -267,12 +283,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap) {
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/true);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    // blob_sub1 will not be added, for it has been skipped by row_ids in scan process.
-    // after blob_sub2 is added, blob_entry is removed
+    // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is a newer
+    // layer and both files are kept for the row-level placeholder fallback.
     ASSERT_OK(blob_bunch->Add(blob_sub2));
-    ASSERT_EQ(blob_bunch->Files().size(), 1);
-    ASSERT_EQ(blob_bunch->Files()[0], blob_sub2);
-    ASSERT_EQ(blob_bunch->RowCount(), 5);
+    ASSERT_EQ(blob_bunch->Files().size(), 2);
+    ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_sub2);
+    ASSERT_EQ(blob_bunch->RowCount(), 10);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) {
@@ -293,13 +311,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) {
     auto blob_bunch = std::make_shared(
         INT64_MAX, /*has_row_ids_selection=*/true);
     ASSERT_OK(blob_bunch->Add(blob_entry));
-    // blob_sub1 will not be added, for it has been skipped by row_ids in scan process.
-    // after blob_sub2 is added, as blob_sub2 has smaller sequence number, blob_sub2 will be
-    // skipped.
+    // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is an older
+    // layer and both files are kept for the row-level placeholder fallback.
     ASSERT_OK(blob_bunch->Add(blob_sub2));
-    ASSERT_EQ(blob_bunch->Files().size(), 1);
+    ASSERT_EQ(blob_bunch->Files().size(), 2);
     ASSERT_EQ(blob_bunch->Files()[0], blob_entry);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_sub2);
     ASSERT_EQ(blob_bunch->RowCount(), 10);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestRowIdSelection) {
@@ -422,15 +441,15 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) {
     std::vector> batch = batches[0];
     ASSERT_EQ(batch.size(), 10);
     ASSERT_EQ(batch[0], data);
-    ASSERT_EQ(batch[1], blob_entry5);  // pick
-    ASSERT_EQ(batch[2], blob_entry2);  // skip
-    ASSERT_EQ(batch[3], blob_entry1);  // skip
-    ASSERT_EQ(batch[4], blob_entry9);  // pick
-    ASSERT_EQ(batch[5], blob_entry6);  // skip
-    ASSERT_EQ(batch[6], blob_entry3);  // skip
-    ASSERT_EQ(batch[7], blob_entry7);  // pick
-    ASSERT_EQ(batch[8], blob_entry4);  // skip
-    ASSERT_EQ(batch[9], blob_entry8);  // pick
+    ASSERT_EQ(batch[1], blob_entry5);
+    ASSERT_EQ(batch[2], blob_entry2);
+    ASSERT_EQ(batch[3], blob_entry1);
+    ASSERT_EQ(batch[4], blob_entry9);
+    ASSERT_EQ(batch[5], blob_entry6);
+    ASSERT_EQ(batch[6], blob_entry3);
+    ASSERT_EQ(batch[7], blob_entry7);
+    ASSERT_EQ(batch[8], blob_entry4);
+    ASSERT_EQ(batch[9], blob_entry8);
 
     auto blob_field_to_field_id = [](const std::shared_ptr&) -> Result {
         return 0;
@@ -442,12 +461,19 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) {
     ASSERT_EQ(bunch.size(), 2);
     auto blob_bunch = std::dynamic_pointer_cast(bunch[1]);
 
-    ASSERT_EQ(blob_bunch->Files().size(), 4);
+    // every sequence layer is kept for the row-level placeholder fallback
+    ASSERT_EQ(blob_bunch->Files().size(), 9);
     ASSERT_EQ(blob_bunch->Files()[0], blob_entry5);
-    ASSERT_EQ(blob_bunch->Files()[1], blob_entry9);
-    ASSERT_EQ(blob_bunch->Files()[2], blob_entry7);
-    ASSERT_EQ(blob_bunch->Files()[3], blob_entry8);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_entry2);
+    ASSERT_EQ(blob_bunch->Files()[2], blob_entry1);
+    ASSERT_EQ(blob_bunch->Files()[3], blob_entry9);
+    ASSERT_EQ(blob_bunch->Files()[4], blob_entry6);
+    ASSERT_EQ(blob_bunch->Files()[5], blob_entry3);
+    ASSERT_EQ(blob_bunch->Files()[6], blob_entry7);
+    ASSERT_EQ(blob_bunch->Files()[7], blob_entry4);
+    ASSERT_EQ(blob_bunch->Files()[8], blob_entry8);
     ASSERT_EQ(blob_bunch->RowCount(), 1000);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) {
@@ -563,20 +589,33 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) {
 
     ASSERT_EQ(bunch.size(), 3);
     auto blob_bunch = std::dynamic_pointer_cast(bunch[1]);
-    ASSERT_EQ(blob_bunch->Files().size(), 4);
+    // every sequence layer is kept for the row-level placeholder fallback
+    ASSERT_EQ(blob_bunch->Files().size(), 9);
     ASSERT_EQ(blob_bunch->Files()[0], blob_entry5);
-    ASSERT_EQ(blob_bunch->Files()[1], blob_entry9);
-    ASSERT_EQ(blob_bunch->Files()[2], blob_entry7);
-    ASSERT_EQ(blob_bunch->Files()[3], blob_entry8);
+    ASSERT_EQ(blob_bunch->Files()[1], blob_entry2);
+    ASSERT_EQ(blob_bunch->Files()[2], blob_entry1);
+    ASSERT_EQ(blob_bunch->Files()[3], blob_entry9);
+    ASSERT_EQ(blob_bunch->Files()[4], blob_entry6);
+    ASSERT_EQ(blob_bunch->Files()[5], blob_entry3);
+    ASSERT_EQ(blob_bunch->Files()[6], blob_entry7);
+    ASSERT_EQ(blob_bunch->Files()[7], blob_entry4);
+    ASSERT_EQ(blob_bunch->Files()[8], blob_entry8);
     ASSERT_EQ(blob_bunch->RowCount(), 1000);
+    ASSERT_FALSE(blob_bunch->SequentialReadOptimize());
 
     auto blob_bunch2 = std::dynamic_pointer_cast(bunch[2]);
-    ASSERT_EQ(blob_bunch2->Files().size(), 4);
+    ASSERT_EQ(blob_bunch2->Files().size(), 9);
     ASSERT_EQ(blob_bunch2->Files()[0], blob_entry15);
-    ASSERT_EQ(blob_bunch2->Files()[1], blob_entry19);
-    ASSERT_EQ(blob_bunch2->Files()[2], blob_entry17);
-    ASSERT_EQ(blob_bunch2->Files()[3], blob_entry18);
+    ASSERT_EQ(blob_bunch2->Files()[1], blob_entry12);
+    ASSERT_EQ(blob_bunch2->Files()[2], blob_entry11);
+    ASSERT_EQ(blob_bunch2->Files()[3], blob_entry19);
+    ASSERT_EQ(blob_bunch2->Files()[4], blob_entry16);
+    ASSERT_EQ(blob_bunch2->Files()[5], blob_entry13);
+    ASSERT_EQ(blob_bunch2->Files()[6], blob_entry17);
+    ASSERT_EQ(blob_bunch2->Files()[7], blob_entry14);
+    ASSERT_EQ(blob_bunch2->Files()[8], blob_entry18);
     ASSERT_EQ(blob_bunch2->RowCount(), 1000);
+    ASSERT_FALSE(blob_bunch2->SequentialReadOptimize());
 }
 
 TEST_F(DataEvolutionSplitReadTest, TestDifferentRowIdRange) {
diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp
index 34567fce..2f2b46cf 100644
--- a/src/paimon/core/operation/merge_file_split_read.cpp
+++ b/src/paimon/core/operation/merge_file_split_read.cpp
@@ -274,7 +274,8 @@ Result> MergeFileSplitRead::CreateNoMergeReader(
         std::vector> raw_file_readers,
         CreateRawFileReaders(data_split->Partition(), data_split->DataFiles(), read_schema,
                              only_filter_key ? predicate_for_keys_ : context_->GetPredicate(),
-                             dv_factory, /*row_ranges=*/{}, data_file_path_factory));
+                             dv_factory, /*row_ranges=*/{}, data_file_path_factory,
+                             /*extra_format_options=*/{}));
 
     auto raw_readers =
         ObjectUtils::MoveVector>(std::move(raw_file_readers));
@@ -498,7 +499,8 @@ Result> MergeFileSplitRead::CreateReaderFo
     PAIMON_ASSIGN_OR_RAISE(
         std::vector> raw_file_readers,
         CreateRawFileReaders(partition, data_files, read_schema_, predicate, dv_factory,
-                             /*row_ranges=*/{}, data_file_path_factory));
+                             /*row_ranges=*/{}, data_file_path_factory,
+                             /*extra_format_options=*/{}));
 
     assert(data_files.size() == raw_file_readers.size());
     // KeyValueDataFileRecordReader converts arrow array from format reader to KeyValue objects
diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp
index b6c9f25a..eabe8426 100644
--- a/src/paimon/core/operation/raw_file_split_read.cpp
+++ b/src/paimon/core/operation/raw_file_split_read.cpp
@@ -83,7 +83,8 @@ Result> RawFileSplitRead::CreateReader(
     PAIMON_ASSIGN_OR_RAISE(
         std::vector> raw_file_readers,
         CreateRawFileReaders(partition, data_files, raw_read_schema_, predicate, dv_factory,
-                             /*row_ranges=*/{}, data_file_path_factory));
+                             /*row_ranges=*/{}, data_file_path_factory,
+                             /*extra_format_options=*/{}));
 
     auto raw_readers =
         ObjectUtils::MoveVector>(std::move(raw_file_readers));
diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.cpp b/src/paimon/core/table/source/data_evolution_batch_scan.cpp
index d43addfc..ed09b693 100644
--- a/src/paimon/core/table/source/data_evolution_batch_scan.cpp
+++ b/src/paimon/core/table/source/data_evolution_batch_scan.cpp
@@ -19,6 +19,9 @@
 
 #include "paimon/core/table/source/data_evolution_batch_scan.h"
 
+#include 
+#include 
+
 #include "paimon/core/global_index/global_index_scan_impl.h"
 #include "paimon/core/global_index/indexed_split_impl.h"
 #include "paimon/core/table/source/data_split_impl.h"
@@ -75,7 +78,7 @@ Result> DataEvolutionBatchScan::CreatePlan() {
 
 Result> DataEvolutionBatchScan::WrapToIndexedSplits(
     const std::shared_ptr& data_plan, const RowRangeIndex& row_range_index,
-    const std::map& id_to_score) const {
+    const std::map& id_to_score) {
     // TODO(lisizhuo.lsz): add executor here
     auto data_splits = data_plan->Splits();
     std::vector> indexed_splits;
@@ -89,11 +92,23 @@ Result> DataEvolutionBatchScan::WrapToIndexedSplits(
         if (files.empty()) {
             return Status::Invalid("Empty data files in WrapToIndexedSplits");
         }
-        PAIMON_ASSIGN_OR_RAISE(int64_t min, files[0]->NonNullFirstRowId());
-        PAIMON_ASSIGN_OR_RAISE(int64_t max, files[files.size() - 1]->NonNullFirstRowId());
-        max += files[files.size() - 1]->row_count - 1;
+        // The row-id ranges of the files in a split may be unordered, discontiguous, or
+        // overlapping, so intersect the index with each file's range separately, then sort and
+        // merge the intersected ranges.
+        std::vector intersected;
+        int64_t min = std::numeric_limits::max();
+        int64_t max = std::numeric_limits::min();
+        for (const auto& file : files) {
+            PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId());
+            int64_t last_row_id = first_row_id + file->row_count - 1;
+            min = std::min(min, first_row_id);
+            max = std::max(max, last_row_id);
+            std::vector file_ranges =
+                row_range_index.IntersectedRanges(first_row_id, last_row_id);
+            intersected.insert(intersected.end(), file_ranges.begin(), file_ranges.end());
+        }
 
-        std::vector expected = row_range_index.IntersectedRanges(min, max);
+        std::vector expected = Range::SortAndMergeOverlap(intersected, /*adjacent=*/true);
         if (expected.empty()) {
             return Status::Invalid(
                 fmt::format("There should be intersected ranges for split with min row id {} and "
diff --git a/src/paimon/core/table/source/data_evolution_batch_scan.h b/src/paimon/core/table/source/data_evolution_batch_scan.h
index 1dc2f296..cfa29785 100644
--- a/src/paimon/core/table/source/data_evolution_batch_scan.h
+++ b/src/paimon/core/table/source/data_evolution_batch_scan.h
@@ -19,6 +19,7 @@
 
 #pragma once
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -40,10 +41,14 @@ class DataEvolutionBatchScan : public AbstractTableScan {
 
     Result> CreatePlan() override;
 
- private:
-    Result> WrapToIndexedSplits(
+    /// Wraps each DataSplit in `data_plan` into an IndexedSplit whose row ranges are the
+    /// intersection of `row_range_index` and the row-id range of each data file. Visible for
+    /// testing.
+    static Result> WrapToIndexedSplits(
         const std::shared_ptr& data_plan, const RowRangeIndex& row_range_index,
-        const std::map& id_to_score) const;
+        const std::map& id_to_score);
+
+ private:
     Result> EvalGlobalIndex() const;
 
  private:
diff --git a/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp
new file mode 100644
index 00000000..14fae397
--- /dev/null
+++ b/src/paimon/core/table/source/data_evolution_batch_scan_test.cpp
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/source/data_evolution_batch_scan.h"
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "gtest/gtest.h"
+#include "paimon/common/data/binary_row.h"
+#include "paimon/core/global_index/indexed_split_impl.h"
+#include "paimon/core/io/data_file_meta.h"
+#include "paimon/core/manifest/file_source.h"
+#include "paimon/core/stats/simple_stats.h"
+#include "paimon/core/table/source/data_split_impl.h"
+#include "paimon/core/table/source/plan_impl.h"
+#include "paimon/data/timestamp.h"
+#include "paimon/testing/utils/testharness.h"
+#include "paimon/utils/range.h"
+#include "paimon/utils/row_range_index.h"
+
+namespace paimon::test {
+namespace {
+std::shared_ptr NewAppendFile(const std::string& file_name, int64_t first_row_id,
+                                            int64_t row_count) {
+    return std::make_shared(
+        file_name, /*file_size=*/1024l, row_count, BinaryRow::EmptyRow(), BinaryRow::EmptyRow(),
+        SimpleStats::EmptyStats(), SimpleStats::EmptyStats(), /*min_sequence_number=*/0l,
+        /*max_sequence_number=*/first_row_id + row_count - 1, /*schema_id=*/0, /*level=*/0,
+        std::vector>(), Timestamp(0l, 0), /*delete_row_count=*/0,
+        /*embedded_index=*/nullptr, FileSource::Append(), std::nullopt, std::nullopt, first_row_id,
+        std::nullopt);
+}
+
+std::shared_ptr NewDataPlan(std::vector> files) {
+    DataSplitImpl::Builder builder(
+        /*partition=*/BinaryRow::EmptyRow(),
+        /*bucket=*/0, /*bucket_path=*/"data/test_table/bucket-0", std::move(files));
+    std::shared_ptr data_split =
+        builder.WithSnapshot(1).IsStreaming(false).RawConvertible(true).Build().value();
+    return std::make_shared(/*snapshot_id=*/1,
+                                      std::vector>({data_split}));
+}
+}  // namespace
+
+TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithUnorderedAndDiscontiguousDataFiles) {
+    // The files cover [4650, 4700], [4300, 4450] and [4200, 4407]: unordered, partially
+    // overlapping, with a gap [4451, 4649] that must not appear in the wrapped ranges.
+    std::shared_ptr data_plan =
+        NewDataPlan({NewAppendFile("file-1", 4650l, 51l), NewAppendFile("file-2", 4300l, 151l),
+                     NewAppendFile("file-3", 4200l, 208l)});
+    ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 5000)}));
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr plan,
+                         DataEvolutionBatchScan::WrapToIndexedSplits(data_plan, row_range_index,
+                                                                     /*id_to_score=*/{}));
+
+    ASSERT_EQ(plan->Splits().size(), 1);
+    auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]);
+    ASSERT_NE(indexed_split, nullptr);
+    ASSERT_EQ(indexed_split->GetDataSplit(), data_plan->Splits()[0]);
+    ASSERT_EQ(indexed_split->RowRanges(),
+              std::vector({Range(4200, 4450), Range(4650, 4700)}));
+    ASSERT_TRUE(indexed_split->Scores().empty());
+}
+
+TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsExcludesRowIdsInFileRangeGaps) {
+    // The split covers [0, 9] and [20, 29] while the index hits {5, 15, 25}. Row id 15 lies in
+    // the gap between the two files and must be excluded, together with its score.
+    std::shared_ptr data_plan =
+        NewDataPlan({NewAppendFile("file-1", 0l, 10l), NewAppendFile("file-2", 20l, 10l)});
+    ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index,
+                         RowRangeIndex::Create({Range(5, 5), Range(15, 15), Range(25, 25)}));
+    std::map id_to_score = {{5l, 0.5f}, {15l, 0.7f}, {25l, 0.9f}};
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, DataEvolutionBatchScan::WrapToIndexedSplits(
+                                                         data_plan, row_range_index, id_to_score));
+
+    ASSERT_EQ(plan->Splits().size(), 1);
+    auto indexed_split = std::dynamic_pointer_cast(plan->Splits()[0]);
+    ASSERT_NE(indexed_split, nullptr);
+    ASSERT_EQ(indexed_split->RowRanges(), std::vector({Range(5, 5), Range(25, 25)}));
+    ASSERT_EQ(indexed_split->Scores(), std::vector({0.5f, 0.9f}));
+}
+
+TEST(DataEvolutionBatchScanTest, TestWrapToIndexedSplitsWithoutIntersection) {
+    std::shared_ptr data_plan = NewDataPlan({NewAppendFile("file-1", 100l, 10l)});
+    ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(0, 50)}));
+
+    ASSERT_NOK_WITH_MSG(DataEvolutionBatchScan::WrapToIndexedSplits(data_plan, row_range_index,
+                                                                    /*id_to_score=*/{}),
+                        "There should be intersected ranges for split with min row id 100 and "
+                        "max row id 109.");
+}
+
+}  // namespace paimon::test
diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp
index 03bc78a2..3cd94ed0 100644
--- a/src/paimon/format/blob/blob_file_batch_reader.cpp
+++ b/src/paimon/format/blob/blob_file_batch_reader.cpp
@@ -19,6 +19,7 @@
 #include "paimon/format/blob/blob_file_batch_reader.h"
 
 #include 
+#include 
 #include 
 
 #include "arrow/api.h"
@@ -40,7 +41,7 @@ namespace paimon::blob {
 
 Result> BlobFileBatchReader::Create(
     const std::shared_ptr& input_stream, int32_t batch_size, bool blob_as_descriptor,
-    const std::shared_ptr& pool) {
+    bool emit_placeholder_sentinel, const std::shared_ptr& pool) {
     if (input_stream == nullptr) {
         return Status::Invalid("blob file batch reader create failed: input stream is nullptr");
     }
@@ -85,14 +86,15 @@ Result> BlobFileBatchReader::Create(
     int64_t offset = 0;
     for (const auto& blob_length : blob_lengths) {
         blob_offsets.push_back(offset);
-        // Null blobs (bin_length == -1) don't occupy file space
+        // null (-1) and placeholder (-2) entries occupy no file space
         if (blob_length >= 0) {
             offset += blob_length;
         }
     }
     PAIMON_ASSIGN_OR_RAISE(std::string file_path, input_stream->GetUri());
-    auto reader = std::unique_ptr(new BlobFileBatchReader(
-        input_stream, file_path, blob_lengths, blob_offsets, batch_size, blob_as_descriptor, pool));
+    auto reader = std::unique_ptr(
+        new BlobFileBatchReader(input_stream, file_path, blob_lengths, blob_offsets, batch_size,
+                                blob_as_descriptor, emit_placeholder_sentinel, pool));
     return reader;
 }
 
@@ -101,6 +103,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp
                                          const std::vector& blob_lengths,
                                          const std::vector& blob_offsets,
                                          int32_t batch_size, bool blob_as_descriptor,
+                                         bool emit_placeholder_sentinel,
                                          const std::shared_ptr& pool)
     : input_stream_(input_stream),
       file_path_(file_path),
@@ -110,6 +113,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp
       target_blob_offsets_(blob_offsets),
       batch_size_(batch_size),
       blob_as_descriptor_(blob_as_descriptor),
+      emit_placeholder_sentinel_(emit_placeholder_sentinel),
       pool_(pool),
       arrow_pool_(GetArrowPool(pool_)),
       metrics_(std::make_shared()) {
@@ -158,7 +162,7 @@ Status BlobFileBatchReader::SetReadSchema(::ArrowSchema* read_schema,
     }
     target_type_ = arrow::struct_(arrow_schema->fields());
     current_pos_ = 0;
-    previous_batch_first_row_number_ = std::numeric_limits::max();
+    previous_batch_start_pos_ = std::numeric_limits::max();
     previous_batch_row_count_ = 0;
     return Status::OK();
 }
@@ -170,11 +174,7 @@ Result> BlobFileBatchReader::NextBlobOffsets(
     PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(0));
     int64_t data_length = 0;
     for (int32_t k = 0; k < rows_to_read; ++k) {
-        const size_t i = current_pos_ + k;
-        // Null blobs contribute zero bytes to content
-        if (!IsTargetNull(i)) {
-            data_length += GetTargetContentLength(i);
-        }
+        data_length += GetTargetOutputLength(current_pos_ + k);
         PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(data_length));
     }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr offset_buffer,
@@ -186,10 +186,7 @@ Result> BlobFileBatchReader::NextBlobContents(
     int32_t rows_to_read) const {
     int64_t total_length = 0;
     for (int32_t k = 0; k < rows_to_read; ++k) {
-        const size_t i = current_pos_ + k;
-        if (!IsTargetNull(i)) {
-            total_length += GetTargetContentLength(i);
-        }
+        total_length += GetTargetOutputLength(current_pos_ + k);
     }
     PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr data_buffer,
                                       arrow::AllocateBuffer(total_length, arrow_pool_.get()));
@@ -199,6 +196,13 @@ Result> BlobFileBatchReader::NextBlobContents(
         if (IsTargetNull(i)) {
             continue;
         }
+        if (IsTargetPlaceholder(i)) {
+            // a placeholder entry has no data bytes in the file; emit the sentinel for the
+            // data-evolution blob fallback merge to identify it
+            memcpy(buffer, BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength);
+            buffer += BlobDefs::kPlaceholderSentinelLength;
+            continue;
+        }
         int64_t offset = GetTargetContentOffset(i);
         int64_t length = GetTargetContentLength(i);
         PAIMON_RETURN_NOT_OK(ReadBlobContentAt(offset, length, buffer));
@@ -270,6 +274,9 @@ Result> BlobFileBatchReader::BuildTargetArray(
         PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append());
         if (IsTargetNull(i)) {
             PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->AppendNull());
+        } else if (IsTargetPlaceholder(i)) {
+            PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append(
+                BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength));
         } else {
             int64_t offset = GetTargetContentOffset(i);
             int64_t length = GetTargetContentLength(i);
@@ -293,18 +300,29 @@ Result BlobFileBatchReader::NextBatch() {
         return Status::Invalid("target type is nullptr, call SetReadSchema first");
     }
     if (current_pos_ >= target_blob_lengths_.size()) {
-        PAIMON_ASSIGN_OR_RAISE(previous_batch_first_row_number_, GetNumberOfRows());
+        previous_batch_start_pos_ = target_blob_lengths_.size();
         previous_batch_row_count_ = 0;
         return BatchReader::MakeEofBatch();
     }
     int32_t left_rows = target_blob_lengths_.size() - current_pos_;
     int32_t rows_to_read = std::min(left_rows, batch_size_);
+    if (!emit_placeholder_sentinel_) {
+        for (int32_t k = 0; k < rows_to_read; ++k) {
+            if (IsTargetPlaceholder(current_pos_ + k)) {
+                return Status::Invalid(fmt::format(
+                    "blob file {} contains a placeholder entry (bin_length {}) written by a "
+                    "data-evolution partial update; it can only be resolved by the data-evolution "
+                    "blob fallback read path",
+                    file_path_, BlobDefs::kPlaceholderBinLength));
+            }
+        }
+    }
     PAIMON_ASSIGN_OR_RAISE(std::shared_ptr blob_array,
                            BuildTargetArray(rows_to_read));
     std::unique_ptr c_array = std::make_unique();
     std::unique_ptr c_schema = std::make_unique();
     PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*blob_array, c_array.get(), c_schema.get()));
-    previous_batch_first_row_number_ = target_blob_row_indexes_[current_pos_];
+    previous_batch_start_pos_ = current_pos_;
     current_pos_ += rows_to_read;
     previous_batch_row_count_ = c_array->length;
     return make_pair(std::move(c_array), std::move(c_schema));
diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h
index e55e5944..ccb0afe8 100644
--- a/src/paimon/format/blob/blob_file_batch_reader.h
+++ b/src/paimon/format/blob/blob_file_batch_reader.h
@@ -51,7 +51,8 @@ namespace paimon::blob {
 /// ====================================================================
 /// 1. Data Bins Section
 /// ====================================================================
-/// The file consists of one or more contiguous 'bins' (bin_0, bin_1, bin_2, ...).
+/// The file consists of zero or more contiguous 'bins' (bin_0, bin_1, bin_2, ...); rows whose
+/// index entry is negative (see Section 2) have no bin.
 /// The structure of each bin is as follows:
 ///
 /// | Field Name        | Length (bytes) | Description                                             |
@@ -69,7 +70,13 @@ namespace paimon::blob {
 /// ====================================================================
 /// The Index is located after all data bins and is used for quick lookup and management.
 ///
-/// Purpose: Records the lengths (record lens) of all data bins.
+/// Purpose: Records one signed length entry per row, in row order.
+///
+/// Special lengths: an entry does not always describe a bin present in the Data Bins Section.
+/// - -1 (BlobDefs::kNullBinLength): a null blob; no bin is written.
+/// - -2 (BlobDefs::kPlaceholderBinLength): a placeholder blob written by a data-evolution
+///   partial update for a row it did not touch; no bin is written and the value must be
+///   resolved from an older blob file covering the same row.
 ///
 /// Encoding:
 /// - Uses Delta Encoding to store differences between successive length values.
@@ -89,9 +96,16 @@ namespace paimon::blob {
 /// - Current version is 1.
 class BlobFileBatchReader : public FileBatchReader {
  public:
+    /// `emit_placeholder_sentinel` controls how placeholder entries (bin_length ==
+    /// BlobDefs::kPlaceholderBinLength) are read: when false they fail the read, as resolving
+    /// them requires the data-evolution blob fallback path; when true they are returned as the
+    /// non-null BlobDefs::kPlaceholderSentinel bytes for that path to merge away. Stored values
+    /// are returned verbatim; see BlobDefs::kPlaceholderSentinel for the accepted collision
+    /// with a user value exactly equal to the sentinel.
     static Result> Create(
         const std::shared_ptr& input_stream, int32_t batch_size,
-        bool blob_as_descriptor, const std::shared_ptr& pool);
+        bool blob_as_descriptor, bool emit_placeholder_sentinel,
+        const std::shared_ptr& pool);
 
     Result> GetFileSchema() const override;
 
@@ -101,14 +115,8 @@ class BlobFileBatchReader : public FileBatchReader {
     Result NextBatch() override;
 
     Result GetPreviousBatchFileRowId(uint64_t batch_row_id) const override {
-        if (all_blob_lengths_.size() != target_blob_lengths_.size()) {
-            return Status::NotImplemented(
-                "Cannot call GetPreviousBatchFileRowId in BlobFileBatchReader because, after "
-                "bitmap pushdown, rows in the array returned by NextBatch are no longer "
-                "contiguous.");
-        }
         if (previous_batch_row_count_ == 0) {
-            if (previous_batch_first_row_number_ == std::numeric_limits::max()) {
+            if (previous_batch_start_pos_ == std::numeric_limits::max()) {
                 return Status::Invalid("No batch has been read yet.");
             } else {
                 return Status::Invalid("Last batch was EOF.");
@@ -119,7 +127,9 @@ class BlobFileBatchReader : public FileBatchReader {
                 fmt::format("batch_row_id {} is out of range, last batch row count is {}",
                             batch_row_id, previous_batch_row_count_));
         }
-        return previous_batch_first_row_number_ + batch_row_id;
+        // target_blob_row_indexes_ maps every selected position back to its original file row
+        // index, so this stays correct after a bitmap selection removed rows.
+        return target_blob_row_indexes_[previous_batch_start_pos_ + batch_row_id];
     }
 
     Result GetNumberOfRows() const override {
@@ -146,7 +156,8 @@ class BlobFileBatchReader : public FileBatchReader {
     BlobFileBatchReader(const std::shared_ptr& input_stream,
                         const std::string& file_path, const std::vector& blob_lengths,
                         const std::vector& blob_offsets, int32_t batch_size,
-                        bool blob_as_descriptor, const std::shared_ptr& pool);
+                        bool blob_as_descriptor, bool emit_placeholder_sentinel,
+                        const std::shared_ptr& pool);
 
     Status ReadBlobContentAt(const int64_t offset, const int64_t length, uint8_t* content) const;
 
@@ -162,6 +173,23 @@ class BlobFileBatchReader : public FileBatchReader {
         return target_blob_lengths_[index] == BlobDefs::kNullBinLength;
     }
 
+    bool IsTargetPlaceholder(size_t index) const {
+        return target_blob_lengths_[index] == BlobDefs::kPlaceholderBinLength;
+    }
+
+    /// Content bytes the blob at the given index contributes to the output buffer: nothing for
+    /// a null entry, the sentinel bytes for a placeholder entry (which occupies no file space),
+    /// the stored bytes otherwise.
+    int64_t GetTargetOutputLength(size_t index) const {
+        if (IsTargetNull(index)) {
+            return 0;
+        }
+        if (IsTargetPlaceholder(index)) {
+            return BlobDefs::kPlaceholderSentinelLength;
+        }
+        return GetTargetContentLength(index);
+    }
+
     int64_t GetTargetContentOffset(size_t index) const {
         return target_blob_offsets_[index] + BlobDefs::kContentStartOffset;
     }
@@ -181,6 +209,7 @@ class BlobFileBatchReader : public FileBatchReader {
 
     const int32_t batch_size_;
     const bool blob_as_descriptor_;
+    const bool emit_placeholder_sentinel_;
     std::shared_ptr pool_;
     std::shared_ptr arrow_pool_;
 
@@ -188,7 +217,9 @@ class BlobFileBatchReader : public FileBatchReader {
     std::shared_ptr metrics_;
 
     size_t current_pos_ = 0;
-    uint64_t previous_batch_first_row_number_ = std::numeric_limits::max();
+    /// Start position of the previous batch in the (possibly selection-filtered) target index
+    /// space; max() until the first batch is read.
+    size_t previous_batch_start_pos_ = std::numeric_limits::max();
     uint64_t previous_batch_row_count_ = 0;
     bool closed_ = false;
 };
diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp
index c53397f6..95637720 100644
--- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp
+++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp
@@ -46,9 +46,9 @@ class BlobFileBatchReaderTest : public testing::Test, public ::testing::WithPara
         std::shared_ptr fs = std::make_shared();
         ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
                              fs->Open(table_path + "/bucket-0/" + paimon_blob_file));
-        ASSERT_OK_AND_ASSIGN(auto reader,
-                             BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024,
-                                                         blob_as_descriptor, pool_));
+        ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create(
+                                              input_stream, /*batch_size=*/1024, blob_as_descriptor,
+                                              /*emit_placeholder_sentinel=*/false, pool_));
         ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, selection_bitmap));
         ASSERT_OK_AND_ASSIGN(auto chunked_array,
                              paimon::test::ReadResultCollector::CollectResult(reader.get()));
@@ -164,9 +164,10 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) {
     ASSERT_OK_AND_ASSIGN(
         std::shared_ptr input_stream,
         fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob"));
-    ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create(
-                                          input_stream,
-                                          /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         BlobFileBatchReader::Create(input_stream,
+                                                     /*batch_size=*/1, /*blob_as_descriptor=*/true,
+                                                     /*emit_placeholder_sentinel=*/false, pool_));
 
     ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt));
     ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows());
@@ -189,6 +190,45 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) {
     ASSERT_TRUE(BatchReader::IsEofBatch(batch4));
 }
 
+TEST_F(BlobFileBatchReaderTest, TestRowNumbersWithSelectionBitmap) {
+    // a selection bitmap removes rows, but batch positions must still map back to the original
+    // file row indexes so _ROW_ID completion works under row-range pushdown
+    auto schema = arrow::schema({BlobUtils::ToArrowField("my_blob_field", false)});
+    ::ArrowSchema c_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+
+    std::string test_data_path = paimon::test::GetDataDir() + "/db_with_blob.db/table_with_blob/";
+    auto dir = paimon::test::UniqueTestDirectory::Create();
+    std::string table_path = dir->Str();
+    ASSERT_TRUE(paimon::test::TestUtil::CopyDirectory(test_data_path, table_path));
+
+    std::shared_ptr fs = std::make_shared();
+    ASSERT_OK_AND_ASSIGN(
+        std::shared_ptr input_stream,
+        fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob"));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         BlobFileBatchReader::Create(input_stream,
+                                                     /*batch_size=*/1, /*blob_as_descriptor=*/true,
+                                                     /*emit_placeholder_sentinel=*/false, pool_));
+
+    RoaringBitmap32 selection;
+    selection.Add(0);
+    selection.Add(2);
+    ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, selection));
+    ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "No batch has been read yet");
+    ASSERT_OK_AND_ASSIGN(auto batch1, reader->NextBatch());
+    ArrowArrayRelease(batch1.first.get());
+    ArrowSchemaRelease(batch1.second.get());
+    ASSERT_EQ(0, reader->GetPreviousBatchFileRowId(0).value());
+    ASSERT_OK_AND_ASSIGN(auto batch2, reader->NextBatch());
+    ArrowArrayRelease(batch2.first.get());
+    ArrowSchemaRelease(batch2.second.get());
+    ASSERT_EQ(2, reader->GetPreviousBatchFileRowId(0).value());
+    ASSERT_OK_AND_ASSIGN(auto batch3, reader->NextBatch());
+    ASSERT_NOK_WITH_MSG(reader->GetPreviousBatchFileRowId(0), "Last batch was EOF");
+    ASSERT_TRUE(BatchReader::IsEofBatch(batch3));
+}
+
 TEST_F(BlobFileBatchReaderTest, InvalidScenario) {
     auto dir = paimon::test::UniqueTestDirectory::Create();
     ASSERT_TRUE(dir);
@@ -204,20 +244,22 @@ TEST_F(BlobFileBatchReaderTest, InvalidScenario) {
     {
         ASSERT_NOK_WITH_MSG(
             BlobFileBatchReader::Create(input_stream,
-                                        /*batch_size=*/0, /*blob_as_descriptor=*/true, pool_),
+                                        /*batch_size=*/0, /*blob_as_descriptor=*/true,
+                                        /*emit_placeholder_sentinel=*/false, pool_),
             "blob file batch reader create failed: read batch size '0' should be larger than zero");
     }
     {
         ASSERT_NOK_WITH_MSG(
             BlobFileBatchReader::Create(/*input_stream=*/nullptr,
-                                        /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_),
+                                        /*batch_size=*/1, /*blob_as_descriptor=*/true,
+                                        /*emit_placeholder_sentinel=*/false, pool_),
             "blob file batch reader create failed: input stream is nullptr");
     }
     {
         ASSERT_OK_AND_ASSIGN(
-            auto reader,
-            BlobFileBatchReader::Create(/*input_stream=*/input_stream,
-                                        /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_));
+            auto reader, BlobFileBatchReader::Create(/*input_stream=*/input_stream,
+                                                     /*batch_size=*/1, /*blob_as_descriptor=*/true,
+                                                     /*emit_placeholder_sentinel=*/false, pool_));
         ASSERT_NOK_WITH_MSG(reader->GetFileSchema(),
                             "blob file has no self-describing file schema");
         ASSERT_TRUE(reader->GetReaderMetrics());
@@ -239,7 +281,8 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) {
     ASSERT_OK_AND_ASSIGN(
         std::shared_ptr writer,
         BlobFormatWriter::Create(output_stream, struct_type, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/false, file_system, pool_));
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, file_system, pool_));
 
     ASSERT_OK(writer->Flush());
     ASSERT_OK(writer->Finish());
@@ -250,9 +293,10 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) {
 
     ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
                          file_system->Open(dir->Str() + "/file.blob"));
-    ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create(
-                                          input_stream,
-                                          /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_));
+    ASSERT_OK_AND_ASSIGN(auto reader,
+                         BlobFileBatchReader::Create(input_stream,
+                                                     /*batch_size=*/1, /*blob_as_descriptor=*/true,
+                                                     /*emit_placeholder_sentinel=*/false, pool_));
 
     ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt));
     ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows());
@@ -275,9 +319,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) {
             std::shared_ptr input_stream,
             fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob"));
         ASSERT_OK_AND_ASSIGN(
-            auto reader,
-            BlobFileBatchReader::Create(input_stream,
-                                        /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_));
+            auto reader, BlobFileBatchReader::Create(input_stream,
+                                                     /*batch_size=*/1, /*blob_as_descriptor=*/true,
+                                                     /*emit_placeholder_sentinel=*/false, pool_));
         ASSERT_NOK_WITH_MSG(reader->SetReadSchema(/*read_schema=*/nullptr, /*predicate=*/nullptr,
                                                   /*selection_bitmap=*/std::nullopt),
                             "SetReadSchema failed: read schema cannot be nullptr");
@@ -300,9 +344,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) {
             std::shared_ptr input_stream,
             fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob"));
         ASSERT_OK_AND_ASSIGN(
-            auto reader,
-            BlobFileBatchReader::Create(input_stream,
-                                        /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_));
+            auto reader, BlobFileBatchReader::Create(input_stream,
+                                                     /*batch_size=*/1, /*blob_as_descriptor=*/true,
+                                                     /*emit_placeholder_sentinel=*/false, pool_));
         ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr,
                                                   /*selection_bitmap=*/std::nullopt),
                             "read schema field number 2 is not 1");
@@ -325,9 +369,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) {
             std::shared_ptr input_stream,
             fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob"));
         ASSERT_OK_AND_ASSIGN(
-            auto reader,
-            BlobFileBatchReader::Create(input_stream,
-                                        /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_));
+            auto reader, BlobFileBatchReader::Create(input_stream,
+                                                     /*batch_size=*/1, /*blob_as_descriptor=*/true,
+                                                     /*emit_placeholder_sentinel=*/false, pool_));
         ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr,
                                                   /*selection_bitmap=*/std::nullopt),
                             "field my_blob_field: large_binary is not BLOB");
@@ -348,9 +392,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) {
             std::shared_ptr input_stream,
             fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob"));
         ASSERT_OK_AND_ASSIGN(
-            auto reader,
-            BlobFileBatchReader::Create(input_stream,
-                                        /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_));
+            auto reader, BlobFileBatchReader::Create(input_stream,
+                                                     /*batch_size=*/1, /*blob_as_descriptor=*/true,
+                                                     /*emit_placeholder_sentinel=*/false, pool_));
         RoaringBitmap32 roaring;
         roaring.Add(0);
         roaring.Add(1);
diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp
index 22db5e2f..c3666569 100644
--- a/src/paimon/format/blob/blob_format_writer.cpp
+++ b/src/paimon/format/blob/blob_format_writer.cpp
@@ -41,7 +41,7 @@ namespace paimon::blob {
 BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, const std::string& uri,
                                    const std::shared_ptr& data_type,
                                    bool write_null_on_missing_file,
-                                   bool write_null_on_fetch_failure,
+                                   bool write_null_on_fetch_failure, bool write_placeholder,
                                    const std::shared_ptr& fs,
                                    const std::shared_ptr& pool)
     : out_(out),
@@ -50,7 +50,8 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con
       fs_(fs),
       pool_(pool),
       write_null_on_missing_file_(write_null_on_missing_file),
-      write_null_on_fetch_failure_(write_null_on_fetch_failure) {
+      write_null_on_fetch_failure_(write_null_on_fetch_failure),
+      write_placeholder_(write_placeholder) {
     // Create() has already checked that data_type has exactly one BLOB field.
     blob_field_name_ = data_type_->field(0)->name();
     metrics_ = std::make_shared();
@@ -61,7 +62,7 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con
 
 Result> BlobFormatWriter::Create(
     const std::shared_ptr& out, const std::shared_ptr& data_type,
-    bool write_null_on_missing_file, bool write_null_on_fetch_failure,
+    bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder,
     const std::shared_ptr& fs, const std::shared_ptr& pool) {
     if (out == nullptr) {
         return Status::Invalid("blob format writer create failed. out is nullptr");
@@ -84,8 +85,9 @@ Result> BlobFormatWriter::Create(
             fmt::format("field {} is not BLOB", data_type->field(0)->ToString()));
     }
     PAIMON_ASSIGN_OR_RAISE(std::string uri, out->GetUri());
-    return std::unique_ptr(new BlobFormatWriter(
-        out, uri, data_type, write_null_on_missing_file, write_null_on_fetch_failure, fs, pool));
+    return std::unique_ptr(
+        new BlobFormatWriter(out, uri, data_type, write_null_on_missing_file,
+                             write_null_on_fetch_failure, write_placeholder, fs, pool));
 }
 
 Status BlobFormatWriter::AddBatch(ArrowArray* batch) {
@@ -119,7 +121,16 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) {
     const auto& blob_array =
         arrow::internal::checked_cast(*child_array);
     assert(blob_array.length() == 1);
-    PAIMON_RETURN_NOT_OK(WriteBlob(blob_array.GetView(0)));
+    std::string_view blob_data = blob_array.GetView(0);
+    // Only a data-evolution partial-update write interprets the sentinel, which marks a row
+    // the update did not touch; any other write stores the bytes verbatim. See
+    // BlobDefs::kPlaceholderSentinel for the accepted collision with a sentinel-equal user
+    // value.
+    if (write_placeholder_ && BlobDefs::IsPlaceholderSentinel(blob_data.data(), blob_data.size())) {
+        bin_lengths_.push_back(BlobDefs::kPlaceholderBinLength);
+        return Status::OK();
+    }
+    PAIMON_RETURN_NOT_OK(WriteBlob(blob_data));
     PAIMON_RETURN_NOT_OK(Flush());
     return Status::OK();
 }
diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h
index 78535c32..9537b347 100644
--- a/src/paimon/format/blob/blob_format_writer.h
+++ b/src/paimon/format/blob/blob_format_writer.h
@@ -70,9 +70,15 @@ class BlobFormatWriter : public FormatWriter {
     /// `write_null_on_missing_file` is enabled; otherwise a missing file follows
     /// `write_null_on_fetch_failure` like any other failed open.
     /// See Options::BLOB_WRITE_NULL_ON_MISSING_FILE / BLOB_WRITE_NULL_ON_FETCH_FAILURE.
+    ///
+    /// `write_placeholder` (see BlobDefs::kWritePlaceholderKey, false unless the write is a
+    /// data-evolution partial update) persists a value exactly equal to
+    /// BlobDefs::kPlaceholderSentinel as a placeholder entry (bin_length -2, no data bytes);
+    /// any other value is stored verbatim. When disabled, values are never interpreted and can
+    /// never be turned into placeholder entries.
     static Result> Create(
         const std::shared_ptr& out, const std::shared_ptr& data_type,
-        bool write_null_on_missing_file, bool write_null_on_fetch_failure,
+        bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder,
         const std::shared_ptr& fs, const std::shared_ptr& pool);
 
     Status AddBatch(ArrowArray* batch) override;
@@ -93,7 +99,7 @@ class BlobFormatWriter : public FormatWriter {
     BlobFormatWriter(const std::shared_ptr& out, const std::string& uri,
                      const std::shared_ptr& data_type,
                      bool write_null_on_missing_file, bool write_null_on_fetch_failure,
-                     const std::shared_ptr& fs,
+                     bool write_placeholder, const std::shared_ptr& fs,
                      const std::shared_ptr& pool);
 
     Status WriteBlob(std::string_view blob_data);
@@ -137,6 +143,7 @@ class BlobFormatWriter : public FormatWriter {
     std::shared_ptr metrics_;
     bool write_null_on_missing_file_ = false;
     bool write_null_on_fetch_failure_ = false;
+    bool write_placeholder_ = false;
     uint64_t null_on_missing_file_count_ = 0;
     uint64_t null_on_fetch_failure_count_ = 0;
     std::unique_ptr logger_;
diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp
index f530bda1..16c9e2af 100644
--- a/src/paimon/format/blob/blob_format_writer_test.cpp
+++ b/src/paimon/format/blob/blob_format_writer_test.cpp
@@ -131,7 +131,16 @@ class BlobFormatWriterTestBase : public ::testing::Test {
     Result> CreateDefaultWriter() const {
         return BlobFormatWriter::Create(output_stream_, struct_type_,
                                         /*write_null_on_missing_file=*/false,
-                                        /*write_null_on_fetch_failure=*/false, file_system_, pool_);
+                                        /*write_null_on_fetch_failure=*/false,
+                                        /*write_placeholder=*/false, file_system_, pool_);
+    }
+
+    /// Create a writer in placeholder mode, as used by data-evolution partial updates.
+    Result> CreatePlaceholderWriter() const {
+        return BlobFormatWriter::Create(output_stream_, struct_type_,
+                                        /*write_null_on_missing_file=*/false,
+                                        /*write_null_on_fetch_failure=*/false,
+                                        /*write_placeholder=*/true, file_system_, pool_);
     }
 
     Status AddBatchOnce(const std::shared_ptr& format_writer,
@@ -146,8 +155,7 @@ class BlobFormatWriterTestBase : public ::testing::Test {
         return paimon::test::TestHelper::MakeBlobDescriptorArray(struct_type_, blob, pool_);
     }
 
-    /// Build a single-row blob array holding `bytes` verbatim, for bytes no Blob can produce,
-    /// such as a truncated descriptor.
+    /// Build a single-row blob array holding `bytes` verbatim, bypassing the Blob helpers.
     Result> MakeBlobArrayFromBytes(const std::string& bytes) const {
         arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(),
                                             {std::make_shared()});
@@ -163,9 +171,11 @@ class BlobFormatWriterTestBase : public ::testing::Test {
     Result> ReadBackAsData() const {
         PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream,
                                file_system_->Open(dir_->Str() + "/file.blob"));
-        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader,
-                               BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024,
-                                                           /*blob_as_descriptor=*/false, pool_));
+        PAIMON_ASSIGN_OR_RAISE(
+            std::unique_ptr reader,
+            BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024,
+                                        /*blob_as_descriptor=*/false,
+                                        /*emit_placeholder_sentinel=*/false, pool_));
         auto schema = arrow::schema(struct_type_->fields());
         ::ArrowSchema c_schema;
         PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema));
@@ -255,7 +265,8 @@ TEST_P(BlobFormatWriterTest, TestSimple) {
     ASSERT_TRUE(input_stream);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr reader,
-        BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_));
+        BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_,
+                                    /*emit_placeholder_sentinel=*/false, pool_));
     auto schema = arrow::schema(struct_type_->fields());
     ::ArrowSchema c_schema;
     ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
@@ -286,41 +297,47 @@ TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) {
     // Test with nullptr output stream
     ASSERT_NOK_WITH_MSG(
         BlobFormatWriter::Create(nullptr, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/false, file_system_, pool_),
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, file_system_, pool_),
         "blob format writer create failed. out is nullptr");
 
     // Test with nullptr data type
     ASSERT_NOK_WITH_MSG(
         BlobFormatWriter::Create(output_stream_, nullptr, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/false, file_system_, pool_),
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, file_system_, pool_),
         "blob format writer create failed. data_type is nullptr");
 
     // Test with nullptr memory pool
     ASSERT_NOK_WITH_MSG(
         BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/false, file_system_, nullptr),
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, file_system_, nullptr),
         "blob format writer create failed. pool is nullptr");
 
     // Test with nullptr file system
     ASSERT_NOK_WITH_MSG(
         BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/false, nullptr, pool_),
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, nullptr, pool_),
         "blob format writer create failed. fs is nullptr");
 
     // Test with invalid field count (more than 1 field)
     auto multi_field_type = arrow::struct_(
         {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())});
-    ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(
-                            output_stream_, multi_field_type, /*write_null_on_missing_file=*/false,
-                            /*write_null_on_fetch_failure=*/false, file_system_, pool_),
+    ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, multi_field_type,
+                                                 /*write_null_on_missing_file=*/false,
+                                                 /*write_null_on_fetch_failure=*/false,
+                                                 /*write_placeholder=*/false, file_system_, pool_),
                         "blob data type field number 2 is not 1");
 
     // Test with non-blob field (missing blob metadata)
     auto non_blob_field = arrow::field("regular_col", arrow::binary());
     auto non_blob_type = arrow::struct_({non_blob_field});
-    ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(
-                            output_stream_, non_blob_type, /*write_null_on_missing_file=*/false,
-                            /*write_null_on_fetch_failure=*/false, file_system_, pool_),
+    ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, non_blob_type,
+                                                 /*write_null_on_missing_file=*/false,
+                                                 /*write_null_on_fetch_failure=*/false,
+                                                 /*write_placeholder=*/false, file_system_, pool_),
                         "field regular_col: binary is not BLOB");
 }
 
@@ -445,7 +462,8 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) {
                          file_system_->Open(dir_->Str() + "/file.blob"));
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr reader,
-        BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_));
+        BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_,
+                                    /*emit_placeholder_sentinel=*/false, pool_));
     auto schema = arrow::schema(struct_type_->fields());
     ::ArrowSchema c_schema;
     ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
@@ -494,7 +512,8 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) {
     ASSERT_TRUE(input_stream);
     ASSERT_OK_AND_ASSIGN(
         std::unique_ptr reader,
-        BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_));
+        BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_,
+                                    /*emit_placeholder_sentinel=*/false, pool_));
     auto schema = arrow::schema(struct_type_->fields());
     ::ArrowSchema c_schema;
     ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
@@ -527,7 +546,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnMissingFile) {
     ASSERT_OK_AND_ASSIGN(
         std::shared_ptr writer,
         BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/false, file_system_, pool_));
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, file_system_, pool_));
 
     ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob,
                          Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0,
@@ -565,7 +585,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnFetchFailure) {
     ASSERT_OK_AND_ASSIGN(
         std::shared_ptr writer,
         BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/true, file_system_, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, file_system_, pool_));
 
     std::string file = paimon::test::GetDataDir() + "/xxhash.data";
     ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob,
@@ -604,7 +625,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) {
     ASSERT_OK_AND_ASSIGN(
         std::shared_ptr writer,
         BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/true, file_system_, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, file_system_, pool_));
 
     // Row 0: missing file -> NULL.
     ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob,
@@ -669,7 +691,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/false, io_error_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, io_error_fs, pool_));
         ASSERT_OK(AddBatchOnce(writer, missing_array));
         ASSERT_EQ(io_error_fs->OpenCallCount(), 0);
     }
@@ -678,7 +701,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/false, io_error_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, io_error_fs, pool_));
         ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error");
     }
     // The same fetch failure, now converted to NULL.
@@ -686,7 +710,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/true, io_error_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, io_error_fs, pool_));
         ASSERT_OK(AddBatchOnce(writer, existing_array));
     }
     // Missing file with only fetch-failure enabled: no existence check runs, so the file is
@@ -697,7 +722,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/true, io_error_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, io_error_fs, pool_));
         ASSERT_OK(AddBatchOnce(writer, missing_array));
         ASSERT_EQ(io_error_fs->OpenCallCount(), open_calls_before + 1);
         ASSERT_OK_AND_ASSIGN(
@@ -715,7 +741,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, vanishing_fs, pool_));
         ASSERT_OK(AddBatchOnce(writer, existing_array));
         // One check before the open and one after it.
         ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2);
@@ -735,7 +762,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, vanishing_fs, pool_));
         ASSERT_OK(AddBatchOnce(writer, existing_array));
         ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2);
         ASSERT_OK_AND_ASSIGN(
@@ -754,7 +782,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, vanishing_fs, pool_));
         ASSERT_OK(AddBatchOnce(writer, existing_array));
         ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0);
     }
@@ -764,7 +793,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, vanishing_fs, pool_));
         ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error");
         ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0);
     }
@@ -787,14 +817,16 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnInvalidDescriptor) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/false, file_system_, pool_));
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, file_system_, pool_));
         ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "invalid blob descriptor");
     }
     {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/true, file_system_, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, file_system_, pool_));
         ASSERT_OK(AddBatchOnce(writer, array));
         ASSERT_OK(writer->Flush());
         ASSERT_OK(writer->Finish());
@@ -822,7 +854,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, exists_fail_fs, pool_));
         ASSERT_OK(AddBatchOnce(writer, array));
         ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 1);
         ASSERT_OK(writer->Flush());
@@ -852,7 +885,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/false, exists_fail_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/false,
+                                 /*write_placeholder=*/false, exists_fail_fs, pool_));
         // The reported failure names the check and keeps the underlying status message.
         Status check_status = AddBatchOnce(writer, array);
         ASSERT_NOK_WITH_MSG(check_status, "failed to check existence of blob file");
@@ -866,7 +900,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  output_stream_, struct_type_, /*write_null_on_missing_file=*/true,
-                                 /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, exists_fail_fs, pool_));
         ASSERT_OK(AddBatchOnce(writer, array));
         // One check before the open and one after it failed.
         ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 2);
@@ -886,7 +921,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) {
         ASSERT_OK_AND_ASSIGN(std::shared_ptr writer,
                              BlobFormatWriter::Create(
                                  side_stream, struct_type_, /*write_null_on_missing_file=*/false,
-                                 /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_));
+                                 /*write_null_on_fetch_failure=*/true,
+                                 /*write_placeholder=*/false, exists_fail_fs, pool_));
         ASSERT_OK(AddBatchOnce(writer, array));
         ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 0);
         ASSERT_OK_AND_ASSIGN(
@@ -933,4 +969,238 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) {
     ASSERT_EQ(buffer, expected);
 }
 
+/// Placeholder tests always feed the sentinel bytes of the placeholder write protocol, so
+/// they do not depend on the blob_as_descriptor_ parameter and run once on the
+/// non-parameterized fixture.
+using BlobFormatWriterPlaceholderTest = BlobFormatWriterTestBase;
+
+std::string PlaceholderSentinelBytes() {
+    return std::string(BlobDefs::PlaceholderSentinelView());
+}
+
+TEST_F(BlobFormatWriterPlaceholderTest, TestWritePlaceholderGoldenBytes) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter());
+
+    // row 0: inline bytes "inline"; row 1: null; row 2: placeholder
+    ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline"));
+    ASSERT_OK(AddBatchOnce(writer, inline_array));
+
+    arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(),
+                                        {std::make_shared()});
+    auto blob_builder = static_cast(struct_builder.field_builder(0));
+    ASSERT_TRUE(struct_builder.Append().ok());
+    ASSERT_TRUE(blob_builder->AppendNull().ok());
+    std::shared_ptr null_array;
+    ASSERT_TRUE(struct_builder.Finish(&null_array).ok());
+    ASSERT_OK(AddBatchOnce(writer, null_array));
+
+    ASSERT_OK_AND_ASSIGN(auto placeholder_array,
+                         MakeBlobArrayFromBytes(PlaceholderSentinelBytes()));
+    ASSERT_OK(AddBatchOnce(writer, placeholder_array));
+
+    ASSERT_OK(writer->Flush());
+    ASSERT_OK(writer->Finish());
+
+    // Verify byte-level alignment with the Java writer (BlobFormatWriterTest
+    // testRawBlobGoldenBytes): null and placeholder rows occupy no data bytes; the index
+    // records [22, -1, -2] as zigzag varint deltas [0x2c, 0x2d, 0x01].
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
+                         file_system_->Open(dir_->Str() + "/file.blob"));
+    ASSERT_TRUE(input_stream);
+    ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length());
+    ASSERT_EQ(file_length, 30);
+    std::vector buffer(file_length);
+    ASSERT_OK_AND_ASSIGN(auto read_length,
+                         input_stream->Read(reinterpret_cast(buffer.data()), buffer.size()));
+    ASSERT_EQ(read_length, 30);
+    std::vector expected = {{// record 0: magic + "inline" + bin_length(22) + crc32
+                                      0xcf, 0x11, 0x4e, 0x58, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65,
+                                      0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x60,
+                                      0xc8, 0xe9,
+                                      // index of [22, -1, -2]
+                                      0x2c, 0x2d, 0x01,
+                                      // footer: index length + version
+                                      0x03, 0x00, 0x00, 0x00, 0x01}};
+    ASSERT_EQ(buffer, expected);
+}
+
+TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderStrictAndAwareModes) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter());
+    ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline"));
+    ASSERT_OK(AddBatchOnce(writer, inline_array));
+    ASSERT_OK_AND_ASSIGN(auto placeholder_array,
+                         MakeBlobArrayFromBytes(PlaceholderSentinelBytes()));
+    ASSERT_OK(AddBatchOnce(writer, placeholder_array));
+    ASSERT_OK(writer->Flush());
+    ASSERT_OK(writer->Finish());
+
+    auto schema = arrow::schema(struct_type_->fields());
+
+    // default (strict) mode: reading a placeholder entry fails
+    {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
+                             file_system_->Open(dir_->Str() + "/file.blob"));
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr reader,
+            BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024,
+                                        /*blob_as_descriptor=*/false,
+                                        /*emit_placeholder_sentinel=*/false, pool_));
+        ::ArrowSchema c_schema;
+        ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+        ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr,
+                                        /*selection_bitmap=*/std::nullopt));
+        ASSERT_NOK_WITH_MSG(reader->NextBatch(), "placeholder");
+    }
+
+    // placeholder-aware mode: the entry is returned as the sentinel bytes
+    {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
+                             file_system_->Open(dir_->Str() + "/file.blob"));
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr reader,
+            BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024,
+                                        /*blob_as_descriptor=*/false,
+                                        /*emit_placeholder_sentinel=*/true, pool_));
+        ::ArrowSchema c_schema;
+        ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+        ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr,
+                                        /*selection_bitmap=*/std::nullopt));
+        ASSERT_OK_AND_ASSIGN(auto chunked_array,
+                             paimon::test::ReadResultCollector::CollectResult(reader.get()));
+        auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie();
+        auto struct_array = arrow::internal::checked_pointer_cast(concat_array);
+        ASSERT_EQ(struct_array->length(), 2);
+        auto binary_array =
+            arrow::internal::checked_pointer_cast(struct_array->field(0));
+        ASSERT_EQ(binary_array->GetString(0), "inline");
+        ASSERT_FALSE(binary_array->IsNull(1));
+        ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes());
+    }
+
+    // placeholder-aware descriptor mode also returns the sentinel bytes
+    {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
+                             file_system_->Open(dir_->Str() + "/file.blob"));
+        ASSERT_OK_AND_ASSIGN(
+            std::unique_ptr reader,
+            BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024,
+                                        /*blob_as_descriptor=*/true,
+                                        /*emit_placeholder_sentinel=*/true, pool_));
+        ::ArrowSchema c_schema;
+        ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+        ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr,
+                                        /*selection_bitmap=*/std::nullopt));
+        ASSERT_OK_AND_ASSIGN(auto chunked_array,
+                             paimon::test::ReadResultCollector::CollectResult(reader.get()));
+        auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie();
+        auto struct_array = arrow::internal::checked_pointer_cast(concat_array);
+        auto binary_array =
+            arrow::internal::checked_pointer_cast(struct_array->field(0));
+        ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes());
+    }
+}
+
+TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderWithSelectionBitmap) {
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter());
+    ASSERT_OK_AND_ASSIGN(auto array0, MakeBlobArrayFromBytes("first"));
+    ASSERT_OK(AddBatchOnce(writer, array0));
+    ASSERT_OK_AND_ASSIGN(auto array1, MakeBlobArrayFromBytes(PlaceholderSentinelBytes()));
+    ASSERT_OK(AddBatchOnce(writer, array1));
+    ASSERT_OK_AND_ASSIGN(auto array2, MakeBlobArrayFromBytes("third"));
+    ASSERT_OK(AddBatchOnce(writer, array2));
+    ASSERT_OK(writer->Flush());
+    ASSERT_OK(writer->Finish());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
+                         file_system_->Open(dir_->Str() + "/file.blob"));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr reader,
+                         BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024,
+                                                     /*blob_as_descriptor=*/false,
+                                                     /*emit_placeholder_sentinel=*/true, pool_));
+    auto schema = arrow::schema(struct_type_->fields());
+    ::ArrowSchema c_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+    RoaringBitmap32 selection;
+    selection.Add(1);
+    selection.Add(2);
+    ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, selection));
+    ASSERT_OK_AND_ASSIGN(auto chunked_array,
+                         paimon::test::ReadResultCollector::CollectResult(reader.get()));
+    auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie();
+    auto struct_array = arrow::internal::checked_pointer_cast(concat_array);
+    ASSERT_EQ(struct_array->length(), 2);
+    auto binary_array =
+        arrow::internal::checked_pointer_cast(struct_array->field(0));
+    ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes());
+    ASSERT_EQ(binary_array->GetString(1), "third");
+}
+
+TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelBytesVerbatimWithoutPlaceholderMode) {
+    // outside placeholder mode a user blob whose bytes equal the sentinel is a normal value: it
+    // must be stored as a real entry (not persisted as bin_length -2) and read back unchanged
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter());
+    ASSERT_OK_AND_ASSIGN(auto sentinel_array, MakeBlobArrayFromBytes(PlaceholderSentinelBytes()));
+    ASSERT_OK(AddBatchOnce(writer, sentinel_array));
+    ASSERT_OK(writer->Flush());
+    ASSERT_OK(writer->Finish());
+
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
+                         file_system_->Open(dir_->Str() + "/file.blob"));
+    ASSERT_OK_AND_ASSIGN(std::unique_ptr reader,
+                         BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024,
+                                                     /*blob_as_descriptor=*/false,
+                                                     /*emit_placeholder_sentinel=*/false, pool_));
+    auto schema = arrow::schema(struct_type_->fields());
+    ::ArrowSchema c_schema;
+    ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+    ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr,
+                                    /*selection_bitmap=*/std::nullopt));
+    ASSERT_OK_AND_ASSIGN(auto chunked_array,
+                         paimon::test::ReadResultCollector::CollectResult(reader.get()));
+    auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie();
+    auto struct_array = arrow::internal::checked_pointer_cast(concat_array);
+    ASSERT_EQ(struct_array->length(), 1);
+    auto binary_array =
+        arrow::internal::checked_pointer_cast(struct_array->field(0));
+    ASSERT_FALSE(binary_array->IsNull(0));
+    ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes());
+}
+
+TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelPrefixedValueVerbatimInPlaceholderMode) {
+    // placeholders are identified by exact equality only: even in placeholder mode a real
+    // value that merely starts with the sentinel bytes is stored verbatim and read back
+    // unchanged in both strict and placeholder-aware modes
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter());
+    std::string sentinel = PlaceholderSentinelBytes();
+    ASSERT_OK_AND_ASSIGN(auto doubled_sentinel_array, MakeBlobArrayFromBytes(sentinel + sentinel));
+    ASSERT_OK(AddBatchOnce(writer, doubled_sentinel_array));
+    ASSERT_OK_AND_ASSIGN(auto prefixed_array, MakeBlobArrayFromBytes(sentinel + "suffix"));
+    ASSERT_OK(AddBatchOnce(writer, prefixed_array));
+    ASSERT_OK(writer->Flush());
+    ASSERT_OK(writer->Finish());
+
+    auto schema = arrow::schema(struct_type_->fields());
+    for (bool emit_placeholder_sentinel : {false, true}) {
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream,
+                             file_system_->Open(dir_->Str() + "/file.blob"));
+        ASSERT_OK_AND_ASSIGN(std::unique_ptr reader,
+                             BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024,
+                                                         /*blob_as_descriptor=*/false,
+                                                         emit_placeholder_sentinel, pool_));
+        ::ArrowSchema c_schema;
+        ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok());
+        ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr,
+                                        /*selection_bitmap=*/std::nullopt));
+        ASSERT_OK_AND_ASSIGN(auto chunked_array,
+                             paimon::test::ReadResultCollector::CollectResult(reader.get()));
+        auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie();
+        auto struct_array = arrow::internal::checked_pointer_cast(concat_array);
+        auto binary_array =
+            arrow::internal::checked_pointer_cast(struct_array->field(0));
+        ASSERT_EQ(struct_array->length(), 2);
+        ASSERT_EQ(binary_array->GetString(0), sentinel + sentinel);
+        ASSERT_EQ(binary_array->GetString(1), sentinel + "suffix");
+    }
+}
+
 }  // namespace paimon::blob::test
diff --git a/src/paimon/format/blob/blob_reader_builder.h b/src/paimon/format/blob/blob_reader_builder.h
index 85bebe96..46bb1d25 100644
--- a/src/paimon/format/blob/blob_reader_builder.h
+++ b/src/paimon/format/blob/blob_reader_builder.h
@@ -45,7 +45,11 @@ class BlobReaderBuilder : public ReaderBuilder {
         PAIMON_ASSIGN_OR_RAISE(
             bool blob_as_descriptor,
             OptionsUtils::GetValueFromMap(options_, Options::BLOB_AS_DESCRIPTOR, false));
-        return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, pool_);
+        PAIMON_ASSIGN_OR_RAISE(bool emit_placeholder_sentinel,
+                               OptionsUtils::GetValueFromMap(
+                                   options_, BlobDefs::kEmitPlaceholderSentinelKey, false));
+        return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor,
+                                           emit_placeholder_sentinel, pool_);
     }
 
  private:
diff --git a/src/paimon/format/blob/blob_stats_extractor.cpp b/src/paimon/format/blob/blob_stats_extractor.cpp
index 4b5845c5..ca671e4e 100644
--- a/src/paimon/format/blob/blob_stats_extractor.cpp
+++ b/src/paimon/format/blob/blob_stats_extractor.cpp
@@ -50,10 +50,13 @@ BlobStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& file_
             fmt::format("field {} is not BLOB", write_schema_->field(0)->ToString()));
     }
 
+    // The reader only serves footer metadata (GetNumberOfRows); NextBatch is never called, so
+    // the strict placeholder mode is irrelevant even for files containing placeholder entries.
     PAIMON_ASSIGN_OR_RAISE(
         std::unique_ptr blob_reader,
         BlobFileBatchReader::Create(input_stream,
-                                    /*batch_size=*/1024, /*blob_as_descriptor=*/true, pool));
+                                    /*batch_size=*/1024, /*blob_as_descriptor=*/true,
+                                    /*emit_placeholder_sentinel=*/false, pool));
     ColumnStatsVector result_stats;
     result_stats.push_back(
         ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt));
diff --git a/src/paimon/format/blob/blob_writer_builder.h b/src/paimon/format/blob/blob_writer_builder.h
index dfd09c00..55254534 100644
--- a/src/paimon/format/blob/blob_writer_builder.h
+++ b/src/paimon/format/blob/blob_writer_builder.h
@@ -26,6 +26,7 @@
 #include 
 
 #include "arrow/api.h"
+#include "paimon/common/data/blob_defs.h"
 #include "paimon/common/utils/options_utils.h"
 #include "paimon/defs.h"
 #include "paimon/format/blob/blob_format_writer.h"
@@ -75,8 +76,11 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder {
         PAIMON_ASSIGN_OR_RAISE(bool write_null_on_fetch_failure,
                                OptionsUtils::GetValueFromMap(
                                    options_, Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, false));
+        PAIMON_ASSIGN_OR_RAISE(
+            bool write_placeholder,
+            OptionsUtils::GetValueFromMap(options_, BlobDefs::kWritePlaceholderKey, false));
         return BlobFormatWriter::Create(out, data_type_, write_null_on_missing_file,
-                                        write_null_on_fetch_failure, fs_, pool_);
+                                        write_null_on_fetch_failure, write_placeholder, fs_, pool_);
     }
 
  private:
diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp
index 85c73ef7..8e0add5a 100644
--- a/test/inte/blob_table_inte_test.cpp
+++ b/test/inte/blob_table_inte_test.cpp
@@ -44,6 +44,7 @@
 #include "paimon/common/data/binary_array_writer.h"
 #include "paimon/common/data/binary_row.h"
 #include "paimon/common/data/binary_row_writer.h"
+#include "paimon/common/data/blob_defs.h"
 #include "paimon/common/data/blob_descriptor.h"
 #include "paimon/common/data/blob_utils.h"
 #include "paimon/common/data/blob_view_struct.h"
@@ -1107,6 +1108,595 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) {
     ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id));
 }
 
+/// Build a single-blob-column StructArray for a data-evolution partial update: "PH" marks a row
+/// whose blob is not updated (persisted as a placeholder entry), std::nullopt a null blob.
+std::shared_ptr MakeBlobUpdateArray(
+    const std::shared_ptr& blob_field,
+    const std::vector>& rows) {
+    auto struct_type = arrow::struct_({blob_field});
+    arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(),
+                                        {std::make_shared()});
+    auto blob_builder = static_cast(struct_builder.field_builder(0));
+    for (const auto& row : rows) {
+        EXPECT_TRUE(struct_builder.Append().ok());
+        if (!row) {
+            EXPECT_TRUE(blob_builder->AppendNull().ok());
+        } else if (*row == "PH") {
+            std::string_view sentinel = BlobDefs::PlaceholderSentinelView();
+            EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok());
+        } else {
+            EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok());
+        }
+    }
+    std::shared_ptr array;
+    EXPECT_TRUE(struct_builder.Finish(&array).ok());
+    return std::dynamic_pointer_cast(array);
+}
+
+TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) {
+    // the blob column is updated to null below, so it must be nullable
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 arrow::field("f1", arrow::utf8()),
+                                 BlobUtils::ToArrowField("b0", /*nullable=*/true)};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+    auto schema = arrow::schema(fields);
+
+    // Initial full-row write assigns row ids 0-2.
+    auto src_array0 = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [1, "a", "blob_a"],
+        [2, "b", "blob_b"],
+        [3, "c", "blob_c"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0,
+                         WriteArray(table_path, {}, schema->field_names(), {src_array0}));
+    ASSERT_OK(Commit(table_path, commit_msgs0));
+
+    // Partial update: only row 1 gets a new blob, the untouched rows are written as
+    // placeholder entries and must fall back to the previous blob file when read.
+    auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b", "PH"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    auto expected_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [1, "a", "blob_a"],
+        [2, "b", "updated_b"],
+        [3, "c", "blob_c"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
+
+    // updating a blob to null is not a placeholder: the null must win over older layers
+    auto null_update_array = MakeBlobUpdateArray(fields[2], {std::nullopt, "PH", "PH"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs2,
+                         WriteArray(table_path, {}, {"b0"}, {null_update_array}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs2);
+    ASSERT_OK(Commit(table_path, commit_msgs2));
+
+    auto expected_array2 = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [1, "a", null],
+        [2, "b", "updated_b"],
+        [3, "c", "blob_c"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array2));
+
+    // row ids still come from the data files and stay aligned with the fallback result
+    auto expected_with_row_id = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}),
+            R"([
+        [1, "a", null, 0],
+        [2, "b", "updated_b", 1],
+        [3, "c", "blob_c", 2]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id));
+
+    // blob_as_descriptor read mode: the fallback-merged values come back as descriptors and
+    // must still resolve to the same bytes
+    ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path));
+    std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}};
+    ASSERT_OK_AND_ASSIGN(auto desc_result, ReadTable(table_path, schema->field_names(), plan,
+                                                     /*predicate=*/nullptr, read_options));
+    ASSERT_TRUE(desc_result.chunked_array);
+    auto desc_concat = arrow::Concatenate(desc_result.chunked_array->chunks()).ValueOrDie();
+    auto desc_struct = std::dynamic_pointer_cast(desc_concat);
+    ASSERT_TRUE(desc_struct);
+    ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(desc_struct, {"b0"}));
+    ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array2));
+    ASSERT_TRUE(resolved->Equals(expected_with_rk))
+        << "result:" << resolved->ToString() << "\nexpected:" << expected_with_rk->ToString();
+}
+
+TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+    auto schema = arrow::schema(fields);
+
+    auto src_array0 = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [1, "a", "blob_0"],
+        [2, "b", "blob_1"],
+        [3, "c", "blob_2"],
+        [4, "d", "blob_3"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0,
+                         WriteArray(table_path, {}, schema->field_names(), {src_array0}));
+    ASSERT_OK(Commit(table_path, commit_msgs0));
+
+    // second layer updates row 0 within rows [0, 1]
+    auto update_array1 = MakeBlobUpdateArray(fields[2], {"update1_0", "PH"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    // third layer updates row 3 within rows [2, 3]; each layer only partially covers the
+    // range, the uncovered parts behave as placeholders
+    auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "update2_3"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2}));
+    SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs2);
+    ASSERT_OK(Commit(table_path, commit_msgs2));
+
+    auto expected_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [1, "a", "update1_0"],
+        [2, "b", "blob_1"],
+        [3, "c", "blob_2"],
+        [4, "d", "update2_3"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
+
+    // row-range pushdown where the selected rows fall in one layer's file and in another
+    // layer's uncovered gap: row 1 is a gap row for the third layer, row 2 for the second
+    auto expected_middle = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [2, "b", "blob_1"],
+        [3, "c", "blob_2"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_middle,
+                          /*predicate=*/nullptr, /*row_ranges=*/{Range(1, 2)}));
+
+    // per-row reads resolve every row independently
+    const std::vector expected_rows = {
+        R"([[1, "a", "update1_0"]])", R"([[2, "b", "blob_1"]])", R"([[3, "c", "blob_2"]])",
+        R"([[4, "d", "update2_3"]])"};
+    for (int32_t i = 0; i < static_cast(expected_rows.size()); i++) {
+        auto expected_single = std::dynamic_pointer_cast(
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_rows[i])
+                .ValueOrDie());
+        ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single,
+                              /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)}));
+    }
+}
+
+TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+    auto schema = arrow::schema(fields);
+
+    // Mirrors the layer shapes of Java's BlobUpdateTest.testReadCompactedBlobSequenceGroups
+    // (with a single base write, as the C++ write path assigns one sequence per commit):
+    //   row id:  0    1    2    3    4    5    6    7    8    9
+    //   seq1:   [b0   b1   b2   b3   b4   b5   b6   b7   b8   b9]
+    //   seq2:   [u20  P    P    u23  u24] .    .    .    .    .
+    //   seq3:    .    .    .    .    .   [P    u46  P    u48  P]
+    //   seq4:   [P    u61  P    P    P    P    P    P    P    u69]
+    //   result:  u20  u61  b2   u23  u24  b5   u46  b7   u48  u69
+    auto base_array = PrepareBulkData(
+        10,
+        [](int32_t i) {
+            return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" +
+                   std::to_string(i) + "\"";
+        },
+        fields);
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0,
+                         WriteArray(table_path, {}, schema->field_names(), {base_array}));
+    ASSERT_OK(Commit(table_path, commit_msgs0));
+
+    auto update_array1 = MakeBlobUpdateArray(fields[2], {"u20", "PH", "PH", "u23", "u24"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "u46", "PH", "u48", "PH"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2}));
+    SetFirstRowId(/*reset_first_row_id=*/5, commit_msgs2);
+    ASSERT_OK(Commit(table_path, commit_msgs2));
+
+    auto update_array3 = MakeBlobUpdateArray(
+        fields[2], {"PH", "u61", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "u69"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs3, WriteArray(table_path, {}, {"b0"}, {update_array3}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs3);
+    ASSERT_OK(Commit(table_path, commit_msgs3));
+
+    const std::vector expected_blobs = {"u20",    "u61", "blob_2", "u23", "u24",
+                                                     "blob_5", "u46", "blob_7", "u48", "u69"};
+    auto expected_row = [&](int32_t i) {
+        return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"" + expected_blobs[i] +
+               "\"";
+    };
+    auto expected_full = PrepareBulkData(10, expected_row, fields);
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full));
+
+    // every row resolves independently under row-range pushdown
+    for (int32_t i = 0; i < 10; i++) {
+        auto expected_single = std::dynamic_pointer_cast(
+            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields),
+                                                      "[[" + expected_row(i) + "]]")
+                .ValueOrDie());
+        ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single,
+                              /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)}));
+    }
+}
+
+TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+    auto schema = arrow::schema(fields);
+
+    auto base_array = PrepareBulkData(
+        10,
+        [](int32_t i) {
+            return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" +
+                   std::to_string(i) + "\"";
+        },
+        fields);
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0,
+                         WriteArray(table_path, {}, schema->field_names(), {base_array}));
+    ASSERT_OK(Commit(table_path, commit_msgs0));
+
+    // partial update touching rows 1 and 9 only
+    auto update_array = MakeBlobUpdateArray(
+        fields[2], {"PH", "update_1", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "update_9"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    // push down row ranges hitting updated and untouched rows
+    auto expected_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [1, "name_1", "update_1"],
+        [5, "name_5", "blob_5"],
+        [9, "name_9", "update_9"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array,
+                          /*predicate=*/nullptr,
+                          /*row_ranges=*/{Range(1, 1), Range(5, 5), Range(9, 9)}));
+
+    // full read still resolves every row
+    auto expected_full = PrepareBulkData(
+        10,
+        [](int32_t i) {
+            std::string blob = (i == 1 || i == 9) ? "\"update_" + std::to_string(i) + "\""
+                                                  : "\"blob_" + std::to_string(i) + "\"";
+            return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", " + blob;
+        },
+        fields);
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full));
+}
+
+/// A BLOB sequence layer whose physical file covers only a strict subrange of the full row id
+/// range: the newest group is internally Gap(2), File([u2, u3]), Gap(6). The table needs a
+/// normal column to be creatable, but only the blob column is ever written, so the split
+/// contains only blob files and the blob bunch itself carries the row-tracking fields: the
+/// fallback reader must keep _ROW_ID correct and report each row's _SEQUENCE_NUMBER from the
+/// layer that resolved it, without ever exposing the internal placeholder sentinel.
+TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubrangeLayer) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 BlobUtils::ToArrowField("b0", /*nullable=*/true)};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+
+    // old layer: rows 0-9 all real
+    std::vector> base_rows;
+    for (int32_t i = 0; i < 10; i++) {
+        base_rows.emplace_back("b" + std::to_string(i));
+    }
+    auto base_array = MakeBlobUpdateArray(fields[1], base_rows);
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0);
+    ASSERT_OK(Commit(table_path, commit_msgs0));
+
+    // new layer: first_row_id=2, row_count=2, covering only rows 2-3
+    auto update_array = MakeBlobUpdateArray(fields[1], {"u2", "u3"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array}));
+    SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    ASSERT_OK_AND_ASSIGN(auto scan_read,
+                         ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"}));
+    ASSERT_TRUE(scan_read.chunked_array);
+    auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie();
+    auto struct_array = std::dynamic_pointer_cast(concat_array);
+    ASSERT_TRUE(struct_array);
+    ASSERT_EQ(struct_array->length(), 10);
+    auto blob_col =
+        std::dynamic_pointer_cast(struct_array->GetFieldByName("b0"));
+    auto row_id_col = std::dynamic_pointer_cast(
+        struct_array->GetFieldByName(SpecialFields::RowId().Name()));
+    auto seq_col = std::dynamic_pointer_cast(
+        struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name()));
+    ASSERT_TRUE(blob_col && row_id_col && seq_col);
+    int64_t old_layer_seq = seq_col->Value(0);
+    int64_t new_layer_seq = seq_col->Value(2);
+    ASSERT_GT(new_layer_seq, old_layer_seq);
+    for (int64_t i = 0; i < 10; i++) {
+        ASSERT_FALSE(blob_col->IsNull(i));
+        std::string expected_blob =
+            (i == 2 || i == 3) ? "u" + std::to_string(i) : "b" + std::to_string(i);
+        // the leading and trailing gaps never expose the internal placeholder sentinel
+        ASSERT_EQ(blob_col->GetString(i), expected_blob) << "row " << i;
+        ASSERT_EQ(row_id_col->Value(i), i);
+        ASSERT_EQ(seq_col->Value(i), (i == 2 || i == 3) ? new_layer_seq : old_layer_seq)
+            << "row " << i;
+    }
+
+    // row-range pushdown with the row-tracking projection: the selection removes rows inside
+    // the blob files, so batch positions must still map back to the right file row indexes
+    ASSERT_OK_AND_ASSIGN(auto range_read,
+                         ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                                           /*predicate=*/nullptr,
+                                           /*row_ranges=*/{Range(1, 2), Range(8, 8)}));
+    ASSERT_TRUE(range_read.chunked_array);
+    auto range_concat = arrow::Concatenate(range_read.chunked_array->chunks()).ValueOrDie();
+    auto range_struct = std::dynamic_pointer_cast(range_concat);
+    ASSERT_TRUE(range_struct);
+    ASSERT_EQ(range_struct->length(), 3);
+    auto range_blob_col =
+        std::dynamic_pointer_cast(range_struct->GetFieldByName("b0"));
+    auto range_row_id_col = std::dynamic_pointer_cast(
+        range_struct->GetFieldByName(SpecialFields::RowId().Name()));
+    auto range_seq_col = std::dynamic_pointer_cast(
+        range_struct->GetFieldByName(SpecialFields::SequenceNumber().Name()));
+    ASSERT_TRUE(range_blob_col && range_row_id_col && range_seq_col);
+    const std::vector expected_row_ids = {1, 2, 8};
+    const std::vector expected_blobs = {"b1", "u2", "b8"};
+    for (int64_t i = 0; i < 3; i++) {
+        ASSERT_EQ(range_blob_col->GetString(i), expected_blobs[i]) << "row " << i;
+        ASSERT_EQ(range_row_id_col->Value(i), expected_row_ids[i]) << "row " << i;
+        ASSERT_EQ(range_seq_col->Value(i), expected_row_ids[i] == 2 ? new_layer_seq : old_layer_seq)
+            << "row " << i;
+    }
+}
+
+/// A row that is a placeholder in every BLOB layer degrades to a null blob but keeps its
+/// _ROW_ID and reports -1 as its _SEQUENCE_NUMBER. Only the blob column is ever written, so
+/// the blob bunch itself carries the row-tracking fields.
+TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateAllPlaceholderRowTracking) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 BlobUtils::ToArrowField("b0", /*nullable=*/true)};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+
+    // the base layer itself holds a placeholder at row 1, so after the second layer also
+    // leaves it untouched, row 1 is a placeholder in every layer
+    auto base_array = MakeBlobUpdateArray(fields[1], {"blob_a", "PH", "blob_c"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0);
+    ASSERT_OK(Commit(table_path, commit_msgs0));
+
+    auto update_array = MakeBlobUpdateArray(fields[1], {"PH", "PH", "update_c"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    ASSERT_OK_AND_ASSIGN(auto scan_read,
+                         ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"}));
+    ASSERT_TRUE(scan_read.chunked_array);
+    auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie();
+    auto struct_array = std::dynamic_pointer_cast(concat_array);
+    ASSERT_TRUE(struct_array);
+    ASSERT_EQ(struct_array->length(), 3);
+    auto blob_col =
+        std::dynamic_pointer_cast(struct_array->GetFieldByName("b0"));
+    auto row_id_col = std::dynamic_pointer_cast(
+        struct_array->GetFieldByName(SpecialFields::RowId().Name()));
+    auto seq_col = std::dynamic_pointer_cast(
+        struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name()));
+    ASSERT_TRUE(blob_col && row_id_col && seq_col);
+
+    ASSERT_EQ(blob_col->GetString(0), "blob_a");
+    ASSERT_EQ(blob_col->GetString(2), "update_c");
+    // the all-placeholder row: null blob, row id kept, sequence number -1
+    ASSERT_TRUE(blob_col->IsNull(1));
+    ASSERT_EQ(row_id_col->Value(1), 1);
+    ASSERT_EQ(seq_col->Value(1), -1);
+    for (int64_t i : {static_cast(0), static_cast(2)}) {
+        ASSERT_EQ(row_id_col->Value(i), i);
+        ASSERT_GE(seq_col->Value(i), 0);
+    }
+    ASSERT_GT(seq_col->Value(2), seq_col->Value(0));
+}
+
+/// A normal (full-row) write never interprets blob bytes: a user value whose bytes exactly
+/// equal the placeholder sentinel must be stored verbatim (not persisted as a bin_length -2
+/// entry) and read back unchanged. The sentinel is reserved only inside the data-evolution
+/// partial-update channels (see BlobDefs::kPlaceholderSentinel).
+TEST_P(BlobTableInteTest, TestBlobValueEqualToPlaceholderSentinelBytes) {
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 arrow::field("f1", arrow::utf8()),
+                                 BlobUtils::ToArrowField("b0", /*nullable=*/true)};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+    auto schema = arrow::schema(fields);
+    std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView());
+
+    // full-row write: row 0's blob is exactly the sentinel bytes
+    auto struct_type = arrow::struct_(fields);
+    arrow::StructBuilder struct_builder(
+        struct_type, arrow::default_memory_pool(),
+        {std::make_shared(), std::make_shared(),
+         std::make_shared()});
+    auto f0_builder = static_cast(struct_builder.field_builder(0));
+    auto f1_builder = static_cast(struct_builder.field_builder(1));
+    auto b0_builder = static_cast(struct_builder.field_builder(2));
+    ASSERT_TRUE(struct_builder.Append().ok());
+    ASSERT_TRUE(f0_builder->Append(1).ok());
+    ASSERT_TRUE(f1_builder->Append("a").ok());
+    ASSERT_TRUE(b0_builder->Append(sentinel_bytes.data(), sentinel_bytes.size()).ok());
+    ASSERT_TRUE(struct_builder.Append().ok());
+    ASSERT_TRUE(f0_builder->Append(2).ok());
+    ASSERT_TRUE(f1_builder->Append("b").ok());
+    ASSERT_TRUE(b0_builder->Append("normal", 6).ok());
+    std::shared_ptr src_array;
+    ASSERT_TRUE(struct_builder.Finish(&src_array).ok());
+    auto src_struct = std::dynamic_pointer_cast(src_array);
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0,
+                         WriteArray(table_path, {}, schema->field_names(), {src_struct}));
+    ASSERT_OK(Commit(table_path, commit_msgs0));
+
+    // read back unchanged: the sentinel-equal bytes are a normal value
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_struct));
+
+    // a value that merely starts with the sentinel bytes stays a normal value even through a
+    // partial-update fallback (placeholders are identified by exact equality only)
+    std::string prefixed_bytes = sentinel_bytes + "suffix";
+    auto update_array = MakeBlobUpdateArray(fields[2], {prefixed_bytes, "updated_b"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    ASSERT_OK_AND_ASSIGN(auto scan_read, ScanAndReadResult(table_path, schema->field_names()));
+    ASSERT_TRUE(scan_read.chunked_array);
+    auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie();
+    auto struct_result = std::dynamic_pointer_cast(concat_array);
+    ASSERT_TRUE(struct_result);
+    ASSERT_EQ(struct_result->length(), 2);
+    auto blob_col =
+        std::dynamic_pointer_cast(struct_result->GetFieldByName("b0"));
+    ASSERT_TRUE(blob_col);
+    ASSERT_FALSE(blob_col->IsNull(0));
+    ASSERT_EQ(blob_col->GetString(0), prefixed_bytes);
+    ASSERT_EQ(blob_col->GetString(1), "updated_b");
+}
+
+TEST_P(BlobTableInteTest, TestBlobSentinelValueInBaseLayerDegradesToNull) {
+    // Pins the accepted collision of the byte-identified placeholder protocol (see
+    // BlobDefs::kPlaceholderSentinel): the fallback merge byte-compares every layer, so a user
+    // blob equal to the sentinel bytes that a later partial update leaves untouched reads as a
+    // placeholder in every layer and degrades to a null blob instead of falling back.
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 arrow::field("f1", arrow::utf8()),
+                                 BlobUtils::ToArrowField("b0", /*nullable=*/true)};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+    auto schema = arrow::schema(fields);
+    std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView());
+
+    // the full-row write stores row 0's sentinel-equal bytes verbatim (the write channel is off)
+    std::string src_json =
+        std::string(R"([[1, "a", ")") + sentinel_bytes + R"("], [2, "b", "blob_b"]])";
+    auto src_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), src_json).ValueOrDie());
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs0,
+                         WriteArray(table_path, {}, schema->field_names(), {src_array}));
+    ASSERT_OK(Commit(table_path, commit_msgs0));
+
+    // the partial update leaves row 0 untouched, so its base value collides with the
+    // placeholder markers of the newer layer
+    auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b"});
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array}));
+    SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1);
+    ASSERT_OK(Commit(table_path, commit_msgs1));
+
+    // row 0's non-blob fields survive (they come from the data file); only the blob degrades
+    auto expected_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+        [1, "a", null],
+        [2, "b", "updated_b"]
+    ])")
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
+}
+
+TEST_P(BlobTableInteTest, TestUserSuppliedInternalPlaceholderOptionsIgnored) {
+    // blob.internal.* options are reserved for the data-evolution write and read paths. Set in
+    // the user table options they must be ignored: were the write key honored, this full-row
+    // write would persist the sentinel-equal user value as a placeholder entry that no older
+    // layer can resolve, making the table unreadable.
+    arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
+                                 arrow::field("f1", arrow::utf8()),
+                                 BlobUtils::ToArrowField("b0", /*nullable=*/true)};
+    std::map options = {{Options::MANIFEST_FORMAT, "orc"},
+                                                  {Options::FILE_FORMAT, GetParam()},
+                                                  {Options::FILE_SYSTEM, "local"},
+                                                  {Options::ROW_TRACKING_ENABLED, "true"},
+                                                  {Options::DATA_EVOLUTION_ENABLED, "true"},
+                                                  {BlobDefs::kWritePlaceholderKey, "true"},
+                                                  {BlobDefs::kEmitPlaceholderSentinelKey, "true"}};
+    CreateTable(fields, /*partition_keys=*/{}, options);
+    std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+    auto schema = arrow::schema(fields);
+    std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView());
+
+    std::string src_json =
+        std::string(R"([[1, "a", ")") + sentinel_bytes + R"("], [2, "b", "blob_b"]])";
+    auto src_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), src_json).ValueOrDie());
+    ASSERT_OK_AND_ASSIGN(auto commit_msgs,
+                         WriteArray(table_path, {}, schema->field_names(), {src_array}));
+    ASSERT_OK(Commit(table_path, commit_msgs));
+
+    // the sentinel-equal value round-trips verbatim: the user-supplied keys were stripped
+    ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_array));
+}
+
 TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) {
     arrow::FieldVector fields = {arrow::field("f0", arrow::int32()),
                                  arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};

From 2b74264d349b0cbb69822d2a0f35d2a99ddd1b93 Mon Sep 17 00:00:00 2001
From: Zhou Hongfeng <87103887+zhf999@users.noreply.github.com>
Date: Fri, 31 Jul 2026 11:35:44 +0800
Subject: [PATCH 132/138] feat(parquet): support configuring Arrow pre-buffer
 hole-size-limit for range coalescing

---
 .../parquet/parquet_file_batch_reader.cpp     | 16 ++++++++
 .../parquet_file_batch_reader_test.cpp        | 38 ++++++++++++++++++-
 .../format/parquet/parquet_format_defs.h      |  7 ++++
 3 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
index 28dadeea..4f1d013f 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
@@ -618,13 +618,29 @@ Result<::parquet::ArrowReaderProperties> ParquetFileBatchReader::CreateArrowRead
         int64_t cache_prefetch_limit,
         OptionsUtils::GetValueFromMap(options, PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT,
                                                DEFAULT_PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT));
+    PAIMON_ASSIGN_OR_RAISE(
+        int64_t cache_hole_size_limit,
+        OptionsUtils::GetValueFromMap(options, PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT,
+                                               DEFAULT_PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT));
     PAIMON_ASSIGN_OR_RAISE(
         int64_t cache_range_size_limit,
         OptionsUtils::GetValueFromMap(options, PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT,
                                                DEFAULT_PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT));
+    if (cache_hole_size_limit < 0) {
+        return Status::Invalid(fmt::format("{} must be non-negative, but was {}",
+                                           PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT,
+                                           cache_hole_size_limit));
+    }
+    if (cache_range_size_limit <= cache_hole_size_limit) {
+        return Status::Invalid(fmt::format("{} must be greater than {}, but was {} <= {}",
+                                           PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT,
+                                           PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT,
+                                           cache_range_size_limit, cache_hole_size_limit));
+    }
     auto cache_option = arrow::io::CacheOptions::Defaults();
     cache_option.lazy = cache_lazy;
     cache_option.prefetch_limit = cache_prefetch_limit;
+    cache_option.hole_size_limit = cache_hole_size_limit;
     cache_option.range_size_limit = cache_range_size_limit;
     arrow_reader_props.set_cache_options(cache_option);
     return arrow_reader_props;
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
index 7a4328ff..dd323460 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader_test.cpp
@@ -705,7 +705,9 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) {
         ASSERT_EQ(arrow_reader_properties.batch_size(), 1024);
         ASSERT_EQ(arrow_reader_properties.use_threads(), true);
         ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 3);
-        ASSERT_EQ(arrow_reader_properties.cache_options(), arrow::io::CacheOptions::Defaults());
+        auto expected_cache_options = arrow::io::CacheOptions::Defaults();
+        expected_cache_options.hole_size_limit = DEFAULT_PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT;
+        ASSERT_EQ(arrow_reader_properties.cache_options(), expected_cache_options);
     }
     {
         std::map options = {{PARQUET_READ_EXECUTOR_THREAD_COUNT, "0"}};
@@ -724,6 +726,40 @@ TEST_F(ParquetFileBatchReaderTest, TestCreateArrowReaderProperties) {
         ASSERT_EQ(arrow_reader_properties.use_threads(), true);
         ASSERT_EQ(arrow::GetCpuThreadPoolCapacity(), 6);
     }
+    {
+        std::map options = {
+            {PARQUET_READ_CACHE_OPTION_LAZY, "true"},
+            {PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT, "2"},
+            {PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, "1048576"},
+            {PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT, "8388608"},
+        };
+        ASSERT_OK_AND_ASSIGN(
+            auto arrow_reader_properties,
+            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024));
+        const auto& cache_options = arrow_reader_properties.cache_options();
+        ASSERT_TRUE(cache_options.lazy);
+        ASSERT_EQ(cache_options.prefetch_limit, 2);
+        ASSERT_EQ(cache_options.hole_size_limit, 1024 * 1024);
+        ASSERT_EQ(cache_options.range_size_limit, 8 * 1024 * 1024);
+    }
+    {
+        std::map options = {
+            {PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, "-1"},
+        };
+        ASSERT_NOK_WITH_MSG(
+            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024),
+            "parquet.read.cache-option.hole-size-limit must be non-negative");
+    }
+    {
+        std::map options = {
+            {PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT, "1048576"},
+            {PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT, "1048576"},
+        };
+        ASSERT_NOK_WITH_MSG(
+            ParquetFileBatchReader::CreateArrowReaderProperties(pool_, options, 1024),
+            "parquet.read.cache-option.range-size-limit must be greater than "
+            "parquet.read.cache-option.hole-size-limit");
+    }
 }
 
 TEST_F(ParquetFileBatchReaderTest, TestBitmapRowGroupPushDownWithMultiRowGroups) {
diff --git a/src/paimon/format/parquet/parquet_format_defs.h b/src/paimon/format/parquet/parquet_format_defs.h
index cb171078..433103e5 100644
--- a/src/paimon/format/parquet/parquet_format_defs.h
+++ b/src/paimon/format/parquet/parquet_format_defs.h
@@ -65,6 +65,11 @@ static constexpr uint32_t DEFAULT_PARQUET_READ_EXECUTOR_THREAD_COUNT = 3;
 static inline const char PARQUET_READ_CACHE_OPTION_LAZY[] = "parquet.read.cache-option.lazy";
 static inline const char PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT[] =
     "parquet.read.cache-option.prefetch-limit";
+// Arrow I/O range coalescing limits, both measured in bytes. Adjacent ranges whose gap is
+// <= hole-size-limit can be merged; range-size-limit stops further merging once the combined
+// span would exceed it. A single input range may itself be larger than range-size-limit.
+static inline const char PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT[] =
+    "parquet.read.cache-option.hole-size-limit";
 static inline const char PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT[] =
     "parquet.read.cache-option.range-size-limit";
 // Strategy for refining row ranges using the selection bitmap produced by pushed-down
@@ -98,6 +103,8 @@ static inline const char PARQUET_READ_ENABLE_PAGE_INDEX_FILTER[] =
 static inline const char PARQUET_READ_ENABLE_PRE_BUFFER[] = "parquet.read.enable-pre-buffer";
 
 static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_PREFETCH_LIMIT = 0;
+// Default value of hole size limit, inherited from Arrow
+static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT = 8 * 1024;
 static constexpr uint32_t DEFAULT_PARQUET_READ_CACHE_OPTION_RANGE_SIZE_LIMIT = 32 * 1024 * 1024;
 static constexpr uint32_t DEFAULT_PARQUET_READ_PREDICATE_NODE_COUNT_LIMIT = 512;
 static constexpr bool DEFAULT_PARQUET_READ_ENABLE_PAGE_INDEX_FILTER = true;

From eab6a6c08586e6b944cff5eded4449194e7e3e64 Mon Sep 17 00:00:00 2001
From: lxy <38709059+lxy-9602@users.noreply.github.com>
Date: Fri, 31 Jul 2026 14:51:09 +0800
Subject: [PATCH 133/138] fix: align floating-point semantics with Java

---
 .../compact/aggregate/field_max_agg.h         |  21 ++-
 .../compact/aggregate/field_min_agg.h         |  21 ++-
 .../aggregate/field_min_max_agg_test.cpp      |  44 +++++
 .../format/parquet/column_index_filter.cpp    |  12 +-
 .../parquet/column_index_filter_test.cpp      | 150 ++++++++++++++++++
 .../parquet/floating_point_predicate_utils.h  |  79 +++++++++
 .../format/parquet/predicate_converter.cpp    |  76 +++++++++
 .../format/parquet/predicate_converter.h      |   3 +
 .../parquet/predicate_converter_test.cpp      | 126 ++++++++++++++-
 .../parquet/predicate_pushdown_test.cpp       |  49 +++++-
 10 files changed, 565 insertions(+), 16 deletions(-)
 create mode 100644 src/paimon/format/parquet/floating_point_predicate_utils.h

diff --git a/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h
index 15d98fa5..955f23ed 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h
+++ b/src/paimon/core/mergetree/compact/aggregate/field_max_agg.h
@@ -21,6 +21,7 @@
 #include 
 #include 
 
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/mergetree/compact/aggregate/field_aggregator.h"
 
 namespace paimon {
@@ -60,8 +61,6 @@ class FieldMaxAgg : public FieldAggregator {
             case arrow::Type::type::INT32:
             case arrow::Type::type::DATE32:
             case arrow::Type::type::INT64:
-            case arrow::Type::type::FLOAT:
-            case arrow::Type::type::DOUBLE:
             case arrow::Type::type::TIMESTAMP:
             case arrow::Type::type::DECIMAL128:
             case arrow::Type::type::STRING:
@@ -70,6 +69,24 @@ class FieldMaxAgg : public FieldAggregator {
                                        const VariantType& input_field) -> VariantType {
                     return accumulator < input_field ? input_field : accumulator;
                 });
+            case arrow::Type::type::FLOAT:
+                return FieldMaxFunc([](const VariantType& accumulator,
+                                       const VariantType& input_field) -> VariantType {
+                    auto accumulator_value = DataDefine::GetVariantValue(accumulator);
+                    auto input_value = DataDefine::GetVariantValue(input_field);
+                    int32_t compare_result =
+                        FieldsComparator::CompareFloatingPoint(accumulator_value, input_value);
+                    return compare_result < 0 ? input_field : accumulator;
+                });
+            case arrow::Type::type::DOUBLE:
+                return FieldMaxFunc([](const VariantType& accumulator,
+                                       const VariantType& input_field) -> VariantType {
+                    auto accumulator_value = DataDefine::GetVariantValue(accumulator);
+                    auto input_value = DataDefine::GetVariantValue(input_field);
+                    int32_t compare_result =
+                        FieldsComparator::CompareFloatingPoint(accumulator_value, input_value);
+                    return compare_result < 0 ? input_field : accumulator;
+                });
             default:
                 return Status::Invalid(
                     fmt::format("type {} not support in FieldMaxAgg", field_type->ToString()));
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h b/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h
index e57a5ea5..96c25b48 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h
+++ b/src/paimon/core/mergetree/compact/aggregate/field_min_agg.h
@@ -21,6 +21,7 @@
 #include 
 #include 
 
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/core/mergetree/compact/aggregate/field_aggregator.h"
 
 namespace paimon {
@@ -60,8 +61,6 @@ class FieldMinAgg : public FieldAggregator {
             case arrow::Type::type::INT32:
             case arrow::Type::type::DATE32:
             case arrow::Type::type::INT64:
-            case arrow::Type::type::FLOAT:
-            case arrow::Type::type::DOUBLE:
             case arrow::Type::type::TIMESTAMP:
             case arrow::Type::type::DECIMAL128:
             case arrow::Type::type::STRING:
@@ -70,6 +69,24 @@ class FieldMinAgg : public FieldAggregator {
                                        const VariantType& input_field) -> VariantType {
                     return accumulator < input_field ? accumulator : input_field;
                 });
+            case arrow::Type::type::FLOAT:
+                return FieldMinFunc([](const VariantType& accumulator,
+                                       const VariantType& input_field) -> VariantType {
+                    auto accumulator_value = DataDefine::GetVariantValue(accumulator);
+                    auto input_value = DataDefine::GetVariantValue(input_field);
+                    int32_t compare_result =
+                        FieldsComparator::CompareFloatingPoint(accumulator_value, input_value);
+                    return compare_result < 0 ? accumulator : input_field;
+                });
+            case arrow::Type::type::DOUBLE:
+                return FieldMinFunc([](const VariantType& accumulator,
+                                       const VariantType& input_field) -> VariantType {
+                    auto accumulator_value = DataDefine::GetVariantValue(accumulator);
+                    auto input_value = DataDefine::GetVariantValue(input_field);
+                    int32_t compare_result =
+                        FieldsComparator::CompareFloatingPoint(accumulator_value, input_value);
+                    return compare_result < 0 ? accumulator : input_field;
+                });
             default:
                 return Status::Invalid(
                     fmt::format("type {} not support in FieldMinAgg", field_type->ToString()));
diff --git a/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp b/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp
index 0f79a527..c51b76cd 100644
--- a/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp
+++ b/src/paimon/core/mergetree/compact/aggregate/field_min_max_agg_test.cpp
@@ -16,7 +16,11 @@
  * limitations under the License.
  */
 
+#include 
+#include 
+#include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -40,6 +44,41 @@ class DataType;
 
 namespace paimon::test {
 
+template 
+void AssertSameFloatingPoint(T actual, T expected) {
+    if (std::isnan(expected)) {
+        ASSERT_TRUE(std::isnan(actual));
+        return;
+    }
+    ASSERT_EQ(actual, expected);
+    if (expected == static_cast(0.0)) {
+        ASSERT_EQ(std::signbit(actual), std::signbit(expected));
+    }
+}
+
+template 
+void CheckJavaCompatibleFloatingPointMinMax(const std::shared_ptr& type) {
+    const T infinity = std::numeric_limits::infinity();
+    const T nan = std::numeric_limits::quiet_NaN();
+    // This is the total order defined by Java Float.compare and Double.compare.
+    const std::array values = {-infinity, -static_cast(0.0), static_cast(0.0), infinity,
+                                     nan};
+
+    ASSERT_OK_AND_ASSIGN(auto field_min_agg, FieldMinAgg::Create(type));
+    ASSERT_OK_AND_ASSIGN(auto field_max_agg, FieldMaxAgg::Create(type));
+    for (size_t i = 0; i < values.size(); ++i) {
+        for (size_t j = 0; j < values.size(); ++j) {
+            VariantType min_result = field_min_agg->Agg(values[i], values[j]);
+            AssertSameFloatingPoint(DataDefine::GetVariantValue(min_result),
+                                    values[std::min(i, j)]);
+
+            VariantType max_result = field_max_agg->Agg(values[i], values[j]);
+            AssertSameFloatingPoint(DataDefine::GetVariantValue(max_result),
+                                    values[std::max(i, j)]);
+        }
+    }
+}
+
 TEST(FieldMinMaxAggTest, TestSimple) {
     {
         ASSERT_OK_AND_ASSIGN(auto field_min_agg, FieldMinAgg::Create(arrow::int32()));
@@ -53,6 +92,11 @@ TEST(FieldMinMaxAggTest, TestSimple) {
     }
 }
 
+TEST(FieldMinMaxAggTest, TestJavaCompatibleFloatingPointOrder) {
+    CheckJavaCompatibleFloatingPointMinMax(arrow::float32());
+    CheckJavaCompatibleFloatingPointMinMax(arrow::float64());
+}
+
 TEST(FieldMinMaxAggTest, TestInvalidType) {
     auto field_min_agg = FieldMinAgg::Create(arrow::boolean());
     ASSERT_FALSE(field_min_agg.ok());
diff --git a/src/paimon/format/parquet/column_index_filter.cpp b/src/paimon/format/parquet/column_index_filter.cpp
index 5dd3af9c..61849802 100644
--- a/src/paimon/format/parquet/column_index_filter.cpp
+++ b/src/paimon/format/parquet/column_index_filter.cpp
@@ -20,12 +20,12 @@
 #include "paimon/format/parquet/column_index_filter.h"
 
 #include 
-#include 
 #include 
 #include 
 #include 
 
 #include "fmt/format.h"
+#include "paimon/common/utils/fields_comparator.h"
 #include "paimon/data/decimal.h"
 #include "paimon/memory/bytes.h"
 #include "paimon/memory/memory_pool.h"
@@ -558,10 +558,7 @@ std::optional ColumnIndexFilter::CompareEncodedWithLiteral(const std::s
             float enc_val;
             std::memcpy(&enc_val, encoded.data(), sizeof(float));
             auto lit_val = literal.GetValue();
-            if (std::isnan(enc_val) || std::isnan(lit_val)) {
-                return std::nullopt;
-            }
-            return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0;
+            return FieldsComparator::CompareFloatingPoint(enc_val, lit_val);
         }
         case FieldType::DOUBLE: {
             if (encoded.size() < sizeof(double)) {
@@ -570,10 +567,7 @@ std::optional ColumnIndexFilter::CompareEncodedWithLiteral(const std::s
             double enc_val;
             std::memcpy(&enc_val, encoded.data(), sizeof(double));
             auto lit_val = literal.GetValue();
-            if (std::isnan(enc_val) || std::isnan(lit_val)) {
-                return std::nullopt;
-            }
-            return (enc_val < lit_val) ? -1 : (enc_val > lit_val) ? 1 : 0;
+            return FieldsComparator::CompareFloatingPoint(enc_val, lit_val);
         }
         case FieldType::STRING:
         case FieldType::BINARY: {
diff --git a/src/paimon/format/parquet/column_index_filter_test.cpp b/src/paimon/format/parquet/column_index_filter_test.cpp
index 4c63ea0b..35e02576 100644
--- a/src/paimon/format/parquet/column_index_filter_test.cpp
+++ b/src/paimon/format/parquet/column_index_filter_test.cpp
@@ -19,7 +19,9 @@
 
 #include "paimon/format/parquet/column_index_filter.h"
 
+#include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -471,6 +473,154 @@ TEST_F(ColumnIndexFilterTest, OrCompound) {
     EXPECT_EQ(99, ranges.GetRanges()[1].to);
 }
 
+TEST_F(ColumnIndexFilterTest, SignedZeroUsesJavaOrderForFloatingPointPages) {
+    for (FieldType field_type : {FieldType::FLOAT, FieldType::DOUBLE}) {
+        std::shared_ptr values;
+        if (field_type == FieldType::FLOAT) {
+            arrow::FloatBuilder builder;
+            ASSERT_TRUE(builder.Reserve(30).ok());
+            for (int32_t i = 0; i < 10; ++i) {
+                builder.UnsafeAppend(-0.0f);
+            }
+            for (int32_t i = 0; i < 10; ++i) {
+                builder.UnsafeAppend(0.0f);
+            }
+            for (int32_t i = 0; i < 10; ++i) {
+                builder.UnsafeAppend(1.0f);
+            }
+            values = builder.Finish().ValueOrDie();
+        } else {
+            arrow::DoubleBuilder builder;
+            ASSERT_TRUE(builder.Reserve(30).ok());
+            for (int32_t i = 0; i < 10; ++i) {
+                builder.UnsafeAppend(-0.0);
+            }
+            for (int32_t i = 0; i < 10; ++i) {
+                builder.UnsafeAppend(0.0);
+            }
+            for (int32_t i = 0; i < 10; ++i) {
+                builder.UnsafeAppend(1.0);
+            }
+            values = builder.Finish().ValueOrDie();
+        }
+
+        auto field = arrow::field(
+            "value", field_type == FieldType::FLOAT ? arrow::float32() : arrow::float64());
+        auto data = arrow::StructArray::Make({values}, {field}).ValueOrDie();
+        std::string file_name =
+            dir_->Str() + (field_type == FieldType::FLOAT ? "/float_signed_zero.parquet"
+                                                          : "/double_signed_zero.parquet");
+        WriteTestFile(file_name, data, /*write_batch_size=*/10,
+                      /*max_row_group_length=*/30);
+
+        ASSERT_OK_AND_ASSIGN(std::shared_ptr in, fs_->Open(file_name));
+        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
+        auto in_stream = std::make_shared(in, length, arrow_pool_);
+        auto reader = ::parquet::ParquetFileReader::Open(in_stream);
+        ASSERT_TRUE(reader);
+        auto page_index_reader = reader->GetPageIndexReader();
+        ASSERT_TRUE(page_index_reader);
+        auto row_group_page_index = page_index_reader->RowGroup(0);
+        ASSERT_TRUE(row_group_page_index);
+        auto column_index = row_group_page_index->GetColumnIndex(0);
+        ASSERT_TRUE(column_index);
+        // Pages 0 and 1 contain only -0.0 and +0.0 respectively. Parquet normalizes both
+        // zero-only page bounds to [-0.0, +0.0], so page pruning must remain a safe superset.
+        ASSERT_EQ(3, column_index->encoded_min_values().size());
+        ASSERT_EQ(3, column_index->encoded_max_values().size());
+        for (int32_t page_index : {0, 1}) {
+            if (field_type == FieldType::FLOAT) {
+                float min_value;
+                float max_value;
+                std::memcpy(&min_value, column_index->encoded_min_values()[page_index].data(),
+                            sizeof(float));
+                std::memcpy(&max_value, column_index->encoded_max_values()[page_index].data(),
+                            sizeof(float));
+                ASSERT_TRUE(std::signbit(min_value));
+                ASSERT_FALSE(std::signbit(max_value));
+            } else {
+                double min_value;
+                double max_value;
+                std::memcpy(&min_value, column_index->encoded_min_values()[page_index].data(),
+                            sizeof(double));
+                std::memcpy(&max_value, column_index->encoded_max_values()[page_index].data(),
+                            sizeof(double));
+                ASSERT_TRUE(std::signbit(min_value));
+                ASSERT_FALSE(std::signbit(max_value));
+            }
+        }
+
+        auto less_negative_zero = PredicateBuilder::LessThan(
+            /*field_index=*/0, /*field_name=*/"value", field_type,
+            field_type == FieldType::FLOAT ? Literal(-0.0f) : Literal(-0.0));
+        ASSERT_OK_AND_ASSIGN(
+            auto ranges, ColumnIndexFilter::CalculateRowRanges(
+                             less_negative_zero, page_index_reader, {{"value", 0}},
+                             /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows()));
+        ASSERT_TRUE(ranges.IsEmpty()) << "field type: " << static_cast(field_type);
+
+        auto less_positive_zero = PredicateBuilder::LessThan(
+            /*field_index=*/0, /*field_name=*/"value", field_type,
+            field_type == FieldType::FLOAT ? Literal(0.0f) : Literal(0.0));
+        ASSERT_OK_AND_ASSIGN(
+            ranges, ColumnIndexFilter::CalculateRowRanges(
+                        less_positive_zero, page_index_reader, {{"value", 0}},
+                        /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows()));
+        ASSERT_EQ(20, ranges.RowCount());
+        ASSERT_EQ(1, ranges.GetRanges().size());
+        ASSERT_EQ(0, ranges.GetRanges()[0].from);
+        ASSERT_EQ(19, ranges.GetRanges()[0].to);
+
+        auto greater_negative_zero = PredicateBuilder::GreaterThan(
+            /*field_index=*/0, /*field_name=*/"value", field_type,
+            field_type == FieldType::FLOAT ? Literal(-0.0f) : Literal(-0.0));
+        ASSERT_OK_AND_ASSIGN(
+            ranges, ColumnIndexFilter::CalculateRowRanges(
+                        greater_negative_zero, page_index_reader, {{"value", 0}},
+                        /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows()));
+        ASSERT_EQ(30, ranges.RowCount());
+
+        auto not_equal_negative_zero = PredicateBuilder::NotEqual(
+            /*field_index=*/0, /*field_name=*/"value", field_type,
+            field_type == FieldType::FLOAT ? Literal(-0.0f) : Literal(-0.0));
+        ASSERT_OK_AND_ASSIGN(
+            ranges, ColumnIndexFilter::CalculateRowRanges(
+                        not_equal_negative_zero, page_index_reader, {{"value", 0}},
+                        /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows()));
+        ASSERT_EQ(30, ranges.RowCount());
+
+        auto greater_finite = PredicateBuilder::GreaterThan(
+            /*field_index=*/0, /*field_name=*/"value", field_type,
+            field_type == FieldType::FLOAT ? Literal(2.0f) : Literal(2.0));
+        ASSERT_OK_AND_ASSIGN(
+            ranges, ColumnIndexFilter::CalculateRowRanges(
+                        greater_finite, page_index_reader, {{"value", 0}},
+                        /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows()));
+        ASSERT_TRUE(ranges.IsEmpty());
+
+        auto greater_between_pages = PredicateBuilder::GreaterThan(
+            /*field_index=*/0, /*field_name=*/"value", field_type,
+            field_type == FieldType::FLOAT ? Literal(0.5f) : Literal(0.5));
+        ASSERT_OK_AND_ASSIGN(
+            ranges, ColumnIndexFilter::CalculateRowRanges(
+                        greater_between_pages, page_index_reader, {{"value", 0}},
+                        /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows()));
+        ASSERT_EQ(10, ranges.RowCount());
+        ASSERT_EQ(1, ranges.GetRanges().size());
+        ASSERT_EQ(20, ranges.GetRanges()[0].from);
+        ASSERT_EQ(29, ranges.GetRanges()[0].to);
+
+        auto equal_finite = PredicateBuilder::Equal(
+            /*field_index=*/0, /*field_name=*/"value", field_type,
+            field_type == FieldType::FLOAT ? Literal(2.0f) : Literal(2.0));
+        ASSERT_OK_AND_ASSIGN(
+            ranges, ColumnIndexFilter::CalculateRowRanges(
+                        equal_finite, page_index_reader, {{"value", 0}},
+                        /*row_group_index=*/0, reader->metadata()->RowGroup(0)->num_rows()));
+        ASSERT_TRUE(ranges.IsEmpty());
+    }
+}
+
 /// Predicates referencing fields absent from the data file are stripped upstream
 /// by FieldMappingBuilder, so reaching ColumnIndexFilter with such a predicate is
 /// a contract violation and surfaces as an error.
diff --git a/src/paimon/format/parquet/floating_point_predicate_utils.h b/src/paimon/format/parquet/floating_point_predicate_utils.h
new file mode 100644
index 00000000..a1fb9a7f
--- /dev/null
+++ b/src/paimon/format/parquet/floating_point_predicate_utils.h
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include 
+
+#include "paimon/defs.h"
+#include "paimon/predicate/function.h"
+#include "paimon/predicate/literal.h"
+
+namespace paimon::parquet {
+
+class FloatingPointPredicateUtils {
+ public:
+    FloatingPointPredicateUtils() = delete;
+    ~FloatingPointPredicateUtils() = delete;
+
+    static bool IsType(FieldType field_type) {
+        return field_type == FieldType::FLOAT || field_type == FieldType::DOUBLE;
+    }
+
+    static bool IsZero(const Literal& literal) {
+        if (literal.IsNull()) {
+            return false;
+        }
+        if (literal.GetType() == FieldType::FLOAT) {
+            return literal.GetValue() == 0.0f;
+        }
+        if (literal.GetType() == FieldType::DOUBLE) {
+            return literal.GetValue() == 0.0;
+        }
+        return false;
+    }
+
+    static bool IsNegativeZero(const Literal& literal) {
+        if (!IsZero(literal)) {
+            return false;
+        }
+        if (literal.GetType() == FieldType::FLOAT) {
+            return std::signbit(literal.GetValue());
+        }
+        return std::signbit(literal.GetValue());
+    }
+
+    static bool IsComparison(Function::Type function_type) {
+        switch (function_type) {
+            case Function::Type::EQUAL:
+            case Function::Type::NOT_EQUAL:
+            case Function::Type::GREATER_THAN:
+            case Function::Type::GREATER_OR_EQUAL:
+            case Function::Type::LESS_THAN:
+            case Function::Type::LESS_OR_EQUAL:
+            case Function::Type::IN:
+            case Function::Type::NOT_IN:
+                return true;
+            default:
+                return false;
+        }
+    }
+};
+
+}  // namespace paimon::parquet
diff --git a/src/paimon/format/parquet/predicate_converter.cpp b/src/paimon/format/parquet/predicate_converter.cpp
index 819d2cc1..e81a675e 100644
--- a/src/paimon/format/parquet/predicate_converter.cpp
+++ b/src/paimon/format/parquet/predicate_converter.cpp
@@ -29,6 +29,7 @@
 #include "fmt/format.h"
 #include "paimon/data/decimal.h"
 #include "paimon/defs.h"
+#include "paimon/format/parquet/floating_point_predicate_utils.h"
 #include "paimon/predicate/compound_predicate.h"
 #include "paimon/predicate/function.h"
 #include "paimon/predicate/leaf_predicate.h"
@@ -36,6 +37,49 @@
 #include "paimon/predicate/predicate.h"
 
 namespace paimon::parquet {
+namespace {
+
+Result ConvertFloatingPointComparison(
+    const std::string& field_name, Function::Type function_type, const Literal& literal,
+    const arrow::compute::Expression& arrow_literal) {
+    arrow::compute::Expression field = arrow::compute::field_ref(field_name);
+    switch (function_type) {
+        case Function::Type::EQUAL:
+            return arrow::compute::equal(std::move(field), arrow_literal);
+        case Function::Type::NOT_EQUAL:
+            if (FloatingPointPredicateUtils::IsZero(literal)) {
+                // Arrow treats both zero signs as equal. Keep every non-null row group so the
+                // exact predicate can distinguish the opposite zero sign.
+                return arrow::compute::is_valid(std::move(field));
+            }
+            return arrow::compute::not_equal(std::move(field), arrow_literal);
+        case Function::Type::GREATER_THAN:
+            if (FloatingPointPredicateUtils::IsNegativeZero(literal)) {
+                // parquet-mr normalizes a zero upper bound to +0.0. Arrow treats both zero signs
+                // as equal, so >= is the safe pruning equivalent of Java's > -0.0.
+                return arrow::compute::greater_equal(std::move(field), arrow_literal);
+            }
+            return arrow::compute::greater(std::move(field), arrow_literal);
+        case Function::Type::GREATER_OR_EQUAL:
+            return arrow::compute::greater_equal(std::move(field), arrow_literal);
+        case Function::Type::LESS_THAN:
+            if (FloatingPointPredicateUtils::IsZero(literal) &&
+                !FloatingPointPredicateUtils::IsNegativeZero(literal)) {
+                // parquet-mr normalizes a zero lower bound to -0.0. Arrow's <= 0.0 retains the
+                // page for Java's -0.0 < +0.0.
+                return arrow::compute::less_equal(std::move(field), arrow_literal);
+            }
+            return arrow::compute::less(std::move(field), arrow_literal);
+        case Function::Type::LESS_OR_EQUAL:
+            return arrow::compute::less_equal(std::move(field), arrow_literal);
+        default:
+            return Status::Invalid(fmt::format("invalid floating-point comparison type {}",
+                                               static_cast(function_type)));
+    }
+}
+
+}  // namespace
+
 arrow::compute::Expression PredicateConverter::AlwaysTrue() {
     static const arrow::compute::Expression expr = arrow::compute::literal(true);
     return expr;
@@ -143,6 +187,10 @@ Result PredicateConverter::ConvertLeaf(
     const auto& literals = leaf_predicate->Literals();
     const auto& function = leaf_predicate->GetFunction();
     auto function_type = function.GetType();
+    if (FloatingPointPredicateUtils::IsType(leaf_predicate->GetFieldType()) &&
+        FloatingPointPredicateUtils::IsComparison(function_type)) {
+        return ConvertFloatingPointLeaf(leaf_predicate);
+    }
     switch (function_type) {
         case Function::Type::IS_NULL: {
             return arrow::compute::is_null(arrow::compute::field_ref(field_name),
@@ -244,6 +292,34 @@ Result PredicateConverter::ConvertLeaf(
     return Status::OK();
 }
 
+Result PredicateConverter::ConvertFloatingPointLeaf(
+    const std::shared_ptr& leaf_predicate) {
+    const auto& field_name = leaf_predicate->FieldName();
+    const auto& literals = leaf_predicate->Literals();
+    const auto& function = leaf_predicate->GetFunction();
+    Function::Type function_type = function.GetType();
+    PAIMON_RETURN_NOT_OK(CheckLiteralNotEmpty(literals, function, field_name));
+
+    if (function_type == Function::Type::IN || function_type == Function::Type::NOT_IN) {
+        Function::Type comparison_type =
+            function_type == Function::Type::IN ? Function::Type::EQUAL : Function::Type::NOT_EQUAL;
+        std::vector sub_exprs;
+        sub_exprs.reserve(literals.size());
+        for (const auto& literal : literals) {
+            CONVERT_TO_ARROW_LITERAL(literal);
+            PAIMON_ASSIGN_OR_RAISE(arrow::compute::Expression sub_expr,
+                                   ConvertFloatingPointComparison(field_name, comparison_type,
+                                                                  literal, arrow_literal));
+            sub_exprs.push_back(std::move(sub_expr));
+        }
+        return function_type == Function::Type::IN ? arrow::compute::or_(sub_exprs)
+                                                   : arrow::compute::and_(sub_exprs);
+    }
+
+    CONVERT_TO_ARROW_LITERAL(literals[0]);
+    return ConvertFloatingPointComparison(field_name, function_type, literals[0], arrow_literal);
+}
+
 Result PredicateConverter::ConvertToArrowLiteral(
     const Literal& literal) {
     auto literal_type = literal.GetType();
diff --git a/src/paimon/format/parquet/predicate_converter.h b/src/paimon/format/parquet/predicate_converter.h
index 5b44f846..e4df06be 100644
--- a/src/paimon/format/parquet/predicate_converter.h
+++ b/src/paimon/format/parquet/predicate_converter.h
@@ -68,6 +68,9 @@ class PredicateConverter {
     static Result ConvertLeaf(
         const std::shared_ptr& leaf_predicate);
 
+    static Result ConvertFloatingPointLeaf(
+        const std::shared_ptr& leaf_predicate);
+
     static Result ConvertToArrowLiteral(const Literal& literal);
 };
 
diff --git a/src/paimon/format/parquet/predicate_converter_test.cpp b/src/paimon/format/parquet/predicate_converter_test.cpp
index a6661dc3..542ea0f4 100644
--- a/src/paimon/format/parquet/predicate_converter_test.cpp
+++ b/src/paimon/format/parquet/predicate_converter_test.cpp
@@ -191,6 +191,128 @@ TEST(PredicateConverterTest, TestSimple) {
     }
 }
 
+TEST(PredicateConverterTest, TestJavaCompatibleSignedZeroExpressions) {
+    struct TestType {
+        FieldType field_type;
+        Literal negative_zero;
+        Literal positive_zero;
+        Literal one;
+    };
+    const std::vector test_types = {
+        {FieldType::FLOAT, Literal(-0.0f), Literal(0.0f), Literal(1.0f)},
+        {FieldType::DOUBLE, Literal(-0.0), Literal(0.0), Literal(1.0)}};
+
+    for (const auto& test_type : test_types) {
+        auto equal_negative_zero = PredicateBuilder::Equal(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero);
+        ASSERT_OK_AND_ASSIGN(auto equal_negative_zero_expr,
+                             PredicateConverter::Convert(equal_negative_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 == -0)", equal_negative_zero_expr.ToString());
+
+        auto equal_positive_zero = PredicateBuilder::Equal(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero);
+        ASSERT_OK_AND_ASSIGN(auto equal_positive_zero_expr,
+                             PredicateConverter::Convert(equal_positive_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 == 0)", equal_positive_zero_expr.ToString());
+
+        auto greater_negative_zero = PredicateBuilder::GreaterThan(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero);
+        ASSERT_OK_AND_ASSIGN(auto greater_negative_zero_expr,
+                             PredicateConverter::Convert(greater_negative_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 >= -0)", greater_negative_zero_expr.ToString());
+
+        auto less_positive_zero = PredicateBuilder::LessThan(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero);
+        ASSERT_OK_AND_ASSIGN(auto less_positive_zero_expr,
+                             PredicateConverter::Convert(less_positive_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 <= 0)", less_positive_zero_expr.ToString());
+
+        auto not_equal_negative_zero = PredicateBuilder::NotEqual(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero);
+        ASSERT_OK_AND_ASSIGN(auto not_equal_negative_zero_expr,
+                             PredicateConverter::Convert(not_equal_negative_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("is_valid(f0)", not_equal_negative_zero_expr.ToString());
+
+        auto not_equal_positive_zero = PredicateBuilder::NotEqual(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero);
+        ASSERT_OK_AND_ASSIGN(auto not_equal_positive_zero_expr,
+                             PredicateConverter::Convert(not_equal_positive_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("is_valid(f0)", not_equal_positive_zero_expr.ToString());
+
+        auto greater_positive_zero = PredicateBuilder::GreaterThan(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero);
+        ASSERT_OK_AND_ASSIGN(auto greater_positive_zero_expr,
+                             PredicateConverter::Convert(greater_positive_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 > 0)", greater_positive_zero_expr.ToString());
+
+        auto greater_or_equal_positive_zero = PredicateBuilder::GreaterOrEqual(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero);
+        ASSERT_OK_AND_ASSIGN(auto greater_or_equal_positive_zero_expr,
+                             PredicateConverter::Convert(greater_or_equal_positive_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 >= 0)", greater_or_equal_positive_zero_expr.ToString());
+
+        auto greater_or_equal_negative_zero = PredicateBuilder::GreaterOrEqual(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero);
+        ASSERT_OK_AND_ASSIGN(auto greater_or_equal_negative_zero_expr,
+                             PredicateConverter::Convert(greater_or_equal_negative_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 >= -0)", greater_or_equal_negative_zero_expr.ToString());
+
+        auto less_negative_zero = PredicateBuilder::LessThan(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero);
+        ASSERT_OK_AND_ASSIGN(auto less_negative_zero_expr,
+                             PredicateConverter::Convert(less_negative_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 < -0)", less_negative_zero_expr.ToString());
+
+        auto less_or_equal_negative_zero = PredicateBuilder::LessOrEqual(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.negative_zero);
+        ASSERT_OK_AND_ASSIGN(auto less_or_equal_negative_zero_expr,
+                             PredicateConverter::Convert(less_or_equal_negative_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 <= -0)", less_or_equal_negative_zero_expr.ToString());
+
+        auto less_or_equal_positive_zero = PredicateBuilder::LessOrEqual(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type, test_type.positive_zero);
+        ASSERT_OK_AND_ASSIGN(auto less_or_equal_positive_zero_expr,
+                             PredicateConverter::Convert(less_or_equal_positive_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(f0 <= 0)", less_or_equal_positive_zero_expr.ToString());
+
+        auto in_negative_zero = PredicateBuilder::In(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type,
+            {test_type.negative_zero, test_type.one});
+        ASSERT_OK_AND_ASSIGN(auto in_negative_zero_expr,
+                             PredicateConverter::Convert(in_negative_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("((f0 == -0) or (f0 == 1))", in_negative_zero_expr.ToString());
+
+        auto in_positive_zero = PredicateBuilder::In(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type,
+            {test_type.positive_zero, test_type.one});
+        ASSERT_OK_AND_ASSIGN(auto in_positive_zero_expr,
+                             PredicateConverter::Convert(in_positive_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("((f0 == 0) or (f0 == 1))", in_positive_zero_expr.ToString());
+
+        auto not_in_negative_zero = PredicateBuilder::NotIn(
+            /*field_index=*/0, /*field_name=*/"f0", test_type.field_type,
+            {test_type.negative_zero, test_type.one});
+        ASSERT_OK_AND_ASSIGN(auto not_in_negative_zero_expr,
+                             PredicateConverter::Convert(not_in_negative_zero,
+                                                         /*predicate_node_count_limit=*/100));
+        ASSERT_EQ("(is_valid(f0) and (f0 != 1))", not_in_negative_zero_expr.ToString());
+    }
+}
+
 TEST(PredicateConverterTest, TestCompound) {
     // "struct";
     {
@@ -387,9 +509,9 @@ TEST(PredicateConverterTest, TestWithoutPredicate) {
 }
 
 TEST(PredicateConverterTest, TestInvalidCase) {
-    auto predicate =
+    auto empty_in =
         PredicateBuilder::In(/*field_index=*/0, /*field_name=*/"f0", FieldType::INT, {});
-    ASSERT_NOK_WITH_MSG(PredicateConverter::Convert(predicate, /*predicate_node_count_limit=*/100),
+    ASSERT_NOK_WITH_MSG(PredicateConverter::Convert(empty_in, /*predicate_node_count_limit=*/100),
                         "predicate [In] need literal on field f0");
 }
 
diff --git a/src/paimon/format/parquet/predicate_pushdown_test.cpp b/src/paimon/format/parquet/predicate_pushdown_test.cpp
index b63eaa38..3d169db3 100644
--- a/src/paimon/format/parquet/predicate_pushdown_test.cpp
+++ b/src/paimon/format/parquet/predicate_pushdown_test.cpp
@@ -81,7 +81,8 @@ class PredicatePushdownTest : public ::testing::Test {
 
     void TearDown() override {}
 
-    void PrepareTestData(const std::shared_ptr& struct_array) {
+    void PrepareTestData(const std::shared_ptr& struct_array,
+                         bool enable_page_index = false) {
         auto data_type = struct_array->struct_type();
         auto data_schema = arrow::schema(data_type->fields());
         auto data_arrow_array = std::make_unique();
@@ -90,6 +91,10 @@ class PredicatePushdownTest : public ::testing::Test {
                              fs_->Create(file_name_, /*overwrite=*/false));
         ::parquet::WriterProperties::Builder builder;
         builder.write_batch_size(batch_size_);
+        if (enable_page_index) {
+            builder.enable_write_page_index();
+            builder.data_pagesize(1);
+        }
         auto writer_properties = builder.build();
         ASSERT_OK_AND_ASSIGN(
             auto format_writer,
@@ -427,6 +432,48 @@ TEST_F(PredicatePushdownTest, TestPredicatePushdownWithAllDataNull) {
     }
 }
 
+TEST_F(PredicatePushdownTest, TestSignedZeroRowGroupPruning) {
+    auto value_field = arrow::field("value", arrow::float64());
+    std::shared_ptr data = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({value_field}),
+                                                  R"([[-0.0], [0.0]])")
+            .ValueOrDie());
+    PrepareTestData(data);
+
+    auto predicate = PredicateBuilder::GreaterThan(
+        /*field_index=*/0, /*field_name=*/"value", FieldType::DOUBLE, Literal(-0.0));
+    CheckResult(arrow::schema({value_field}), predicate, data);
+}
+
+TEST_F(PredicatePushdownTest, TestFiniteGreaterThanStillPrunesRowGroup) {
+    auto value_field = arrow::field("value", arrow::float64());
+    std::shared_ptr data = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({value_field}), R"([[0.5]])")
+            .ValueOrDie());
+    PrepareTestData(data);
+
+    auto predicate = PredicateBuilder::GreaterThan(
+        /*field_index=*/0, /*field_name=*/"value", FieldType::DOUBLE, Literal(1.0));
+    CheckResult(arrow::schema({value_field}), predicate, /*expected_array=*/nullptr);
+}
+
+TEST_F(PredicatePushdownTest, TestSignedZeroPageIndexPruning) {
+    auto value_field = arrow::field("value", arrow::float64());
+    std::shared_ptr data = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({value_field}), R"([
+            [-0.0], [-0.0], [-0.0], [-0.0], [-0.0],
+            [-0.0], [-0.0], [-0.0], [-0.0], [-0.0],
+            [1.0], [1.0], [1.0], [1.0], [1.0],
+            [1.0], [1.0], [1.0], [1.0], [1.0]
+        ])")
+            .ValueOrDie());
+    PrepareTestData(data, /*enable_page_index=*/true);
+
+    auto predicate = PredicateBuilder::LessThan(
+        /*field_index=*/0, /*field_name=*/"value", FieldType::DOUBLE, Literal(0.0));
+    CheckResult(arrow::schema({value_field}), predicate, data->Slice(0, batch_size_));
+}
+
 TEST_F(PredicatePushdownTest, TestCompoundPredicate) {
     PrepareTestData(struct_array_);
     auto read_schema = arrow::schema(struct_array_->struct_type()->fields());

From eb172b82cd4c8536d43f5d572181c7500dd6ec5a Mon Sep 17 00:00:00 2001
From: lxy264173 
Date: Fri, 31 Jul 2026 16:52:39 +0800
Subject: [PATCH 134/138] chore: remove target-only migration leftover

---
 .../operation/metrics/commit_metrics_test.cpp | 52 -------------------
 1 file changed, 52 deletions(-)
 delete mode 100644 src/paimon/core/operation/metrics/commit_metrics_test.cpp

diff --git a/src/paimon/core/operation/metrics/commit_metrics_test.cpp b/src/paimon/core/operation/metrics/commit_metrics_test.cpp
deleted file mode 100644
index 787a01dd..00000000
--- a/src/paimon/core/operation/metrics/commit_metrics_test.cpp
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include "paimon/core/operation/metrics/commit_metrics.h"
-
-#include 
-#include 
-
-#include "gtest/gtest.h"
-#include "paimon/common/metrics/metrics_impl.h"
-#include "paimon/testing/utils/testharness.h"
-
-namespace paimon::test {
-
-TEST(CommitMetricsTest, TestSimple) {
-    auto commit_metrics = std::make_shared();
-    commit_metrics->SetCounter("some_metric", 100);
-    commit_metrics->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, 30);
-    ASSERT_OK_AND_ASSIGN(uint64_t counter,
-                         commit_metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS));
-    ASSERT_EQ(30, counter);
-    ASSERT_OK_AND_ASSIGN(counter, commit_metrics->GetCounter("some_metric"));
-    ASSERT_EQ(100, counter);
-    auto other = std::make_shared();
-    other->SetCounter("some_metric_2", 200);
-    other->SetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS, 50);
-    commit_metrics->Merge(other);
-    ASSERT_OK_AND_ASSIGN(counter, commit_metrics->GetCounter(CommitMetrics::LAST_COMMIT_ATTEMPTS));
-    ASSERT_EQ(80, counter);
-    ASSERT_OK_AND_ASSIGN(counter, commit_metrics->GetCounter("some_metric"));
-    ASSERT_EQ(100, counter);
-    ASSERT_OK_AND_ASSIGN(counter, commit_metrics->GetCounter("some_metric_2"));
-    ASSERT_EQ(200, counter);
-}
-
-}  // namespace paimon::test

From a04a174be55d300f8b63900edac597a5cb0d307c Mon Sep 17 00:00:00 2001
From: lxy264173 
Date: Fri, 31 Jul 2026 17:30:47 +0800
Subject: [PATCH 135/138] fix(release): align documentation version

---
 docs/source/_static/versions.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json
index 28cf190c..60390019 100644
--- a/docs/source/_static/versions.json
+++ b/docs/source/_static/versions.json
@@ -1,7 +1,7 @@
 [
     {
-        "name": "0.2.3",
-        "version": "0.2.3",
+        "name": "0.3.0",
+        "version": "0.3.0",
         "url": "https://paimon.apache.org/docs/cpp/"
     }
 ]

From 900a4ddf3ed551ad53631f49963915d553eb42cc Mon Sep 17 00:00:00 2001
From: lxy264173 
Date: Fri, 31 Jul 2026 17:34:00 +0800
Subject: [PATCH 136/138] ci: add consolidated build and test workflow

---
 .github/workflows/build_and_test.yaml | 92 +++++++++++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100644 .github/workflows/build_and_test.yaml

diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml
new file mode 100644
index 00000000..ab3f22bb
--- /dev/null
+++ b/.github/workflows/build_and_test.yaml
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Build and Test
+
+on:
+  push:
+    branches:
+      - '**'
+    tags:
+      - '**'
+  pull_request:
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-test:
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: gcc-release
+            cc: gcc-14
+            cxx: g++-14
+            build_args: --build_type Release
+          - name: clang-release
+            build_args: --build_type Release
+          - name: gcc-debug
+            cc: gcc-14
+            cxx: g++-14
+          - name: clang-debug
+            fetch_depth: '0' # fetch the PR target branch history for clang-tidy
+            build_args: >-
+              --check_clang_tidy
+              --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}"
+          - name: asan-ubsan
+            build_args: --enable_asan --enable_ubsan
+          - name: tsan
+            skip_rust: true
+            build_args: --enable_tsan
+    steps:
+      - name: Checkout paimon-cpp
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          lfs: true
+          fetch-depth: ${{ matrix.fetch_depth || '1' }}
+      - name: Setup ccache
+        uses: ./.github/actions/setup-ccache
+        with:
+          cache-key-prefix: ccache-${{ matrix.name }}
+      - name: Install Rust toolchain (tantivy-fts)
+        if: ${{ !matrix.skip_rust }}
+        shell: bash
+        run: ci/scripts/setup_rust.sh
+      - name: Install HTTP and TLS development dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libcurl4-openssl-dev libssl-dev
+      - name: Build Paimon
+        shell: bash
+        env:
+          CC: ${{ matrix.cc || 'clang' }}
+          CXX: ${{ matrix.cxx || 'clang++' }}
+        run: >-
+          ci/scripts/build_paimon.sh
+          --source_dir "$(pwd)"
+          ${{ matrix.build_args }}
+      - name: Show ccache statistics
+        if: always()
+        run: ccache -s

From 6f8d6bd0544b0980ab8d09a80eb2c85306c40f56 Mon Sep 17 00:00:00 2001
From: lxy264173 
Date: Fri, 31 Jul 2026 18:42:04 +0800
Subject: [PATCH 137/138] fix: align source archive and integration tests

---
 .gitattributes                          |   1 +
 test/inte/blob_table_inte_test.cpp      |  96 +++++----
 test/inte/data_evolution_table_test.cpp | 252 +++++++++++-------------
 3 files changed, 162 insertions(+), 187 deletions(-)
 create mode 100644 .gitattributes

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000..e49bec47
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+CLAUDE.md export-ignore
diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp
index 8e0add5a..fa72f811 100644
--- a/test/inte/blob_table_inte_test.cpp
+++ b/test/inte/blob_table_inte_test.cpp
@@ -861,20 +861,18 @@ TEST_P(BlobTableInteTest, TestBasic) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(
-                arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_,
-                                SpecialFields::RowId().field_, fields_[2]}),
-                R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_,
+                            SpecialFields::RowId().field_, fields_[2]}),
+            R"([
         ["new_blob", 1, 2, 0, "c"]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(BlobTableInteTest, TestBlobFilesAcrossSchemaIds) {
@@ -1022,17 +1020,16 @@ TEST_P(BlobTableInteTest, TestMultipleAppends) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
-                                                          fields_[0],
-                                                          fields_[1],
-                                                          fields_[2],
-                                                          SpecialFields::RowId().field_,
-                                                          SpecialFields::SequenceNumber().field_,
-                                                      }),
-                                                      R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
+                                                      fields_[0],
+                                                      fields_[1],
+                                                      fields_[2],
+                                                      SpecialFields::RowId().field_,
+                                                      SpecialFields::SequenceNumber().field_,
+                                                  }),
+                                                  R"([
         [1, "a", "b", 0, 1],
         [1, "a", "b", 1, 1],
         [1, "a", "b", 2, 1],
@@ -1046,11 +1043,10 @@ TEST_P(BlobTableInteTest, TestMultipleAppends) {
         [1, "a", "b", 10, 2],
         [2, "c", "d", 11, 4]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) {
@@ -1791,25 +1787,23 @@ TEST_P(BlobTableInteTest, TestMultipleAppendsDifferentFirstRowIds) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
-                                                          fields_[0],
-                                                          fields_[1],
-                                                          fields_[2],
-                                                          SpecialFields::RowId().field_,
-                                                          SpecialFields::SequenceNumber().field_,
-                                                      }),
-                                                      R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
+                                                      fields_[0],
+                                                      fields_[1],
+                                                      fields_[2],
+                                                      SpecialFields::RowId().field_,
+                                                      SpecialFields::SequenceNumber().field_,
+                                                  }),
+                                                  R"([
         [1, "a", "b", 0, 1],
         [2, "c", "d", 1, 3]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(BlobTableInteTest, TestMoreDataWithDataEvolution) {
@@ -1946,21 +1940,19 @@ TEST_P(BlobTableInteTest, TestExternalPath) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(
-                arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().field_,
-                                SpecialFields::SequenceNumber().field_}),
-                R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().field_,
+                            SpecialFields::SequenceNumber().field_}),
+            R"([
         ["a", 10, "b", 0, 2],
         ["c", 20, "d", 1, 2]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(BlobTableInteTest, TestPartitionWithPredicate) {
diff --git a/test/inte/data_evolution_table_test.cpp b/test/inte/data_evolution_table_test.cpp
index c47f8a3f..f6a2635d 100644
--- a/test/inte/data_evolution_table_test.cpp
+++ b/test/inte/data_evolution_table_test.cpp
@@ -335,27 +335,25 @@ TEST_P(DataEvolutionTableTest, TestBasic) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(
-                arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_,
-                                SpecialFields::RowId().field_, fields_[2]}),
-                R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_,
+                            SpecialFields::RowId().field_, fields_[2]}),
+            R"([
         ["a", 1, 2, 0, "c"]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"},
-                              expected_row_tracking_array));
+    ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"},
+                          expected_row_tracking_array));
 
-        // read score but not indexed split
-        ASSERT_NOK_WITH_MSG(
-            ScanAndRead(table_path, {"f0", "f1", "_INDEX_SCORE"}, expected_row_tracking_array,
-                        /*predicate=*/nullptr,
-                        /*row_ranges=*/{}),
-            "Invalid read schema, read _INDEX_SCORE while split cannot cast to IndexedSplit");
-    }
+    // read score but not indexed split
+    ASSERT_NOK_WITH_MSG(
+        ScanAndRead(table_path, {"f0", "f1", "_INDEX_SCORE"}, expected_row_tracking_array,
+                    /*predicate=*/nullptr,
+                    /*row_ranges=*/{}),
+        "Invalid read schema, read _INDEX_SCORE while split cannot cast to IndexedSplit");
 }
 
 TEST_P(DataEvolutionTableTest, TestCommitConflictOnOverlappedRowIdAndWriteColumns) {
@@ -504,17 +502,16 @@ TEST_P(DataEvolutionTableTest, TestMultipleAppends) {
                               /*predicate=*/nullptr,
                               /*row_ranges=*/row_ranges));
     }
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
-                                                          fields_[0],
-                                                          fields_[1],
-                                                          fields_[2],
-                                                          SpecialFields::RowId().field_,
-                                                          SpecialFields::SequenceNumber().field_,
-                                                      }),
-                                                      R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
+                                                      fields_[0],
+                                                      fields_[1],
+                                                      fields_[2],
+                                                      SpecialFields::RowId().field_,
+                                                      SpecialFields::SequenceNumber().field_,
+                                                  }),
+                                                  R"([
         [1, "a", "b", 0, 1],
         [1, "a", "b", 1, 1],
         [1, "a", "b", 2, 1],
@@ -528,11 +525,10 @@ TEST_P(DataEvolutionTableTest, TestMultipleAppends) {
         [1, "a", "b", 10, 2],
         [2, "c", "d", 11, 4]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(DataEvolutionTableTest, TestOnlySomeColumns) {
@@ -579,24 +575,22 @@ TEST_P(DataEvolutionTableTest, TestOnlySomeColumns) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
-                                                          fields_[0],
-                                                          fields_[1],
-                                                          fields_[2],
-                                                          SpecialFields::RowId().field_,
-                                                          SpecialFields::SequenceNumber().field_,
-                                                      }),
-                                                      R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
+                                                      fields_[0],
+                                                      fields_[1],
+                                                      fields_[2],
+                                                      SpecialFields::RowId().field_,
+                                                      SpecialFields::SequenceNumber().field_,
+                                                  }),
+                                                  R"([
         [1, "a", "b", 0, 3]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(DataEvolutionTableTest, TestMultipleSharedShreddingMapsPartialOverwrite) {
@@ -791,24 +785,22 @@ TEST_P(DataEvolutionTableTest, TestNullValues) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
-                                                          fields_[0],
-                                                          fields_[1],
-                                                          fields_[2],
-                                                          SpecialFields::RowId().field_,
-                                                          SpecialFields::SequenceNumber().field_,
-                                                      }),
-                                                      R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
+                                                      fields_[0],
+                                                      fields_[1],
+                                                      fields_[2],
+                                                      SpecialFields::RowId().field_,
+                                                      SpecialFields::SequenceNumber().field_,
+                                                  }),
+                                                  R"([
         [1, null, "c", 0, 2]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(DataEvolutionTableTest, TestMultipleAppendsDifferentFirstRowIds) {
@@ -874,25 +866,23 @@ TEST_P(DataEvolutionTableTest, TestMultipleAppendsDifferentFirstRowIds) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
-                                                          fields_[0],
-                                                          fields_[1],
-                                                          fields_[2],
-                                                          SpecialFields::RowId().field_,
-                                                          SpecialFields::SequenceNumber().field_,
-                                                      }),
-                                                      R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({
+                                                      fields_[0],
+                                                      fields_[1],
+                                                      fields_[2],
+                                                      SpecialFields::RowId().field_,
+                                                      SpecialFields::SequenceNumber().field_,
+                                                  }),
+                                                  R"([
         [1, "a", "b", 0, 1],
         [2, "c", "d", 1, 3]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(DataEvolutionTableTest, TestMoreData) {
@@ -965,21 +955,19 @@ TEST_P(DataEvolutionTableTest, TestOnlyRowTrackingEnabled) {
     ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols0, src_array0));
     ASSERT_OK(Commit(table_path, commit_msgs));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(
-                arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_,
-                                SpecialFields::RowId().field_, fields_[2]}),
-                R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields_[1], fields_[0], SpecialFields::SequenceNumber().field_,
+                            SpecialFields::RowId().field_, fields_[2]}),
+            R"([
         ["a", 1, 1, 0, "b"],
         ["c", 2, 1, 1, "d"]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "_SEQUENCE_NUMBER", "_ROW_ID", "f2"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(DataEvolutionTableTest, TestExternalPath) {
@@ -1033,21 +1021,19 @@ TEST_P(DataEvolutionTableTest, TestExternalPath) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(
-                arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().field_,
-                                SpecialFields::SequenceNumber().field_}),
-                R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields_[1], fields_[0], fields_[2], SpecialFields::RowId().field_,
+                            SpecialFields::SequenceNumber().field_}),
+            R"([
         ["a", 10, "b", 0, 2],
         ["c", 20, "d", 1, 2]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f1", "f0", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) {
@@ -1111,10 +1097,9 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) {
             .ValueOrDie());
     ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array));
 
-    {
-        // test only read partition fields
-        auto expected_array_only_partition = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([
+    // test only read partition fields
+    auto expected_array_only_partition = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_({fields_[1]}), R"([
         ["2024"],
         ["2024"],
         ["2024"],
@@ -1122,15 +1107,15 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) {
         ["2025"],
         ["2025"]
     ])")
-                .ValueOrDie());
-        ASSERT_OK(ScanAndRead(table_path, {"f1"}, expected_array_only_partition));
+            .ValueOrDie());
+    ASSERT_OK(ScanAndRead(table_path, {"f1"}, expected_array_only_partition));
 
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(
-                arrow::struct_({fields_[0], fields_[1], fields_[2], SpecialFields::RowId().field_,
-                                SpecialFields::SequenceNumber().field_}),
-                R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields_[0], fields_[1], fields_[2], SpecialFields::RowId().field_,
+                            SpecialFields::SequenceNumber().field_}),
+            R"([
         [1, "2024", "c1", 0, 2],
         [2, "2024", "c2", 1, 2],
         [3, "2024", "c3", 2, 2],
@@ -1138,17 +1123,17 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) {
         [null, "2025", "d2", 4, 3],
         [null, "2025", "d3", 5, 3]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 
-        // read only read partition fields and row tracking
-        auto expected_partition_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(
-                arrow::struct_({fields_[1], SpecialFields::RowId().field_,
-                                SpecialFields::SequenceNumber().field_}),
-                R"([
+    // read only read partition fields and row tracking
+    auto expected_partition_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields_[1], SpecialFields::RowId().field_,
+                            SpecialFields::SequenceNumber().field_}),
+            R"([
         ["2024", 0, 2],
         ["2024", 1, 2],
         ["2024", 2, 2],
@@ -1156,11 +1141,10 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionSimple) {
         ["2025", 4, 3],
         ["2025", 5, 3]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f1", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_partition_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f1", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_partition_row_tracking_array));
 }
 
 TEST_P(DataEvolutionTableTest, TestWithPartitionWithoutPartitionFieldsInFile) {
@@ -1222,22 +1206,20 @@ TEST_P(DataEvolutionTableTest, TestWithPartitionWithoutPartitionFieldsInFile) {
                               /*row_ranges=*/row_ranges));
     }
 
-    {
-        // read with row tracking
-        auto expected_row_tracking_array = std::dynamic_pointer_cast(
-            arrow::ipc::internal::json::ArrayFromJSON(
-                arrow::struct_({fields_[0], fields_[1], fields_[2], SpecialFields::RowId().field_,
-                                SpecialFields::SequenceNumber().field_}),
-                R"([
+    // read with row tracking
+    auto expected_row_tracking_array = std::dynamic_pointer_cast(
+        arrow::ipc::internal::json::ArrayFromJSON(
+            arrow::struct_({fields_[0], fields_[1], fields_[2], SpecialFields::RowId().field_,
+                            SpecialFields::SequenceNumber().field_}),
+            R"([
         [1, "2024", "c1", 0, 2],
         [2, "2024", "c2", 1, 2],
         [3, "2024", "c3", 2, 2]
     ])")
-                .ValueOrDie());
+            .ValueOrDie());
 
-        ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
-                              expected_row_tracking_array));
-    }
+    ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "f2", "_ROW_ID", "_SEQUENCE_NUMBER"},
+                          expected_row_tracking_array));
 }
 
 TEST_P(DataEvolutionTableTest, TestPartitionWithPredicate) {

From 2624faf724fb8fa77d06a8c1361ade842bced32f Mon Sep 17 00:00:00 2001
From: lxy264173 
Date: Fri, 31 Jul 2026 20:09:17 +0800
Subject: [PATCH 138/138] ci: disable clang-tidy

---
 .github/workflows/build_and_test.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml
index ab3f22bb..a015342b 100644
--- a/.github/workflows/build_and_test.yaml
+++ b/.github/workflows/build_and_test.yaml
@@ -51,10 +51,10 @@ jobs:
             cc: gcc-14
             cxx: g++-14
           - name: clang-debug
-            fetch_depth: '0' # fetch the PR target branch history for clang-tidy
-            build_args: >-
-              --check_clang_tidy
-              --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}"
+            # fetch_depth: '0' # fetch the PR target branch history for clang-tidy
+            # build_args: >-
+            #   --check_clang_tidy
+            #   --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}"
           - name: asan-ubsan
             build_args: --enable_asan --enable_ubsan
           - name: tsan