From 37cc0f843edad703e3bc8315ba9dc21066f16a52 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 13 Jul 2026 22:35:12 +0800 Subject: [PATCH] [fix](be) Address Iceberg equality delete edge cases ### What problem does this PR solve? Issue Number: None Related PR: #65502 Problem Summary: Iceberg equality-delete handling could bind stale field ids in BY_NAME mode, lose binary initial-default bytes, treat missing V1 keys as physical columns, select id-less fields by stale ids, derive binary defaults from the latest schema instead of the selected snapshot, and read delete-file TIMESTAMPTZ keys without the data reader mapping option. Complex schema rematerialization could also trust a nested descriptor that omitted nullability, producing Struct(String) under a Struct(Nullable(String)) output type. This change uses shared name mapping, carries lossless binary defaults from the selected snapshot schema, materializes absent V1 keys as full columns, propagates TIMESTAMPTZ mapping, and treats the parent Struct DataType as the authoritative child-type contract while preserving Array and Map wrapper semantics. ### Release note Fix Iceberg equality deletes for evolved schemas, binary defaults, missing V1 keys, TIMESTAMPTZ keys, and nullable nested fields. ### Check List (For Author) - Test: Unit Test - 17 focused BE ASAN tests covering Iceberg readers, the nullable renamed struct reproduction, and rebase compatibility - 5 existing complex projection/materialization BE ASAN tests - 175 ColumnMapper and TableReader BE ASAN tests - 17 Iceberg DDL/DML planning FE tests - 29 FE tests in ExternalUtilTest, IcebergUtilsTest, and IcebergScanNodeTest - Targeted clang-format 16 check - Behavior changed: Yes, equality deletes and evolved complex columns now resolve and materialize using the correct mapping, snapshot schema, and authoritative table types - Does this need documentation: No --- be/src/format/table/iceberg_reader.cpp | 203 +++- be/src/format/table/iceberg_reader.h | 10 +- be/src/format/table/iceberg_reader_mixin.h | 142 +++ be/src/format_v2/column_data.h | 6 + be/src/format_v2/column_mapper.cpp | 88 +- be/src/format_v2/column_mapper.h | 4 + .../expr/equality_delete_predicate.cpp | 47 +- .../expr/equality_delete_predicate.h | 5 +- be/src/format_v2/file_reader.h | 3 +- be/src/format_v2/materialized_reader_util.cpp | 26 +- be/src/format_v2/table/iceberg_reader.cpp | 338 +++++- be/src/format_v2/table/iceberg_reader.h | 30 +- be/src/format_v2/table_reader.cpp | 54 + be/src/format_v2/table_reader.h | 5 +- .../table/iceberg/iceberg_reader_test.cpp | 641 +++++++++++ .../expr/equality_delete_predicate_test.cpp | 21 + be/test/format_v2/json/json_reader_test.cpp | 8 +- .../format_v2/table/iceberg_reader_test.cpp | 1016 +++++++++++++++-- be/test/format_v2/table_reader_test.cpp | 57 + .../apache/doris/datasource/ExternalUtil.java | 44 +- .../datasource/iceberg/IcebergUtils.java | 66 +- .../iceberg/source/IcebergScanNode.java | 35 +- .../doris/datasource/ExternalUtilTest.java | 12 +- .../iceberg/IcebergDDLAndDMLPlanTest.java | 4 + .../datasource/iceberg/IcebergUtilsTest.java | 43 + .../iceberg/source/IcebergScanNodeTest.java | 46 + gensrc/thrift/ExternalTableSchema.thrift | 11 +- ..._equality_delete_with_schema_change.groovy | 16 +- 28 files changed, 2712 insertions(+), 269 deletions(-) diff --git a/be/src/format/table/iceberg_reader.cpp b/be/src/format/table/iceberg_reader.cpp index 110a10f74c0c77..38bb395120df24 100644 --- a/be/src/format/table/iceberg_reader.cpp +++ b/be/src/format/table/iceberg_reader.cpp @@ -221,7 +221,8 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { _all_required_col_names = ctx->column_names; // Create column IDs from field descriptor - auto column_id_result = _create_column_ids(field_desc, ctx->tuple_descriptor); + auto column_id_result = + _create_column_ids(field_desc, ctx->tuple_descriptor, ctx->table_info_node); ctx->column_ids = std::move(column_id_result.column_ids); ctx->filter_column_ids = std::move(column_id_result.filter_column_ids); @@ -240,13 +241,20 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { // - Prefix with __equality_delete_column__ to avoid name conflicts // - Correctly map table_col_name → file_col_name in table_info_node const static std::string EQ_DELETE_PRE = "__equality_delete_column__"; - std::unordered_map field_id_to_file_col_name; + std::unordered_map field_id_to_file_column; + bool all_file_columns_have_field_ids = true; for (int i = 0; i < field_desc->size(); ++i) { - auto field_schema = field_desc->get_column(i); + const auto* field_schema = field_desc->get_column(i); if (field_schema) { - field_id_to_file_col_name[field_schema->field_id] = field_schema->name; + field_id_to_file_column[field_schema->field_id] = field_schema; + if (field_schema->field_id < 0) { + all_file_columns_have_field_ids = false; + } } } + const auto struct_node = + std::dynamic_pointer_cast(ctx->table_info_node); + DORIS_CHECK(struct_node != nullptr); // Rebuild _expand_col_names with proper file-column-based names std::vector new_expand_col_names; @@ -261,12 +269,30 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { } } - std::string file_col_name = old_name; - auto it = field_id_to_file_col_name.find(field_id); - if (it != field_id_to_file_col_name.end()) { - file_col_name = it->second; + const FieldSchema* file_column = nullptr; + if (!all_file_columns_have_field_ids && struct_node->get_children().contains(old_name) && + struct_node->children_column_exists(old_name)) { + // Iceberg files written without field ids must use schema.name-mapping.default. The + // root schema mapper deliberately switches the whole file to BY_NAME when even one + // top-level field id is absent. Hidden equality keys must make the same choice: a + // different physical column may still carry this key's stale id after migration. + const auto& mapped_name = struct_node->children_file_column_name(old_name); + for (int j = 0; j < field_desc->size(); ++j) { + const auto* candidate = field_desc->get_column(j); + if (candidate != nullptr && candidate->name == mapped_name) { + file_column = candidate; + break; + } + } + DORIS_CHECK(file_column != nullptr); + } else if (all_file_columns_have_field_ids) { + auto id_it = field_id_to_file_column.find(field_id); + if (id_it != field_id_to_file_column.end()) { + file_column = id_it->second; + } } + const std::string file_col_name = file_column == nullptr ? old_name : file_column->name; std::string table_col_name = EQ_DELETE_PRE + file_col_name; // Update _id_to_block_column_name @@ -279,18 +305,21 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { _expand_columns[i].name = table_col_name; } + if (file_column == nullptr) { + DORIS_CHECK(i < _expand_columns.size()); + RETURN_IF_ERROR(_register_missing_equality_delete_column(field_id, table_col_name, + _expand_columns[i].type)); + // The old data file predates this equality key. Keep it in the expand block so the + // synthesized-column hook can materialize its logical initial default before reader + // filtering, but do not advertise it to Parquet as a physical child. + new_expand_col_names.push_back(table_col_name); + continue; + } + new_expand_col_names.push_back(table_col_name); // Add column IDs - if (it != field_id_to_file_col_name.end()) { - for (int j = 0; j < field_desc->size(); ++j) { - auto field_schema = field_desc->get_column(j); - if (field_schema && field_schema->field_id == field_id) { - ctx->column_ids.insert(field_schema->get_column_id()); - break; - } - } - } + ctx->column_ids.insert(file_column->get_column_id()); // Register in table_info_node: table_col_name → file_col_name ctx->column_names.push_back(table_col_name); @@ -308,8 +337,9 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { // ============================================================================ // IcebergParquetReader: _create_column_ids // ============================================================================ -ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* field_desc, - const TupleDescriptor* tuple_descriptor) { +ColumnIdResult IcebergParquetReader::_create_column_ids( + const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor, + const std::shared_ptr& table_info_node) { auto* mutable_field_desc = const_cast(field_desc); mutable_field_desc->assign_ids(); @@ -335,11 +365,32 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f }; for (const auto* slot : tuple_descriptor->slots()) { - auto it = iceberg_id_to_field_schema_map.find(slot->col_unique_id()); - if (it == iceberg_id_to_field_schema_map.end()) { + const FieldSchema* field_schema = nullptr; + if (table_info_node != nullptr) { + if (table_info_node->children_column_exists(slot->col_name())) { + // Use the physical child selected by the schema-mapping pass. This keeps partial-id + // files in BY_NAME mode from binding a projected column through an unrelated stale + // field id. + const auto& file_column_name = + table_info_node->children_file_column_name(slot->col_name()); + for (int i = 0; i < field_desc->size(); ++i) { + const auto* candidate = field_desc->get_column(i); + if (candidate != nullptr && candidate->name == file_column_name) { + field_schema = candidate; + break; + } + } + DORIS_CHECK(field_schema != nullptr); + } + } else { + auto it = iceberg_id_to_field_schema_map.find(slot->col_unique_id()); + if (it != iceberg_id_to_field_schema_map.end()) { + field_schema = it->second; + } + } + if (field_schema == nullptr) { continue; } - auto field_schema = it->second; if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY && slot->col_type() != TYPE_MAP)) { @@ -358,7 +409,7 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f process_access_paths(field_schema, predicate_access_paths, filter_column_ids); } } - return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids)); + return {std::move(column_ids), std::move(filter_column_ids)}; } // ============================================================================ @@ -503,7 +554,8 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { _all_required_col_names = ctx->column_names; // Create column IDs from ORC type - auto column_id_result = _create_column_ids(orc_type_ptr, ctx->tuple_descriptor); + auto column_id_result = + _create_column_ids(orc_type_ptr, ctx->tuple_descriptor, ctx->table_info_node); ctx->column_ids = std::move(column_id_result.column_ids); ctx->filter_column_ids = std::move(column_id_result.filter_column_ids); @@ -518,15 +570,20 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { // Add expand column IDs for equality delete and remap expand column names // (matching master's behavior with __equality_delete_column__ prefix) const static std::string EQ_DELETE_PRE = "__equality_delete_column__"; - std::unordered_map field_id_to_file_col_name; + std::unordered_map field_id_to_file_column; + bool all_file_columns_have_field_ids = true; for (uint64_t i = 0; i < orc_type_ptr->getSubtypeCount(); ++i) { - std::string col_name = orc_type_ptr->getFieldName(i); const orc::Type* sub_type = orc_type_ptr->getSubtype(i); if (sub_type->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) { int fid = std::stoi(sub_type->getAttributeValue(ICEBERG_ORC_ATTRIBUTE)); - field_id_to_file_col_name[fid] = col_name; + field_id_to_file_column[fid] = sub_type; + } else { + all_file_columns_have_field_ids = false; } } + const auto struct_node = + std::dynamic_pointer_cast(ctx->table_info_node); + DORIS_CHECK(struct_node != nullptr); std::vector new_expand_col_names; for (size_t i = 0; i < _expand_col_names.size(); ++i) { @@ -539,12 +596,36 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { } } - std::string file_col_name = old_name; - auto it = field_id_to_file_col_name.find(field_id); - if (it != field_id_to_file_col_name.end()) { - file_col_name = it->second; + const orc::Type* file_column = nullptr; + if (!all_file_columns_have_field_ids && struct_node->get_children().contains(old_name) && + struct_node->children_column_exists(old_name)) { + // Match the root ORC schema mapper's all-or-nothing BY_NAME decision. Accepting a + // matching id in a partial-id file could bind this hidden key to an unrelated stale + // column instead of the current name or historical alias selected by table_info_node. + const auto& mapped_name = struct_node->children_file_column_name(old_name); + for (uint64_t j = 0; j < orc_type_ptr->getSubtypeCount(); ++j) { + if (orc_type_ptr->getFieldName(j) == mapped_name) { + file_column = orc_type_ptr->getSubtype(j); + break; + } + } + DORIS_CHECK(file_column != nullptr); + } else if (all_file_columns_have_field_ids) { + auto id_it = field_id_to_file_column.find(field_id); + if (id_it != field_id_to_file_column.end()) { + file_column = id_it->second; + } } + std::string file_col_name = old_name; + if (file_column != nullptr) { + for (uint64_t j = 0; j < orc_type_ptr->getSubtypeCount(); ++j) { + if (orc_type_ptr->getSubtype(j) == file_column) { + file_col_name = orc_type_ptr->getFieldName(j); + break; + } + } + } std::string table_col_name = EQ_DELETE_PRE + file_col_name; if (field_id >= 0) { @@ -553,18 +634,21 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { if (i < _expand_columns.size()) { _expand_columns[i].name = table_col_name; } + if (file_column == nullptr) { + DORIS_CHECK(i < _expand_columns.size()); + RETURN_IF_ERROR(_register_missing_equality_delete_column(field_id, table_col_name, + _expand_columns[i].type)); + // The old data file predates this equality key. Keep it in the expand block so the + // synthesized-column hook can materialize its logical initial default before ORC's + // block-size checks. Adding it to column_names/table_info_node would mark it as an + // existing ORC child and make OrcReader read a column that is not present in the file. + new_expand_col_names.push_back(table_col_name); + continue; + } new_expand_col_names.push_back(table_col_name); // Add column IDs - if (it != field_id_to_file_col_name.end()) { - for (uint64_t j = 0; j < orc_type_ptr->getSubtypeCount(); ++j) { - const orc::Type* sub_type = orc_type_ptr->getSubtype(j); - if (orc_type_ptr->getFieldName(j) == file_col_name) { - ctx->column_ids.insert(sub_type->getColumnId()); - break; - } - } - } + ctx->column_ids.insert(file_column->getColumnId()); ctx->column_names.push_back(table_col_name); ctx->table_info_node->add_children(table_col_name, file_col_name, @@ -578,12 +662,15 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { // ============================================================================ // IcebergOrcReader: _create_column_ids // ============================================================================ -ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type, - const TupleDescriptor* tuple_descriptor) { +ColumnIdResult IcebergOrcReader::_create_column_ids( + const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor, + const std::shared_ptr& table_info_node) { std::unordered_map iceberg_id_to_orc_type_map; for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) { - auto orc_sub_type = orc_type->getSubtype(i); - if (!orc_sub_type) continue; + const auto* orc_sub_type = orc_type->getSubtype(i); + if (!orc_sub_type) { + continue; + } if (!orc_sub_type->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) { continue; } @@ -605,11 +692,31 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type, }; for (const auto* slot : tuple_descriptor->slots()) { - auto it = iceberg_id_to_orc_type_map.find(slot->col_unique_id()); - if (it == iceberg_id_to_orc_type_map.end()) { + const orc::Type* orc_field = nullptr; + if (table_info_node != nullptr) { + if (table_info_node->children_column_exists(slot->col_name())) { + // Select the physical child resolved by the shared schema-mapping pass. Hidden + // equality keys and projected columns must obey the same BY_NAME decision for + // partial-id ORC files. + const auto& file_column_name = + table_info_node->children_file_column_name(slot->col_name()); + for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) { + if (orc_type->getFieldName(i) == file_column_name) { + orc_field = orc_type->getSubtype(i); + break; + } + } + DORIS_CHECK(orc_field != nullptr); + } + } else { + auto it = iceberg_id_to_orc_type_map.find(slot->col_unique_id()); + if (it != iceberg_id_to_orc_type_map.end()) { + orc_field = it->second; + } + } + if (orc_field == nullptr) { continue; } - const orc::Type* orc_field = it->second; if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY && slot->col_type() != TYPE_MAP)) { @@ -629,7 +736,7 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type, } } - return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids)); + return {std::move(column_ids), std::move(filter_column_ids)}; } // ============================================================================ diff --git a/be/src/format/table/iceberg_reader.h b/be/src/format/table/iceberg_reader.h index 205e46a3a1bf7b..9a237848b522f7 100644 --- a/be/src/format/table/iceberg_reader.h +++ b/be/src/format/table/iceberg_reader.h @@ -105,8 +105,9 @@ class IcebergParquetReader final : public IcebergReaderMixin { this->get_state(), this->_meta_cache); } - static ColumnIdResult _create_column_ids(const FieldDescriptor* field_desc, - const TupleDescriptor* tuple_descriptor); + static ColumnIdResult _create_column_ids( + const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor, + const std::shared_ptr& table_info_node = nullptr); private: Status _read_position_delete_file(const TFileRangeDesc* delete_range, @@ -151,8 +152,9 @@ class IcebergOrcReader final : public IcebergReaderMixin { this->get_io_ctx(), this->_meta_cache); } - static ColumnIdResult _create_column_ids(const orc::Type* orc_type, - const TupleDescriptor* tuple_descriptor); + static ColumnIdResult _create_column_ids( + const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor, + const std::shared_ptr& table_info_node = nullptr); static const std::string ICEBERG_ORC_ATTRIBUTE; diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index 2bc7be1741f756..aee211aee019cd 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -17,9 +17,12 @@ #pragma once +#include + #include #include #include +#include #include #include #include @@ -33,6 +36,7 @@ #include "core/column/column_struct.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/primitive_type.h" #include "format/generic_reader.h" #include "format/table/equality_delete.h" #include "format/table/iceberg_delete_file_reader_helper.h" @@ -40,6 +44,7 @@ #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "storage/olap_common.h" +#include "util/url_coding.h" namespace doris { class TIcebergDeleteFileDesc; @@ -135,6 +140,20 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { return _position_delete_base(data_file_path, delete_files); } + void TEST_set_column_name_to_block_index( + std::unordered_map* column_name_to_block_index) { + this->col_name_to_block_idx_ref() = column_name_to_block_index; + } + + Status TEST_register_missing_equality_delete_column(int32_t field_id, const std::string& name, + const DataTypePtr& delete_key_type) { + return _register_missing_equality_delete_column(field_id, name, delete_key_type); + } + + Status TEST_materialize_missing_equality_delete_columns(Block* block, size_t rows) { + return _materialize_missing_equality_delete_columns(block, rows); + } + protected: // ---- Hook implementations ---- @@ -251,6 +270,11 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { Status _expand_block_if_need(Block* block); Status _shrink_block_if_need(Block* block); + Status _register_missing_equality_delete_column(int32_t field_id, const std::string& name, + const DataTypePtr& delete_key_type); + Status _materialize_missing_equality_delete_column(Block* block, const std::string& name, + const ColumnPtr& value, size_t rows); + Status _materialize_missing_equality_delete_columns(Block* block, size_t rows); // Type aliases — must be defined before member function declarations that use them. using DeleteRows = std::vector; @@ -352,6 +376,7 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { const DeletionVector* _iceberg_deletion_vector = nullptr; std::vector _expand_col_names; std::vector _expand_columns; + std::unordered_map _missing_equality_delete_values; std::vector _all_required_col_names; Fileformat _file_format = Fileformat::NONE; @@ -624,6 +649,11 @@ Status IcebergReaderMixin::_expand_block_if_need(Block* block) { auto block_names = block->get_names(); names.insert(block_names.begin(), block_names.end()); for (auto& col : _expand_columns) { + if (_missing_equality_delete_values.contains(col.name)) { + // Missing equality keys are logical columns, not physical file columns. Add them only + // after the base reader has established the batch row count. + continue; + } if (names.contains(col.name)) { return Status::InternalError("Wrong expand column '{}'", col.name); } @@ -634,6 +664,118 @@ Status IcebergReaderMixin::_expand_block_if_need(Block* block) { return Status::OK(); } +template +Status IcebergReaderMixin::_register_missing_equality_delete_column( + int32_t field_id, const std::string& name, const DataTypePtr& delete_key_type) { + DORIS_CHECK(delete_key_type != nullptr); + const schema::external::TField* table_field = nullptr; + const auto& scan_params = this->get_scan_params(); + const schema::external::TSchema* current_schema = nullptr; + if (scan_params.__isset.history_schema_info && !scan_params.history_schema_info.empty()) { + current_schema = &scan_params.history_schema_info.front(); + } + if (current_schema != nullptr && scan_params.__isset.current_schema_id) { + const auto schema_it = std::ranges::find_if( + scan_params.history_schema_info, [&](const schema::external::TSchema& schema) { + return schema.__isset.schema_id && + schema.schema_id == scan_params.current_schema_id; + }); + if (schema_it != scan_params.history_schema_info.end()) { + current_schema = &*schema_it; + } else { + current_schema = nullptr; + } + } + if (current_schema != nullptr && current_schema->__isset.root_field) { + for (const auto& field_ptr : current_schema->root_field.fields) { + if (field_ptr.__isset.field_ptr && field_ptr.field_ptr != nullptr && + field_ptr.field_ptr->__isset.id && field_ptr.field_ptr->id == field_id) { + table_field = field_ptr.field_ptr.get(); + break; + } + } + } + if (table_field == nullptr) { + // A projected descriptor from an older FE may omit a hidden equality key. Without the + // current Iceberg field metadata BE cannot distinguish a true NULL initial default from a + // non-NULL default, so continuing would risk silently keeping or deleting wrong rows. + return Status::InternalError( + "Missing Iceberg schema metadata for equality-delete field id {}", field_id); + } + + Field value; + if (table_field->__isset.initial_default_value) { + const auto nested_type = remove_nullable(delete_key_type); + const bool default_is_base64 = (table_field->__isset.initial_default_value_is_base64 && + table_field->initial_default_value_is_base64) || + (table_field->__isset.type && + thrift_to_type(table_field->type.type) == TYPE_VARBINARY); + if (default_is_base64) { + std::string decoded_default; + if (!base64_decode(table_field->initial_default_value, &decoded_default)) { + return Status::InvalidArgument( + "Invalid Base64 Iceberg initial default for field {}", table_field->name); + } + if (nested_type->get_primitive_type() == TYPE_VARBINARY) { + value = Field::create_field(StringView(decoded_default)); + } else { + DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); + value = Field::create_field(decoded_default); + } + } else { + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string( + table_field->initial_default_value, value)); + } + } + const bool inserted = _missing_equality_delete_values + .emplace(name, delete_key_type->create_column_const(1, value)) + .second; + DORIS_CHECK(inserted); + this->register_synthesized_column_handler( + name, [this, name](Block* block, size_t rows) -> Status { + DORIS_CHECK(_missing_equality_delete_values.contains(name)); + return _materialize_missing_equality_delete_column( + block, name, _missing_equality_delete_values.at(name), rows); + }); + return Status::OK(); +} + +template +Status IcebergReaderMixin::_materialize_missing_equality_delete_column( + Block* block, const std::string& name, const ColumnPtr& value, size_t rows) { + if (!this->col_name_to_block_idx_ref()->contains(name)) { + // ORC must not register a key that is absent from the file as a physical child. In that + // case the reader block has no slot for the synthesized key, so append one here before + // equality-delete filtering. MultiEqualityDelete requires a full, batch-sized column. + const auto expand_col = std::ranges::find_if( + _expand_columns, + [&](const ColumnWithTypeAndName& col) { return col.name == name; }); + DORIS_CHECK(expand_col != _expand_columns.end()); + (*this->col_name_to_block_idx_ref())[name] = block->columns(); + block->insert({value->clone_resized(rows)->convert_to_full_column_if_const(), + expand_col->type, name}); + return Status::OK(); + } + const auto position = this->col_name_to_block_idx_ref()->at(name); + DORIS_CHECK(position < block->columns()); + DORIS_CHECK(block->get_by_position(position).column->empty()); + // MultiEqualityDelete hashes each key column directly. Materialize the repeated default so + // every key has the batch row count; a ColumnConst keeps only one nested value and therefore + // cannot participate in the row-wise multi-column hash contract. + block->get_by_position(position).column = + value->clone_resized(rows)->convert_to_full_column_if_const(); + return Status::OK(); +} + +template +Status IcebergReaderMixin::_materialize_missing_equality_delete_columns(Block* block, + size_t rows) { + for (const auto& [name, value] : _missing_equality_delete_values) { + RETURN_IF_ERROR(_materialize_missing_equality_delete_column(block, name, value, rows)); + } + return Status::OK(); +} + template Status IcebergReaderMixin::_shrink_block_if_need(Block* block) { std::set positions_to_erase; diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index 7816ea8263cb42..ac510caaf07ac4 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -258,6 +258,12 @@ struct ColumnDefinition { // Expression used to materialize missing/default/generated values when the column is not read // directly from the file. VExprContextSPtr default_expr = nullptr; + // Table-format initial default normalized for transport from FE. Binary-like values use Base64 + // and set initial_default_value_is_base64 because they can map to STRING/CHAR or VARBINARY. + // Unlike default_expr, this metadata is also available for hidden delete-predicate columns + // that are absent from the query projection. + std::optional initial_default_value = std::nullopt; + bool initial_default_value_is_base64 = false; // Partition columns are constants from split metadata and should not be matched against file // schema unless table-format logic explicitly asks for it. bool is_partition_key = false; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 572e76280a0e37..6a0daf903f01d4 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -230,6 +230,11 @@ std::string join_debug_strings(const std::vector& values, Formatter formatter } // namespace +const ColumnDefinition* find_column_by_name(const ColumnDefinition& table_column, + const std::vector& file_schema) { + return matcher_for_mode(TableColumnMappingMode::BY_NAME).find(table_column, file_schema); +} + const Field* find_partition_value(const ColumnDefinition& table_column, const std::map& partition_values) { const auto find_by_name = [&](const std::string& name) -> const Field* { @@ -1237,6 +1242,26 @@ static std::vector synthesize_complex_children_from_type( return children; } +static void align_struct_child_types_with_parent(const DataTypePtr& parent_type, + std::vector& children) { + const auto nested_parent_type = remove_nullable(parent_type); + DORIS_CHECK(nested_parent_type->get_primitive_type() == TYPE_STRUCT); + const auto type_children = synthesize_complex_children_from_type(parent_type); + for (auto& child : children) { + const auto type_child = std::ranges::find_if( + type_children, [&](const auto& candidate) { return candidate.name == child.name; }); + DORIS_CHECK(type_child != type_children.end()) + << "Complex child '" << child.name + << "' is absent from its parent table type: " << parent_type->get_name(); + // The parent DataType is the authoritative output contract. Nested schema descriptors can + // omit child nullability even though the parent struct still declares Nullable(String). + // For example, the Iceberg full-schema-change case maps nullable `location` to `city`, but + // its child descriptor carries String. Keeping String here makes rematerialization strip + // the child's null map and creates Struct(String) under a Struct(Nullable(String)) type. + child.type = type_child->type; + } +} + static bool has_table_child_named(const std::vector& children, std::string_view name) { return std::ranges::any_of(children, [&](const ColumnDefinition& child) { @@ -1245,8 +1270,7 @@ static bool has_table_child_named(const std::vector& children, } static void complete_required_complex_children_from_type(const DataTypePtr& type, - std::vector* children) { - DORIS_CHECK(children != nullptr); + std::vector& children) { if (type == nullptr) { return; } @@ -1261,8 +1285,8 @@ static void complete_required_complex_children_from_type(const DataTypePtr& type // In that shape the scanner keeps the value stream readable, but the table projection can // carry only the key child. Add the missing value child so recursive mapping can evolve the // value type instead of letting TableReader cast old/new value structs directly. - if (has_table_child_named(*children, "key") && !has_table_child_named(*children, "value")) { - children->push_back(synthetic_child_definition("value", map_type->get_value_type(), 1)); + if (has_table_child_named(children, "key") && !has_table_child_named(children, "value")) { + children.push_back(synthetic_child_definition("value", map_type->get_value_type(), 1)); } break; } @@ -1279,6 +1303,37 @@ static void complete_required_complex_children_from_type(const DataTypePtr& type } } +struct PreparedTableChildren { + std::vector children; + bool synthesized_from_type = false; +}; + +static PreparedTableChildren prepare_table_children_for_mapping( + const ColumnDefinition& table_column, const DataTypePtr& file_type) { + PreparedTableChildren prepared {.children = table_column.children}; + const auto nested_table_type = remove_nullable(table_column.type); + + // Some scan paths, especially SELECT *, only carry the complete complex DataType for a table + // column and leave ColumnDefinition::children empty. Synthesize the hierarchy so recursive + // mapping can evolve nested fields instead of falling back to an invalid whole-column cast. + prepared.synthesized_from_type = prepared.children.empty() && + is_complex_type(nested_table_type->get_primitive_type()) && + !table_column.type->equals(*file_type); + if (prepared.synthesized_from_type) { + prepared.children = synthesize_complex_children_from_type(table_column.type); + } else if (!prepared.children.empty() && !table_column.type->equals(*file_type)) { + complete_required_complex_children_from_type(table_column.type, prepared.children); + } + + if (!prepared.children.empty() && nested_table_type->get_primitive_type() == TYPE_STRUCT) { + // Struct children are table fields, so the parent Struct type is authoritative for their + // nullability. ARRAY and MAP children are format-level structural wrappers and keep the + // descriptor types used by their recursive mappings. + align_struct_child_types_with_parent(table_column.type, prepared.children); + } + return prepared; +} + static Status validate_file_schema_children(const ColumnDefinition& file_field) { if (file_field.type == nullptr) { return Status::InternalError("File column '{}' has null type", file_field.name); @@ -2016,29 +2071,8 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c mapping->filter_conversion = direct_filter_conversion(*mapping); mapping->child_mappings.clear(); - auto table_children = table_column.children; - const auto nested_table_type = remove_nullable(mapping->table_type); - // Some scan paths, especially SELECT *, only carry the complete complex DataType for a table - // column and leave ColumnDefinition::children empty. If the file type is an older complex - // schema, treating this as a leaf mapping would make TableReader fall back to a plain CAST. - // That is invalid for evolved structs with different field counts. - // - // Example: - // table column type: Map(String, Struct(age, full_name, gender)) - // old file type: Map(String, Struct(age, name)) - // table children: empty - // - // Synthesize key/value/struct-field children from the table type so the normal recursive - // mapping path can rematerialize `name -> full_name` and fill missing `gender` with defaults, - // instead of trying to CAST Struct(age, name) to Struct(age, full_name, gender). - const bool synthesized_table_children = - table_children.empty() && is_complex_type(nested_table_type->get_primitive_type()) && - !mapping->table_type->equals(*mapping->file_type); - if (synthesized_table_children) { - table_children = synthesize_complex_children_from_type(mapping->table_type); - } else if (!table_children.empty() && !mapping->table_type->equals(*mapping->file_type)) { - complete_required_complex_children_from_type(mapping->table_type, &table_children); - } + auto [table_children, synthesized_table_children] = + prepare_table_children_for_mapping(table_column, mapping->file_type); if (!table_children.empty()) { if (!is_complex_type(remove_nullable(mapping->file_type)->get_primitive_type())) { diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 28e31a12639936..4f417a680aad60 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -163,6 +163,10 @@ struct TableColumnMapperOptions { Status clone_table_expr_tree(const VExprSPtr& expr, VExprSPtr* cloned_expr); const Field* find_partition_value(const ColumnDefinition& table_column, const std::map& partition_values); +// Apply the same case-insensitive logical name, string identifier, and bidirectional alias rules +// used by TableColumnMapper's BY_NAME mode. +const ColumnDefinition* find_column_by_name(const ColumnDefinition& table_column, + const std::vector& file_schema); // Generic mapping layer from table schema to file schema. // Iceberg uses BY_FIELD_ID. Plain by-name formats can reuse this component as well, so keep this diff --git a/be/src/format_v2/expr/equality_delete_predicate.cpp b/be/src/format_v2/expr/equality_delete_predicate.cpp index 13454e3b22f116..4d5c511abdb181 100644 --- a/be/src/format_v2/expr/equality_delete_predicate.cpp +++ b/be/src/format_v2/expr/equality_delete_predicate.cpp @@ -19,6 +19,7 @@ #include +#include #include #include "common/status.h" @@ -92,29 +93,52 @@ void EqualityDeletePredicate::close(VExprContext* context, Status EqualityDeletePredicate::execute(VExprContext* context, Block* block, int* result_column_id) const { + size_t rows = 0; + for (const auto& column : block->get_columns()) { + rows = std::max(rows, column->size()); + } + // Lazy readers may leave an unread projected column at block position zero while predicate + // columns (or the all-missing-key row-count carrier) are populated later. Block::rows() only + // inspects the first column, so derive the explicit expression count from all materialized + // columns and let execute_column() enforce that every result has exactly this batch size. + ColumnPtr result_column; + RETURN_IF_ERROR(execute_column(context, block, nullptr, rows, result_column)); + block->insert({std::move(result_column), std::make_shared(), expr_name()}); + *result_column_id = static_cast(block->columns() - 1); + return Status::OK(); +} + +Status EqualityDeletePredicate::execute_column_impl(VExprContext* context, const Block* block, + const Selector* selector, size_t count, + ColumnPtr& result_column) const { if (_children.size() != _field_ids.size()) { return Status::InternalError( "EqualityDeletePredicate should have {} child exprs, but got {}", _field_ids.size(), _children.size()); } + // This path is required when every equality key is a literal, notably when all key columns are + // absent from an older Iceberg data file and are represented by typed NULL literals. Preserve + // `count` so the constant result can be expanded to the caller's batch size when necessary. Block data_key_block; for (const auto& child : _children) { - Block eval_block = *block; - int slot = -1; - RETURN_IF_ERROR(child->execute(context, &eval_block, &slot)); - const auto& key_column = eval_block.get_by_position(slot); - data_key_block.insert({key_column.column, key_column.type, key_column.name}); + ColumnPtr key_column; + RETURN_IF_ERROR(child->execute_column(context, block, selector, count, key_column)); + // Equality comparison operates on row-addressable columns. Materialize literal constants + // so nullable NULL keys and regular slot columns share the same compare_at contract. + data_key_block.insert({key_column->convert_to_full_column_if_const(), + child->execute_type(block), child->expr_name()}); } + result_column = _evaluate_key_block(data_key_block); + return Status::OK(); +} +ColumnPtr EqualityDeletePredicate::_evaluate_key_block(const Block& data_key_block) const { const auto rows = data_key_block.rows(); auto res_col = ColumnBool::create(rows, 0); if (_delete_hash_map.empty() || rows == 0) { - block->insert({std::move(res_col), std::make_shared(), expr_name()}); - *result_column_id = static_cast(block->columns() - 1); - return Status::OK(); + return res_col; } - auto data_hashes = _build_hashes(data_key_block); auto& result_data = res_col->get_data(); for (size_t row = 0; row < rows; ++row) { @@ -126,10 +150,7 @@ Status EqualityDeletePredicate::execute(VExprContext* context, Block* block, } } } - - block->insert({std::move(res_col), std::make_shared(), expr_name()}); - *result_column_id = static_cast(block->columns() - 1); - return Status::OK(); + return res_col; } std::vector EqualityDeletePredicate::_build_hashes(const Block& block) { diff --git a/be/src/format_v2/expr/equality_delete_predicate.h b/be/src/format_v2/expr/equality_delete_predicate.h index cad16ca387ccd8..0e6f127cee23c1 100644 --- a/be/src/format_v2/expr/equality_delete_predicate.h +++ b/be/src/format_v2/expr/equality_delete_predicate.h @@ -46,9 +46,7 @@ class EqualityDeletePredicate final : public VExpr { Status execute(VExprContext* context, Block* block, int* result_column_id) const override; Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, - size_t count, ColumnPtr& result_column) const override { - return Status::InternalError("Not implement EqualityDeletePredicate::execute_column_impl"); - } + size_t count, ColumnPtr& result_column) const override; Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; Status open(RuntimeState* state, VExprContext* context, FunctionContext::FunctionStateScope scope) override; @@ -59,6 +57,7 @@ class EqualityDeletePredicate final : public VExpr { private: static std::vector _build_hashes(const Block& block); + ColumnPtr _evaluate_key_block(const Block& data_key_block) const; bool _equal(const Block& data_block, size_t data_row, size_t delete_row) const; std::string _expr_name; diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 135df2820145e3..59f684121ce1e3 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -74,7 +74,8 @@ struct FileScanRequest { std::map local_positions; // Row-level filters converted to file-local expressions from table-level predicates. VExprContextSPtrs conjuncts; - // Delete predicates converted to file-local expressions. + // Delete predicates converted to file-local expressions. A TRUE result means that the row is + // deleted, so readers must invert each result when building their keep filter. VExprContextSPtrs delete_conjuncts; }; diff --git a/be/src/format_v2/materialized_reader_util.cpp b/be/src/format_v2/materialized_reader_util.cpp index a7e533633510c4..d27f066b30bc2b 100644 --- a/be/src/format_v2/materialized_reader_util.cpp +++ b/be/src/format_v2/materialized_reader_util.cpp @@ -20,7 +20,9 @@ #include #include "core/block/block.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" +#include "core/types.h" #include "exprs/vexpr_context.h" #include "format_v2/file_reader.h" #include "io/io_common.h" @@ -54,8 +56,28 @@ Status apply_materialized_reader_filters(const FileScanRequest* request, io::IOC if (request != nullptr && rows_before_filter > 0 && !request->delete_conjuncts.empty()) { { SCOPED_TIMER(profile == nullptr ? nullptr : profile->delete_conjunct_filter_time); - RETURN_IF_ERROR(VExprContext::filter_block(request->delete_conjuncts, file_block, - file_block->columns())); + // Delete conjuncts use the opposite polarity from ordinary predicates: TRUE marks a + // row for deletion. Multiple delete files are combined as a union, so a row remains + // visible only when every delete conjunct returns FALSE. + IColumn::Filter keep_filter(rows_before_filter, 1); + for (const auto& delete_conjunct : request->delete_conjuncts) { + DORIS_CHECK(delete_conjunct != nullptr); + int result_column_id = -1; + RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, + &result_column_id)); + DORIS_CHECK(result_column_id >= 0 && + result_column_id < static_cast(file_block->columns())); + const auto& delete_filter = + assert_cast( + *file_block->get_by_position(result_column_id).column) + .get_data(); + DORIS_CHECK(delete_filter.size() == rows_before_filter); + for (size_t row = 0; row < rows_before_filter; ++row) { + keep_filter[row] &= !delete_filter[row]; + } + file_block->erase(result_column_id); + } + RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(file_block, keep_filter)); } rows_after_delete_filter = file_block->columns() == 0 ? rows_before_filter : file_block->rows(); diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 1598fa783cda44..42f78d38c46bd7 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -34,6 +34,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/define_primitive_type.h" #include "core/field.h" +#include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" #include "format_v2/expr/cast.h" @@ -43,6 +44,7 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "io/file_factory.h" +#include "util/url_coding.h" namespace doris::format::iceberg { @@ -77,6 +79,54 @@ static bool is_projected_iceberg_rowid(const format::ColumnDefinition& column) { return column.name == BeConsts::ICEBERG_ROWID_COL; } +static Status build_missing_equality_delete_key_expr(const format::ColumnDefinition& table_field, + const DataTypePtr& delete_key_type, + VExprSPtr* key_expr) { + DORIS_CHECK(delete_key_type != nullptr); + DORIS_CHECK(key_expr != nullptr); + if (!table_field.initial_default_value.has_value()) { + // A newly added optional field without an initial default is logically NULL in older + // files. EqualityDeletePredicate treats NULL == NULL as a match. + *key_expr = VLiteral::create_shared(make_nullable(delete_key_type), Field()); + return Status::OK(); + } + + Field initial_default; + if (table_field.initial_default_value_is_base64 || + table_field.type->get_primitive_type() == TYPE_VARBINARY) { + // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its + // Doris mapping. Keep the VARBINARY fallback for scan descriptors produced before that + // marker existed. Decode before parsing so STRING/CHAR and VARBINARY all compare against + // the raw bytes stored in equality-delete files. + std::string decoded_default; + if (!base64_decode(*table_field.initial_default_value, &decoded_default)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + table_field.name); + } + if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { + initial_default = Field::create_field(StringView(decoded_default)); + } else { + DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); + initial_default = Field::create_field(decoded_default); + } + } else { + // An added field's initial default is its logical value in every older data file that lacks + // the physical column. FE normalizes the string for the current Doris table type. + RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( + *table_field.initial_default_value, initial_default)); + } + + auto literal = VLiteral::create_shared(table_field.type, initial_default); + if (table_field.type->equals(*delete_key_type)) { + *key_expr = std::move(literal); + return Status::OK(); + } + auto cast_expr = Cast::create_shared(delete_key_type); + cast_expr->add_child(std::move(literal)); + *key_expr = std::move(cast_expr); + return Status::OK(); +} + static std::string iceberg_delete_file_debug_string(const TIcebergDeleteFileDesc& delete_file) { std::ostringstream out; out << "TIcebergDeleteFileDesc{path=" << (delete_file.__isset.path ? delete_file.path : "null") @@ -143,8 +193,10 @@ static std::string iceberg_params_debug_string(const std::optional(*remove_nullable(pos_column_ptr)); for (size_t row = 0; row < read_rows; ++row) { const auto file_path = file_path_column.get_data_at(row).to_string(); - if (file_path == _data_file_path) { - _rows->push_back(pos_column.get_element(row)); - } + (*_rows_by_data_file)[file_path].push_back(pos_column.get_element(row)); } return Status::OK(); } @@ -180,6 +230,7 @@ Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options _delete_predicates_initialized = false; _position_delete_rows_storage.clear(); _equality_delete_filters.clear(); + _split_cache = options.cache; if (options.current_range.__isset.table_format_params && options.current_range.table_format_params.__isset.iceberg_params) { const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; @@ -438,31 +489,114 @@ Status IcebergTableReader::_append_row_position_output_column(format::FileScanRe return Status::OK(); } +const format::ColumnDefinition* IcebergTableReader::_find_equality_delete_data_field( + const EqualityDeleteFilter& filter, size_t key_idx) const { + DORIS_CHECK(key_idx < filter.field_ids.size()); + DORIS_CHECK(key_idx < filter.field_names.size()); + if (mapping_mode() != format::TableColumnMappingMode::BY_NAME) { + const int field_id = filter.field_ids[key_idx]; + const auto field_it = std::ranges::find_if( + _data_reader.file_schema, [field_id](const format::ColumnDefinition& field) { + return field.has_identifier_field_id() && + field.get_identifier_field_id() == field_id; + }); + return field_it == _data_reader.file_schema.end() ? nullptr : &*field_it; + } + + // Equality keys are hidden scan dependencies and need not appear in the query projection. + // Resolve their current name and aliases from the full table schema supplied by FE, falling + // back to the delete-file name when history metadata is unavailable. Reuse ColumnMapper's + // exact BY_NAME rules so case, string identifiers, and aliases on either side stay consistent. + auto table_field = _find_equality_delete_table_field(filter, key_idx); + return format::find_column_by_name(*table_field, _data_reader.file_schema); +} + +std::optional IcebergTableReader::_find_equality_delete_table_field( + const EqualityDeleteFilter& filter, size_t key_idx) const { + DORIS_CHECK(key_idx < filter.field_ids.size()); + DORIS_CHECK(key_idx < filter.field_names.size()); + const int field_id = filter.field_ids[key_idx]; + auto table_field = _find_current_table_column_by_field_id(field_id, filter.key_types[key_idx]); + if (!table_field.has_value()) { + const auto projected_field = std::ranges::find_if( + _projected_columns, [field_id](const format::ColumnDefinition& field) { + return field.has_identifier_field_id() && + field.get_identifier_field_id() == field_id; + }); + if (projected_field != _projected_columns.end()) { + // Older scan descriptors and focused unit tests may omit history_schema_info. Keep the + // projected metadata as a compatibility fallback, but never require projection when + // the complete current schema is available. + table_field = *projected_field; + } + } + if (!table_field.has_value()) { + table_field = format::ColumnDefinition { + .identifier = {}, + .name = filter.field_names[key_idx], + .type = filter.key_types[key_idx], + }; + } + return table_field; +} + +std::string IcebergTableReader::_delete_file_cache_key(const char* prefix, + const std::string& path) const { + DORIS_CHECK(prefix != nullptr); + std::string fs_name; + if (_current_task != nullptr && _current_task->data_file != nullptr) { + fs_name = _current_task->data_file->fs_name; + } + // Delete descriptors can reuse the same path text in different filesystem namespaces. Encode + // both variable-length strings so neither an fs/path boundary nor equality field-id suffixes + // can be reinterpreted as path content; scan-level credentials/properties are shared here. + std::ostringstream key; + key << prefix << fs_name.size() << ':' << fs_name << ':' << path.size() << ':' << path; + return key.str(); +} + +void IcebergTableReader::_append_equality_delete_row_count_carrier( + format::FileScanRequest* request) { + DORIS_CHECK(request != nullptr); + // Columnar readers establish a filter batch's row count from predicate columns. If all + // equality keys are missing, the predicate consists only of NULL literals and the filter block + // would otherwise have zero rows. Read one physical column eagerly as a row-count carrier; + // normal final materialization ignores this hidden dependency. + const auto carrier_it = std::ranges::find_if( + _data_reader.file_schema, [](const format::ColumnDefinition& field) { + return field.column_type == format::ColumnType::DATA_COLUMN; + }); + DORIS_CHECK(carrier_it != _data_reader.file_schema.end()); + _append_file_scan_column(request, format::LocalColumnId(carrier_it->file_local_id()), + &request->predicate_columns); +} + Status IcebergTableReader::_append_equality_delete_predicates(format::FileScanRequest* request) { DORIS_CHECK(request != nullptr); for (const auto& filter : _equality_delete_filters) { auto delete_predicate = std::make_shared(filter.delete_block, filter.field_ids); DCHECK_EQ(filter.field_ids.size(), filter.key_types.size()); + bool has_missing_key = false; for (size_t idx = 0; idx < filter.field_ids.size(); ++idx) { - const int field_id = filter.field_ids[idx]; - auto field_it = std::ranges::find_if( - _data_reader.file_schema, [field_id](const format::ColumnDefinition& field) { - return field.has_identifier_field_id() && - field.get_identifier_field_id() == field_id; - }); - if (field_it == _data_reader.file_schema.end()) { - return Status::InternalError( - "Can not find equality delete column field id {} in data file schema", - field_id); + const auto* field = _find_equality_delete_data_field(filter, idx); + if (field == nullptr) { + auto table_field = _find_equality_delete_table_field(filter, idx); + DORIS_CHECK(table_field.has_value()); + VExprSPtr key_expr; + RETURN_IF_ERROR(build_missing_equality_delete_key_expr( + *table_field, filter.key_types[idx], &key_expr)); + delete_predicate->add_child(key_expr); + has_missing_key = true; + continue; } - const auto field_column_id = format::LocalColumnId(field_it->file_local_id()); + const auto field_column_id = format::LocalColumnId(field->file_local_id()); _append_file_scan_column(request, field_column_id, &request->predicate_columns); const auto block_position = request->local_positions.at(field_column_id).value(); auto slot = VSlotRef::create_shared(cast_set(block_position), - cast_set(block_position), -1, field_it->type, - field_it->name); - if (field_it->type->equals(*filter.key_types[idx])) { + cast_set(block_position), -1, field->type, + field->name); + if (field->type->equals(*filter.key_types[idx])) { delete_predicate->add_child(std::move(slot)); } else { auto cast_expr = Cast::create_shared(filter.key_types[idx]); @@ -470,6 +604,9 @@ Status IcebergTableReader::_append_equality_delete_predicates(format::FileScanRe delete_predicate->add_child(std::move(cast_expr)); } } + if (has_missing_key && request->predicate_columns.empty()) { + _append_equality_delete_row_count_carrier(request); + } request->delete_conjuncts.push_back( VExprContext::create_shared(std::move(delete_predicate))); } @@ -498,12 +635,16 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe auto system_properties = _delete_file_system_properties(scan_params); auto file_description = _delete_file_description(delete_range); std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); + const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && + scan_params.enable_mapping_timestamp_tz; if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { *reader = std::make_unique( - system_properties, file_description, io_ctx, _scanner_profile); + system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz); } else { *reader = std::make_unique(system_properties, file_description, - io_ctx, _scanner_profile); + io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz); } RETURN_IF_ERROR((*reader)->init(_runtime_state)); return Status::OK(); @@ -566,14 +707,50 @@ Status IcebergTableReader::_read_position_delete_file(const TIcebergDeleteFileDe Status IcebergTableReader::_init_position_delete_rows( const std::vector& delete_files) { + DORIS_CHECK(_split_cache != nullptr); TFileScanRangeParams delete_scan_params = _scan_params == nullptr ? TFileScanRangeParams() : *_scan_params; format::DeleteRows position_delete_rows; IcebergDeleteFileIOContext delete_io_ctx(_runtime_state); - PositionDeleteRowsCollector collector(_data_file_path(), &position_delete_rows); for (const auto& delete_file : delete_files) { - RETURN_IF_ERROR(_read_position_delete_file(delete_file, delete_scan_params, &delete_io_ctx, - &collector)); + Status read_status = Status::OK(); + // A position delete file normally references many data files. Cache the complete + // path-to-position map once; caching only the current data file would still rescan the + // shared delete file for every subsequent split. + auto* rows_by_data_file = + _split_cache->get( + _delete_file_cache_key("iceberg_v2_position_delete_", delete_file.path), + [&]() -> PositionDeleteRowsCollector::PositionDeleteFile* { + auto result = std::make_unique< + PositionDeleteRowsCollector::PositionDeleteFile>(); + PositionDeleteRowsCollector collector(result.get()); + read_status = _read_position_delete_file( + delete_file, delete_scan_params, &delete_io_ctx, &collector); + if (!read_status.ok()) { + return nullptr; + } + for (auto& [_, rows] : *result) { + std::ranges::sort(rows); + } + return result.release(); + }); + RETURN_IF_ERROR(read_status); + DORIS_CHECK(rows_by_data_file != nullptr); + const auto rows_it = rows_by_data_file->find(_data_file_path()); + if (rows_it == rows_by_data_file->end()) { + continue; + } + auto first = rows_it->second.begin(); + auto last = rows_it->second.end(); + // Bounds are inclusive Iceberg position statistics supplied by FE. Apply them after the + // cached per-data-file vector is sorted so irrelevant positions are sliced without a scan. + if (delete_file.__isset.position_lower_bound) { + first = std::lower_bound(first, last, delete_file.position_lower_bound); + } + if (delete_file.__isset.position_upper_bound) { + last = std::upper_bound(first, last, delete_file.position_upper_bound); + } + position_delete_rows.insert(position_delete_rows.end(), first, last); } if (position_delete_rows.empty()) { return Status::OK(); @@ -593,6 +770,7 @@ Status IcebergTableReader::_init_position_delete_rows( Status IcebergTableReader::_init_equality_delete_predicates( const std::vector& delete_files) { + DORIS_CHECK(_split_cache != nullptr); TFileScanRangeParams delete_scan_params = _scan_params == nullptr ? TFileScanRangeParams() : *_scan_params; IcebergDeleteFileIOContext delete_io_ctx(_runtime_state); @@ -603,27 +781,18 @@ Status IcebergTableReader::_init_equality_delete_predicates( return Status::OK(); } -Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, - const TFileScanRangeParams& scan_params, - IcebergDeleteFileIOContext* delete_io_ctx) { - if (!delete_file.__isset.field_ids || delete_file.field_ids.empty()) { - return Status::InternalError("Iceberg equality delete file is missing field ids"); - } - std::unique_ptr reader; - RETURN_IF_ERROR(_create_delete_file_reader(delete_file, scan_params, delete_io_ctx, &reader)); - DORIS_CHECK(reader != nullptr); - - std::vector schema; - RETURN_IF_ERROR(reader->get_schema(&schema)); - std::vector delete_fields; - std::vector delete_field_ids; - std::vector delete_key_types; +Status IcebergTableReader::_resolve_equality_delete_fields( + const TIcebergDeleteFileDesc& delete_file, + const std::vector& schema, + std::vector* delete_fields, EqualityDeleteFilter* result) const { + DORIS_CHECK(delete_fields != nullptr); + DORIS_CHECK(result != nullptr); for (const auto field_id : delete_file.field_ids) { - auto field_it = std::find_if(schema.begin(), schema.end(), - [field_id](const format::ColumnDefinition& field) { - return field.has_identifier_field_id() && - field_id == field.get_identifier_field_id(); - }); + const auto field_it = + std::ranges::find_if(schema, [field_id](const format::ColumnDefinition& field) { + return field.has_identifier_field_id() && + field_id == field.get_identifier_field_id(); + }); if (field_it == schema.end()) { return Status::InternalError("Can not find field id {} in equality delete file {}", field_id, delete_file.path); @@ -632,12 +801,36 @@ Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDe return Status::NotSupported( "Iceberg equality delete does not support complex column {}", field_it->name); } - delete_fields.push_back(*field_it); - delete_field_ids.push_back(field_id); - delete_key_types.push_back(field_it->type); + delete_fields->push_back(*field_it); + result->field_ids.push_back(field_id); + result->field_names.push_back(field_it->name); + result->key_types.push_back(field_it->type); } + return Status::OK(); +} + +Status IcebergTableReader::_load_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + EqualityDeleteFilter* result) { + DORIS_CHECK(result != nullptr); + std::unique_ptr reader; + RETURN_IF_ERROR(_create_delete_file_reader(delete_file, scan_params, delete_io_ctx, &reader)); + DORIS_CHECK(reader != nullptr); + + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + std::vector delete_fields; + RETURN_IF_ERROR(_resolve_equality_delete_fields(delete_file, schema, &delete_fields, result)); auto request = std::make_shared(); + auto build_block = [](const std::vector& fields) -> Block { + Block block; + for (const auto& field : fields) { + block.insert({field.type->create_column(), field.type, field.name}); + } + return block; + }; for (size_t idx = 0; idx < delete_fields.size(); ++idx) { const auto local_column_id = format::LocalColumnId(delete_fields[idx].file_local_id()); request->non_predicate_columns.push_back( @@ -646,19 +839,10 @@ Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDe } RETURN_IF_ERROR(reader->open(request)); - auto build_equality_delete_block = - [](const std::vector fields) -> Block { - Block block; - for (const auto& field : fields) { - block.insert({field.type->create_column(), field.type, field.name}); - } - return block; - }; - Block delete_block = build_equality_delete_block(delete_fields); - MutableBlock mutable_delete_block(std::move(delete_block)); + MutableBlock mutable_delete_block(build_block(delete_fields)); bool eof = false; while (!eof) { - Block block = build_equality_delete_block(delete_fields); + Block block = build_block(delete_fields); size_t read_rows = 0; RETURN_IF_ERROR(reader->get_block(&block, &read_rows, &eof)); if (read_rows > 0) { @@ -666,11 +850,39 @@ Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDe } } RETURN_IF_ERROR(reader->close()); - delete_block = mutable_delete_block.to_block(); - _equality_delete_filters.push_back( - EqualityDeleteFilter {.field_ids = std::move(delete_field_ids), - .key_types = std::move(delete_key_types), - .delete_block = std::move(delete_block)}); + result->delete_block = mutable_delete_block.to_block(); + return Status::OK(); +} + +Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx) { + if (!delete_file.__isset.field_ids || delete_file.field_ids.empty()) { + return Status::InternalError("Iceberg equality delete file is missing field ids"); + } + std::ostringstream cache_key; + cache_key << _delete_file_cache_key("iceberg_v2_equality_delete_", delete_file.path); + cache_key << ':' << delete_file.field_ids.size(); + for (const auto field_id : delete_file.field_ids) { + cache_key << ':' << field_id; + } + Status read_status = Status::OK(); + // Include the ordered equality ids in the key because the same physical delete file can be + // projected with different key layouts. The cached block and its key metadata are immutable + // after construction and therefore safe to copy into each split-local predicate. + auto* cached_filter = _split_cache->get( + cache_key.str(), [&]() -> EqualityDeleteFilter* { + auto result = std::make_unique(); + read_status = _load_equality_delete_file(delete_file, scan_params, delete_io_ctx, + result.get()); + if (!read_status.ok()) { + return nullptr; + } + return result.release(); + }); + RETURN_IF_ERROR(read_status); + DORIS_CHECK(cached_filter != nullptr); + _equality_delete_filters.push_back(*cached_filter); return Status::OK(); } diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index 6903310bb6e34c..d328f574f2b22b 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "common/status.h" @@ -74,6 +75,8 @@ class IcebergTableReader : public format::TableReader { Status _init_delete_predicates(const TTableFormatFileDesc& t_desc); private: + struct EqualityDeleteFilter; + bool _has_field_id(const std::vector& schema) const { for (const auto& field : schema) { // TopN lazy materialization asks the file reader to synthesize GLOBAL_ROWID in the @@ -110,13 +113,14 @@ class IcebergTableReader : public format::TableReader { class PositionDeleteRowsCollector final { public: - PositionDeleteRowsCollector(std::string data_file_path, format::DeleteRows* rows); + using PositionDeleteFile = std::unordered_map; + + explicit PositionDeleteRowsCollector(PositionDeleteFile* rows_by_data_file); Status collect(const Block& block, size_t read_rows); private: - std::string _data_file_path; - format::DeleteRows* _rows = nullptr; + PositionDeleteFile* _rows_by_data_file = nullptr; }; static std::shared_ptr _delete_file_system_properties( @@ -132,6 +136,12 @@ class IcebergTableReader : public format::TableReader { // Append equality delete predicates to file scan request based on the delete files in iceberg // params. DeleteVector and position delete files use the common DeleteRows path in TableReader. Status _append_equality_delete_predicates(format::FileScanRequest* request); + const format::ColumnDefinition* _find_equality_delete_data_field( + const EqualityDeleteFilter& filter, size_t key_idx) const; + std::optional _find_equality_delete_table_field( + const EqualityDeleteFilter& filter, size_t key_idx) const; + void _append_equality_delete_row_count_carrier(format::FileScanRequest* request); + std::string _delete_file_cache_key(const char* prefix, const std::string& path) const; Status _init_equality_delete_predicates( const std::vector& delete_files); @@ -144,6 +154,14 @@ class IcebergTableReader : public format::TableReader { Status _read_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, const TFileScanRangeParams& scan_params, IcebergDeleteFileIOContext* delete_io_ctx); + Status _load_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + EqualityDeleteFilter* result); + Status _resolve_equality_delete_fields(const TIcebergDeleteFileDesc& delete_file, + const std::vector& schema, + std::vector* delete_fields, + EqualityDeleteFilter* result) const; Status _read_position_delete_file(const TIcebergDeleteFileDesc& delete_file, const TFileScanRangeParams& scan_params, IcebergDeleteFileIOContext* delete_io_ctx, @@ -165,10 +183,16 @@ class IcebergTableReader : public format::TableReader { format::DeleteRows _position_delete_rows_storage; struct EqualityDeleteFilter { std::vector field_ids; + // Delete-file names are retained for Iceberg tables imported from formats that did not + // persist field ids. In BY_NAME mode they are the fallback binding key. + std::vector field_names; std::vector key_types; Block delete_block; }; std::vector _equality_delete_filters; + // Scanner-shared cache supplied in SplitReadOptions. Parsed delete files outlive one data-file + // split and can be reused by every split referencing the same delete file. + ShardedKVCache* _split_cache = nullptr; bool _need_row_lineage_row_id() const; bool _need_iceberg_rowid() const; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 09c5d65e82b7fd..5fb32b84d0ab52 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -33,8 +33,10 @@ #include "common/status.h" #include "core/assert_cast.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/primitive_type.h" #include "exprs/vexpr_context.h" #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" @@ -160,8 +162,27 @@ DataTypePtr find_struct_child_type_by_external_field(const DataTypeStruct& struc return nullptr; } +DataTypePtr restore_current_primitive_type(const schema::external::TField& field, + DataTypePtr fallback_type) { + if (!field.__isset.type) { + return fallback_type; + } + const auto primitive_type = thrift_to_type(field.type.type); + if (is_complex_type(primitive_type)) { + return fallback_type; + } + // The delete file can expose an older physical type, but initial defaults belong to the + // current table field. Restore that type from FE before parsing the default and let the table + // reader apply the normal promotion cast to the delete-key type. + return DataTypeFactory::instance().create_data_type( + primitive_type, false, field.type.__isset.precision ? field.type.precision : 0, + field.type.__isset.scale ? field.type.scale : 0, + field.type.__isset.len ? field.type.len : -1); +} + ColumnDefinition build_schema_column_from_external_field(const schema::external::TField& field, DataTypePtr type) { + type = restore_current_primitive_type(field, std::move(type)); ColumnDefinition column { .identifier = field.__isset.id ? Field::create_field(field.id) : Field {}, .name = field.__isset.name ? field.name : "", @@ -170,6 +191,11 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: .type = std::move(type), .children = {}, .default_expr = nullptr, + .initial_default_value = field.__isset.initial_default_value + ? std::make_optional(field.initial_default_value) + : std::nullopt, + .initial_default_value_is_base64 = field.__isset.initial_default_value_is_base64 && + field.initial_default_value_is_base64, .is_partition_key = false, }; if (column.type == nullptr || !field.__isset.nestedField) { @@ -464,6 +490,34 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info return Status::OK(); } +std::optional TableReader::_find_current_table_column_by_field_id( + int32_t field_id, DataTypePtr type) const { + if (_scan_params == nullptr || !_scan_params->__isset.history_schema_info || + _scan_params->history_schema_info.empty()) { + return std::nullopt; + } + const auto* schema = &_scan_params->history_schema_info.front(); + if (_scan_params->__isset.current_schema_id) { + for (const auto& candidate_schema : _scan_params->history_schema_info) { + if (candidate_schema.__isset.schema_id && + candidate_schema.schema_id == _scan_params->current_schema_id) { + schema = &candidate_schema; + break; + } + } + } + if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { + return std::nullopt; + } + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && field->__isset.id && field->id == field_id) { + return build_schema_column_from_external_field(*field, std::move(type)); + } + } + return std::nullopt; +} + Status TableReader::init(TableReadOptions&& options) { _scan_params = options.scan_params; _format = options.format; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index b8457ee9b4a31e..a875058103c145 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -154,7 +154,7 @@ struct SplitReadOptions { // filter could arrive after synthetic rows have already been returned and those rows cannot be // retracted. Standalone TableReader callers have no scanner runtime-filter lifecycle. bool all_runtime_filters_applied = true; - ShardedKVCache* cache; + ShardedKVCache* cache = nullptr; TFileRangeDesc current_range; FileFormat current_split_format = FileFormat::PARQUET; std::optional global_rowid_context; @@ -318,6 +318,9 @@ class TableReader { } protected: + std::optional _find_current_table_column_by_field_id(int32_t field_id, + DataTypePtr type) const; + // Parse deletion vector information from table format specific file description. virtual Status _parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc, bool* has_delete_file) { diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 89bf367d06b053..6c504e0b49a46b 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -17,15 +17,22 @@ #include "format/table/iceberg_reader.h" +#include +#include #include #include #include #include #include #include +#include +#include +#include +#include #include #include +#include #include #include #include @@ -40,13 +47,16 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" #include "format/column_descriptor.h" #include "format/format_common.h" +#include "format/orc/orc_memory_stream_test.h" #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_column_chunk_reader.h" #include "format/parquet/vparquet_reader.h" @@ -62,6 +72,200 @@ namespace doris { +void write_iceberg_int_orc_file(const std::string& file_path, const std::string& field_name, + int32_t field_id, const std::vector& values) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct<" + field_name + ":int>")); + type->getSubtype(0)->setAttribute("iceberg.id", std::to_string(field_id)); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(values.size()); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& value_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + for (size_t row = 0; row < values.size(); ++row) { + value_batch.data[row] = values[row]; + } + struct_batch.numElements = values.size(); + value_batch.numElements = values.size(); + writer->add(*batch); + writer->close(); + + std::ofstream output(file_path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_iceberg_three_int_orc_file( + const std::string& file_path, const std::string& first_name, + std::optional first_id, const std::vector& first_values, + const std::string& second_name, std::optional second_id, + const std::vector& second_values, const std::string& third_name, + std::optional third_id, const std::vector& third_values) { + DORIS_CHECK(first_values.size() == second_values.size()); + DORIS_CHECK(first_values.size() == third_values.size()); + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct<" + first_name + ":int," + second_name + ":int," + third_name + ":int>")); + const std::array field_ids = {first_id, second_id, third_id}; + for (size_t idx = 0; idx < field_ids.size(); ++idx) { + if (field_ids[idx].has_value()) { + type->getSubtype(idx)->setAttribute("iceberg.id", std::to_string(*field_ids[idx])); + } + } + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(first_values.size()); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + const std::array value_sets = {&first_values, &second_values, &third_values}; + for (size_t field_idx = 0; field_idx < value_sets.size(); ++field_idx) { + auto& value_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[field_idx]); + for (size_t row = 0; row < first_values.size(); ++row) { + value_batch.data[row] = (*value_sets[field_idx])[row]; + } + value_batch.numElements = first_values.size(); + } + struct_batch.numElements = first_values.size(); + writer->add(*batch); + writer->close(); + + std::ofstream output(file_path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +std::shared_ptr build_iceberg_int32_array(const std::vector& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + DORIS_CHECK(builder.Append(value).ok()); + } + std::shared_ptr result; + DORIS_CHECK(builder.Finish(&result).ok()); + return result; +} + +void write_iceberg_three_int_parquet_file( + const std::string& file_path, const std::string& first_name, + std::optional first_id, const std::vector& first_values, + const std::string& second_name, std::optional second_id, + const std::vector& second_values, const std::string& third_name, + std::optional third_id, const std::vector& third_values) { + DORIS_CHECK(first_values.size() == second_values.size()); + DORIS_CHECK(first_values.size() == third_values.size()); + std::vector> fields = { + arrow::field(first_name, arrow::int32(), false), + arrow::field(second_name, arrow::int32(), false), + arrow::field(third_name, arrow::int32(), false)}; + const std::array field_ids = {first_id, second_id, third_id}; + for (size_t idx = 0; idx < fields.size(); ++idx) { + if (field_ids[idx].has_value()) { + fields[idx] = fields[idx]->WithMetadata(arrow::key_value_metadata( + {"PARQUET:field_id"}, {std::to_string(*field_ids[idx])})); + } + } + auto schema = arrow::schema(fields); + auto table = arrow::Table::Make(schema, {build_iceberg_int32_array(first_values), + build_iceberg_int32_array(second_values), + build_iceberg_int32_array(third_values)}); + + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + static_cast(first_values.size()), + properties.build())); +} + +void write_iceberg_int_equality_delete_parquet_file(const std::string& file_path, + const std::string& field_name, int32_t field_id, + int32_t value) { + auto field = arrow::field(field_name, arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, + {std::to_string(field_id)})); + auto table = arrow::Table::Make(arrow::schema({field}), {build_iceberg_int32_array({value})}); + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +Status create_single_int_tuple_descriptor(ObjectPool* object_pool, const std::string& column_name, + int32_t field_id, DescriptorTbl** descriptor_table, + const TupleDescriptor** tuple_descriptor) { + TDescriptorTable thrift_table; + TTableDescriptor table_descriptor; + table_descriptor.__set_id(0); + table_descriptor.__set_tableType(TTableType::OLAP_TABLE); + table_descriptor.__set_numCols(1); + table_descriptor.__set_numClusteringCols(0); + thrift_table.__set_tableDescriptors({table_descriptor}); + + TTypeNode type_node; + type_node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::INT); + type_node.__set_scalar_type(scalar_type); + TTypeDesc type; + type.__set_types({type_node}); + + TSlotDescriptor slot_descriptor; + slot_descriptor.__set_id(0); + slot_descriptor.__set_parent(0); + slot_descriptor.__set_col_unique_id(field_id); + slot_descriptor.__set_slotType(type); + slot_descriptor.__set_columnPos(0); + slot_descriptor.__set_byteOffset(0); + slot_descriptor.__set_nullIndicatorByte(0); + slot_descriptor.__set_nullIndicatorBit(-1); + slot_descriptor.__set_colName(column_name); + slot_descriptor.__set_slotIdx(0); + slot_descriptor.__set_isMaterialized(true); + thrift_table.__set_slotDescriptors({slot_descriptor}); + + TTupleDescriptor thrift_tuple; + thrift_tuple.__set_id(0); + thrift_tuple.__set_byteSize(16); + thrift_tuple.__set_numNullBytes(0); + thrift_tuple.__set_tableId(0); + thrift_table.__set_tupleDescriptors({thrift_tuple}); + + RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); + *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); + return Status::OK(); +} + +schema::external::TFieldPtr make_external_int_field(const std::string& name, int32_t field_id, + std::optional initial_default, + std::vector aliases = {}) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(field_id); + TColumnType type; + type.__set_type(TPrimitiveType::INT); + field->__set_type(type); + if (initial_default.has_value()) { + field->__set_initial_default_value(*initial_default); + } + if (!aliases.empty()) { + field->__set_name_mapping(aliases); + } + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; +} + class IcebergReaderTestHelper : public IcebergTableReader { public: using IcebergTableReader::_is_fully_dictionary_encoded; @@ -845,6 +1049,443 @@ TEST_F(IcebergReaderTest, v1_position_delete_read_error_releases_cache_entry) { EXPECT_NE(status.to_string().find(delete_file.path), std::string::npos); } +TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaults) { + auto make_field = [](const std::string& name, int32_t id, TPrimitiveType::type primitive_type, + const std::string& default_value, int32_t len, int32_t scale, + bool default_is_base64) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_initial_default_value(default_value); + if (default_is_base64) { + field->__set_initial_default_value_is_base64(true); + } + TColumnType type; + type.__set_type(primitive_type); + if (len >= 0) { + type.__set_len(len); + } + if (scale >= 0) { + type.__set_scale(scale); + } + field->__set_type(type); + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; + }; + + schema::external::TStructField root_field; + root_field.__set_fields( + {make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, + "2024-01-01 00:00:00.123456", -1, 6, false), + make_field("added_binary", 2, TPrimitiveType::VARBINARY, "AAEC/w==", 4, -1, true), + make_field("added_string_binary", 3, TPrimitiveType::STRING, "AAEC/w==", -1, -1, + true)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({std::move(current_schema)}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state = RuntimeState(TQueryOptions(), TQueryGlobals()); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + + const auto timestamp_type = make_nullable(std::make_shared(6)); + const auto varbinary_type = make_nullable(std::make_shared(4)); + const auto string_type = make_nullable(std::make_shared()); + Block block; + block.insert({timestamp_type->create_column(), timestamp_type, "added_timestamp"}); + block.insert({varbinary_type->create_column(), varbinary_type, "added_binary"}); + block.insert({string_type->create_column(), string_type, "added_string_binary"}); + std::unordered_map positions = { + {"added_timestamp", 0}, {"added_binary", 1}, {"added_string_binary", 2}}; + reader.TEST_set_column_name_to_block_index(&positions); + ASSERT_TRUE(reader.TEST_register_missing_equality_delete_column(1, "added_timestamp", + timestamp_type) + .ok()); + ASSERT_TRUE( + reader.TEST_register_missing_equality_delete_column(2, "added_binary", varbinary_type) + .ok()); + ASSERT_TRUE(reader.TEST_register_missing_equality_delete_column(3, "added_string_binary", + string_type) + .ok()); + ASSERT_TRUE(reader.TEST_materialize_missing_equality_delete_columns(&block, 3).ok()); + + ASSERT_EQ(block.rows(), 3); + EXPECT_EQ(timestamp_type->to_string(*block.get_by_position(0).column, 0), + "2024-01-01 00:00:00.123456"); + EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, 0), + std::string("\x00\x01\x02\xff", 4)); + EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, 0), + std::string("\x00\x01\x02\xff", 4)); + EXPECT_FALSE(is_column_const(*block.get_by_position(0).column)); + EXPECT_FALSE(is_column_const(*block.get_by_position(1).column)); + EXPECT_FALSE(is_column_const(*block.get_by_position(2).column)); +} + +TEST_F(IcebergReaderTest, v1_multi_equality_delete_hashes_materialized_missing_default) { + schema::external::TStructField root_field; + root_field.__set_fields({make_external_int_field("id", 0, std::nullopt), + make_external_int_field("added_column", 1, "7")}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + + const auto int_type = make_nullable(std::make_shared()); + auto id_column = int_type->create_column(); + id_column->insert(Field::create_field(1)); + id_column->insert(Field::create_field(2)); + id_column->insert(Field::create_field(3)); + Block data_block; + data_block.insert({std::move(id_column), int_type, "id"}); + data_block.insert({int_type->create_column(), int_type, "added_column"}); + std::unordered_map positions = {{"id", 0}, {"added_column", 1}}; + reader.TEST_set_column_name_to_block_index(&positions); + ASSERT_TRUE( + reader.TEST_register_missing_equality_delete_column(1, "added_column", int_type).ok()); + ASSERT_TRUE(reader.TEST_materialize_missing_equality_delete_columns(&data_block, 3).ok()); + ASSERT_EQ(data_block.rows(), 3); + ASSERT_FALSE(is_column_const(*data_block.get_by_position(1).column)); + + auto delete_id_column = int_type->create_column(); + delete_id_column->insert(Field::create_field(2)); + auto delete_default_column = int_type->create_column(); + delete_default_column->insert(Field::create_field(7)); + Block delete_block; + delete_block.insert({std::move(delete_id_column), int_type, "id"}); + delete_block.insert({std::move(delete_default_column), int_type, "added_column"}); + MultiEqualityDelete equality_delete(&delete_block, {0, 1}); + ASSERT_TRUE(equality_delete.init(&profile).ok()); + + std::unordered_map id_to_name = {{0, "id"}, {1, "added_column"}}; + IColumn::Filter filter(3, 1); + ASSERT_TRUE( + equality_delete.filter_data_block(&data_block, &positions, id_to_name, filter).ok()); + EXPECT_EQ(filter, IColumn::Filter({1, 0, 1})); +} + +TEST_F(IcebergReaderTest, v1_missing_equality_key_returns_error_for_pruned_descriptor) { + schema::external::TStructField root_field; + root_field.__set_fields({make_external_int_field("id", 0, std::nullopt)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + + const auto int_type = make_nullable(std::make_shared()); + const auto status = + reader.TEST_register_missing_equality_delete_column(1, "hidden_key", int_type); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("field id 1"), std::string::npos); +} + +TEST_F(IcebergReaderTest, v1_orc_equality_delete_matches_missing_initial_default) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_orc_missing_equality_delete_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.orc").string(); + const auto delete_file = (test_dir / "equality-delete.orc").string(); + write_iceberg_int_orc_file(data_file, "id", 0, {1, 2, 3}); + write_iceberg_int_orc_file(delete_file, "added_column", 1, {7}); + + schema::external::TStructField root_field; + root_field.__set_fields({make_external_int_field("id", 0, std::nullopt), + make_external_int_field("added_column", 1, "7")}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + + TIcebergDeleteFileDesc delete_descriptor; + delete_descriptor.__set_content(IcebergReaderMixin::EQUALITY_DELETE); + delete_descriptor.__set_path(delete_file); + delete_descriptor.__set_field_ids({1}); + delete_descriptor.__set_file_format(TFileFormatType::FORMAT_ORC); + TIcebergFileDesc iceberg_descriptor; + iceberg_descriptor.__set_format_version(2); + iceberg_descriptor.__set_original_file_path(data_file); + iceberg_descriptor.__set_delete_files({delete_descriptor}); + TTableFormatFileDesc table_format_descriptor; + table_format_descriptor.__set_iceberg_params(iceberg_descriptor); + + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_table_format_params(table_format_descriptor); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_int_tuple_descriptor(&object_pool, "id", 0, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergOrcReader reader(&kv_cache, &profile, &runtime_state, scan_params, scan_range, 1024, + "UTC", &io_ctx, cache.get()); + + std::vector column_descriptors(1); + column_descriptors[0].name = "id"; + std::unordered_map block_positions = {{"id", 0}}; + OrcInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto id_type = make_nullable(std::make_shared()); + Block block; + block.insert({id_type->create_column(), id_type, "id"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(read_rows, 0); + EXPECT_EQ(block.rows(), 0); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_parquet_partial_id_equality_delete_ignores_stale_field_id) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_parquet_partial_id_equality_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.parquet").string(); + const auto delete_file = (test_dir / "equality-delete.parquet").string(); + // The id-less columns put the complete Parquet file in BY_NAME mode. The unrelated + // stale_added column deliberately retains field id 1, which must not override the historical + // alias selected for the hidden added_column equality key. + write_iceberg_three_int_parquet_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", + std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); + write_iceberg_int_equality_delete_parquet_file(delete_file, "added_column", 1, 7); + + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 0, std::nullopt), + make_external_int_field("added_column", 1, std::nullopt, {"legacy_added"})}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + + TIcebergDeleteFileDesc delete_descriptor; + delete_descriptor.__set_content(IcebergReaderMixin::EQUALITY_DELETE); + delete_descriptor.__set_path(delete_file); + delete_descriptor.__set_field_ids({1}); + delete_descriptor.__set_file_format(TFileFormatType::FORMAT_PARQUET); + TIcebergFileDesc iceberg_descriptor; + iceberg_descriptor.__set_format_version(2); + iceberg_descriptor.__set_original_file_path(data_file); + iceberg_descriptor.__set_delete_files({delete_descriptor}); + TTableFormatFileDesc table_format_descriptor; + table_format_descriptor.__set_iceberg_params(iceberg_descriptor); + + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_table_format_params(table_format_descriptor); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_int_tuple_descriptor(&object_pool, "id", 0, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone("UTC", ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + io::FileReaderSPtr file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(data_file, &file_reader).ok()); + reader.set_file_reader(file_reader); + + std::vector column_descriptors(1); + column_descriptors[0].name = "id"; + std::unordered_map block_positions = {{"id", 0}}; + ParquetInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto id_type = make_nullable(std::make_shared()); + Block block; + block.insert({id_type->create_column(), id_type, "id"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 2); + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 0), "1"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "3"); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_orc_partial_id_equality_delete_ignores_stale_field_id) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_v1_orc_partial_id_equality_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.orc").string(); + const auto delete_file = (test_dir / "equality-delete.orc").string(); + // One id-less column switches the whole file to BY_NAME. The hidden current key is physically + // stored as its id-less historical alias, while an unrelated migrated column still carries + // the key's stale field id. Equality-delete binding must select legacy_added, not stale_added. + write_iceberg_three_int_orc_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", + std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); + write_iceberg_int_orc_file(delete_file, "added_column", 1, {7}); + + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 0, std::nullopt), + make_external_int_field("added_column", 1, std::nullopt, {"legacy_added"})}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + + TIcebergDeleteFileDesc delete_descriptor; + delete_descriptor.__set_content(IcebergReaderMixin::EQUALITY_DELETE); + delete_descriptor.__set_path(delete_file); + delete_descriptor.__set_field_ids({1}); + delete_descriptor.__set_file_format(TFileFormatType::FORMAT_ORC); + TIcebergFileDesc iceberg_descriptor; + iceberg_descriptor.__set_format_version(2); + iceberg_descriptor.__set_original_file_path(data_file); + iceberg_descriptor.__set_delete_files({delete_descriptor}); + TTableFormatFileDesc table_format_descriptor; + table_format_descriptor.__set_iceberg_params(iceberg_descriptor); + + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_table_format_params(table_format_descriptor); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_int_tuple_descriptor(&object_pool, "id", 0, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergOrcReader reader(&kv_cache, &profile, &runtime_state, scan_params, scan_range, 1024, + "UTC", &io_ctx, cache.get()); + + std::vector column_descriptors(1); + column_descriptors[0].name = "id"; + std::unordered_map block_positions = {{"id", 0}}; + OrcInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto id_type = make_nullable(std::make_shared()); + Block block; + block.insert({id_type->create_column(), id_type, "id"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 2); + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 0), "1"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "3"); + + std::filesystem::remove_all(test_dir); +} + // Test reading real Iceberg Orc file using IcebergTableReader TEST_F(IcebergReaderTest, read_iceberg_orc_file) { // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email diff --git a/be/test/format_v2/expr/equality_delete_predicate_test.cpp b/be/test/format_v2/expr/equality_delete_predicate_test.cpp index 886a86713fe8da..0f63e915f28a53 100644 --- a/be/test/format_v2/expr/equality_delete_predicate_test.cpp +++ b/be/test/format_v2/expr/equality_delete_predicate_test.cpp @@ -112,6 +112,27 @@ TEST_F(EqualityDeletePredicateTest, MatchSingleColumn) { EXPECT_EQ(result_column_data(data_block, result_column_id), std::vector({1, 0, 0, 1})); } +TEST_F(EqualityDeletePredicateTest, UsesPopulatedPredicateColumnForLazyBatchSize) { + Block delete_block; + delete_block.insert(make_nullable_int_column("id", {1, 4})); + Block data_block; + // A lazy reader can leave the first projected column unread while decoding a later predicate + // column. Block::rows() is therefore zero even though the equality key has four rows. + data_block.insert(make_nullable_string_column("unread", {})); + data_block.insert(make_nullable_int_column("id", {1, 2, 3, 4})); + + auto predicate = std::make_shared(std::move(delete_block), + std::vector {1}); + predicate->_open_finished = true; + predicate->add_child(std::make_shared(1, data_block.get_by_position(1).type)); + VExprContext context(predicate); + + int result_column_id = -1; + auto status = predicate->execute(&context, &data_block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(result_column_data(data_block, result_column_id), std::vector({1, 0, 0, 1})); +} + TEST_F(EqualityDeletePredicateTest, MatchMultipleColumns) { Block delete_block; delete_block.insert(make_nullable_int_column("id", {1, 2})); diff --git a/be/test/format_v2/json/json_reader_test.cpp b/be/test/format_v2/json/json_reader_test.cpp index 31c77501ce67c6..683a47dee299cc 100644 --- a/be/test/format_v2/json/json_reader_test.cpp +++ b/be/test/format_v2/json/json_reader_test.cpp @@ -590,9 +590,9 @@ TEST(JsonReaderTest, AppliesDeleteAndNormalConjunctsWithPredicateFilterAccountin request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); request->delete_conjuncts = { - prepared_conjunct(&state, std::make_shared(0, 1))}; - request->conjuncts = { prepared_conjunct(&state, std::make_shared(0, 2))}; + request->conjuncts = { + prepared_conjunct(&state, std::make_shared(0, 1))}; ASSERT_TRUE(reader.open(request).ok()); auto block = make_block(schema, {0, 1}); @@ -600,8 +600,8 @@ TEST(JsonReaderTest, AppliesDeleteAndNormalConjunctsWithPredicateFilterAccountin bool eof = false; ASSERT_TRUE(reader.get_block(&block, &rows, &eof).ok()); ASSERT_EQ(rows, 1); - EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 3); - EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "carol"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 2); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "bob"); EXPECT_EQ(io_ctx->predicate_filtered_rows, 1); } diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 8a10fc06a41db3..597816cab51126 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -46,11 +46,14 @@ #include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_varbinary.h" #include "exec/common/endian.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" @@ -61,6 +64,7 @@ #include "format/table/deletion_vector_reader.h" #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" +#include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" #include "io/io_common.h" #include "roaring/roaring64map.hh" @@ -281,12 +285,61 @@ std::shared_ptr build_string_array(const std::vector& return finish_array(&builder); } +schema::external::TFieldPtr external_schema_field( + std::string name, int32_t id, std::vector aliases = {}, + std::optional initial_default = std::nullopt, + std::optional type = std::nullopt, bool initial_default_is_base64 = false) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + if (!aliases.empty()) { + field->__set_name_mapping(aliases); + } + if (initial_default.has_value()) { + field->__set_initial_default_value(*initial_default); + if (initial_default_is_base64) { + field->__set_initial_default_value_is_base64(true); + } + } + if (type.has_value()) { + field->__set_type(*type); + } + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; +} + +TColumnType external_primitive_type(TPrimitiveType::type type, int32_t len = -1, + int32_t scale = -1) { + TColumnType result; + result.__set_type(type); + if (len >= 0) { + result.__set_len(len); + } + if (scale >= 0) { + result.__set_scale(scale); + } + return result; +} + +schema::external::TSchema external_schema(int64_t schema_id, + std::vector fields) { + schema::external::TStructField root_field; + root_field.__set_fields(fields); + schema::external::TSchema schema; + schema.__set_schema_id(schema_id); + schema.__set_root_field(root_field); + return schema; +} + void write_iceberg_equality_delete_parquet_file(const std::string& file_path, int32_t field_id, - int32_t value) { + int32_t value, + const std::string& field_name = "id") { const auto metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), false)->WithMetadata(metadata), + arrow::field(field_name, arrow::int32(), false)->WithMetadata(metadata), }); auto table = arrow::Table::Make(schema, {build_int32_array({value})}); @@ -302,6 +355,168 @@ void write_iceberg_equality_delete_parquet_file(const std::string& file_path, in builder.build())); } +void write_iceberg_timestamp_equality_delete_parquet_file(const std::string& file_path, + int32_t field_id, int64_t value, + const std::string& field_name, + bool adjusted_to_utc = false) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + const auto timestamp_type = adjusted_to_utc ? arrow::timestamp(arrow::TimeUnit::MICRO, "UTC") + : arrow::timestamp(arrow::TimeUnit::MICRO); + arrow::TimestampBuilder value_builder(timestamp_type, arrow::default_memory_pool()); + ASSERT_TRUE(value_builder.Append(value).ok()); + auto value_result = value_builder.Finish(); + ASSERT_TRUE(value_result.ok()) << value_result.status(); + auto schema = arrow::schema({ + arrow::field(field_name, timestamp_type, false)->WithMetadata(metadata), + }); + auto table = arrow::Table::Make(schema, {*value_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_iceberg_binary_equality_delete_parquet_file(const std::string& file_path, + int32_t field_id, const std::string& value, + const std::string& field_name) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + arrow::BinaryBuilder value_builder; + ASSERT_TRUE(value_builder.Append(value).ok()); + auto value_result = value_builder.Finish(); + ASSERT_TRUE(value_result.ok()) << value_result.status(); + auto schema = arrow::schema({ + arrow::field(field_name, arrow::binary(), false)->WithMetadata(metadata), + }); + auto table = arrow::Table::Make(schema, {*value_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_iceberg_null_equality_delete_parquet_file(const std::string& file_path, int32_t field_id, + const std::string& field_name) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + auto schema = arrow::schema({ + arrow::field(field_name, arrow::int32(), true)->WithMetadata(metadata), + }); + arrow::Int32Builder value_builder; + ASSERT_TRUE(value_builder.AppendNull().ok()); + auto value_result = value_builder.Finish(); + ASSERT_TRUE(value_result.ok()) << value_result.status(); + auto table = arrow::Table::Make(schema, {*value_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_single_int_parquet_file(const std::string& file_path, const std::string& field_name, + const std::vector& values, + std::optional field_id) { + auto field = arrow::field(field_name, arrow::int32(), false); + if (field_id.has_value()) { + field = field->WithMetadata( + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(*field_id)})); + } + auto table = arrow::Table::Make(arrow::schema({field}), {build_int32_array(values)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(values.size()), + builder.build())); +} + +void write_two_int_parquet_file(const std::string& file_path, const std::string& first_name, + const std::vector& first_values, + std::optional first_field_id, + const std::string& second_name, + const std::vector& second_values, + std::optional second_field_id) { + ASSERT_EQ(first_values.size(), second_values.size()); + auto first_field = arrow::field(first_name, arrow::int32(), false); + if (first_field_id.has_value()) { + first_field = first_field->WithMetadata( + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(*first_field_id)})); + } + auto second_field = arrow::field(second_name, arrow::int32(), false); + if (second_field_id.has_value()) { + second_field = second_field->WithMetadata(arrow::key_value_metadata( + {"PARQUET:field_id"}, {std::to_string(*second_field_id)})); + } + auto schema = arrow::schema({first_field, second_field}); + auto table = arrow::Table::Make( + schema, {build_int32_array(first_values), build_int32_array(second_values)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(first_values.size()), + builder.build())); +} + +void write_timestamp_int_parquet_file(const std::string& file_path, + const std::vector& timestamps, + const std::vector& ids) { + ASSERT_EQ(timestamps.size(), ids.size()); + const auto timestamp_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"0"}); + const auto id_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"1"}); + const auto timestamp_type = arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); + arrow::TimestampBuilder timestamp_builder(timestamp_type, arrow::default_memory_pool()); + ASSERT_TRUE(timestamp_builder.AppendValues(timestamps).ok()); + auto timestamp_result = timestamp_builder.Finish(); + ASSERT_TRUE(timestamp_result.ok()) << timestamp_result.status(); + auto schema = arrow::schema( + {arrow::field("event_time", timestamp_type, false)->WithMetadata(timestamp_metadata), + arrow::field("id", arrow::int32(), false)->WithMetadata(id_metadata)}); + auto table = arrow::Table::Make(schema, {*timestamp_result, build_int32_array(ids)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(timestamps.size()), + builder.build())); +} + void write_iceberg_equality_delete_bigint_parquet_file(const std::string& file_path, int32_t field_id, int64_t value) { const auto metadata = @@ -597,6 +812,7 @@ void expect_int32_column_values(const IColumn& column, SplitReadOptions build_split_options(const std::string& file_path) { SplitReadOptions options; + EXPECT_EQ(options.cache, nullptr); options.current_range.__set_path(file_path); options.current_range.__set_file_size( static_cast(std::filesystem::file_size(file_path))); @@ -709,6 +925,23 @@ std::vector read_iceberg_ids(doris::format::iceberg::IcebergTableReader return ids; } +void init_iceberg_reader(doris::format::iceberg::IcebergTableReader* reader, + const std::vector& projected_columns, + TFileScanRangeParams* scan_params, + const std::shared_ptr& io_ctx, RuntimeState* state, + RuntimeProfile* profile) { + ASSERT_TRUE(reader->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = scan_params, + .io_ctx = io_ctx, + .runtime_state = state, + .scanner_profile = profile, + }) + .ok()); +} + DataTypePtr make_table_test_type(const DataTypePtr& type, bool nullable_root = true) { DORIS_CHECK(type != nullptr); const auto nested_type = remove_nullable(type); @@ -1870,17 +2103,16 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteCastsDataColumnToDeleteKeyType) { std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergPositionDeleteOnlyMatchesOriginalDataFilePath) { +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesNullForMissingDataColumn) { const auto test_dir = std::filesystem::temp_directory_path() / - "doris_iceberg_position_delete_path_match_test"; + "doris_iceberg_equality_delete_missing_column_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto file_path = (test_dir / "split.parquet").string(); - const auto other_file_path = (test_dir / "other.parquet").string(); - const auto delete_file_path = (test_dir / "position-delete.parquet").string(); - write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); - write_position_delete_parquet_file(delete_file_path, {other_file_path, file_path}, {0, 1}); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + write_iceberg_null_equality_delete_parquet_file(delete_file_path, 1, "added_column"); std::vector projected_columns; projected_columns.push_back(make_table_column(0, "id", std::make_shared())); @@ -1893,126 +2125,772 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteOnlyMatchesOriginalDataFilePath) auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); ShardedKVCache cache(1); doris::format::iceberg::IcebergTableReader reader; - ASSERT_TRUE(reader.init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = &scan_params, - .io_ctx = io_ctx, - .runtime_state = &state, - .scanner_profile = &profile, - }) - .ok()); + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); auto split_options = build_split_options(file_path); split_options.cache = &cache; split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( - file_path, {make_iceberg_position_delete_file(delete_file_path)})); + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); - EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + // Iceberg treats a field missing from an older data file as NULL. The NULL equality-delete + // key therefore matches every row in this file. + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergRowLineageRemainsFileLocalAfterDeleteFiltering) { - const auto test_dir = - std::filesystem::temp_directory_path() / "doris_iceberg_row_lineage_delete_test"; +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesInitialDefaultForMissingDataColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_default_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto file_path = (test_dir / "split.parquet").string(); - const auto delete_file_path = (test_dir / "position-delete.parquet").string(); - write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); - write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 1, 7, "added_column"); std::vector projected_columns; - projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); projected_columns.push_back(make_table_column(0, "id", std::make_shared())); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info( + {external_schema(100, {external_schema_field("id", 0), + external_schema_field("added_column", 1, {}, "7")})}); io::FileReaderStats file_reader_stats; io::FileCacheStatistics file_cache_stats; auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); ShardedKVCache cache(1); doris::format::iceberg::IcebergTableReader reader; - ASSERT_TRUE(reader.init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = &scan_params, - .io_ctx = io_ctx, - .runtime_state = &state, - .scanner_profile = &profile, - }) - .ok()); + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); auto split_options = build_split_options(file_path); split_options.cache = &cache; - TTableFormatFileDesc table_format_params = make_iceberg_table_format_desc( - file_path, {make_iceberg_position_delete_file(delete_file_path)}); - table_format_params.iceberg_params.__set_first_row_id(1000); - split_options.current_range.__set_table_format_params(table_format_params); + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); - Block block = build_table_block(projected_columns); - bool eos = false; - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 2); - expect_nullable_int64_column_values(*block.get_by_position(0).column, {1000, 1002}); - const auto& id_column = assert_cast(expect_not_null_table_column(block, 1)); - EXPECT_EQ(id_column.get_element(0), 1); - EXPECT_EQ(id_column.get_element(1), 3); + // The old file has no physical added_column, but Iceberg defines every row as the initial + // default 7. The equality delete key 7 therefore removes the entire file. + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesPositionDeleteFile) { - const auto test_dir = - std::filesystem::temp_directory_path() / "doris_iceberg_position_delete_file_test"; +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesTimestampInitialDefaultForMissingColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_timestamp_default_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto file_path = (test_dir / "split.parquet").string(); - const auto delete_file_path = (test_dir / "position-delete.parquet").string(); - write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, - {"one", "two", "three", "four", "five"}); - write_position_delete_parquet_file(delete_file_path, {file_path, file_path}, {1, 3}); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + write_iceberg_timestamp_equality_delete_parquet_file(delete_file_path, 1, + 1'704'067'200'123'456L, "added_timestamp"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, + {external_schema_field("id", 0), + external_schema_field("added_timestamp", 1, {}, "2024-01-01 00:00:00.123456", + external_primitive_type(TPrimitiveType::DATETIMEV2, -1, 6))})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultForMissingColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_varbinary_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + const std::string binary_default("\x00\x01\x02\xff", 4); + write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, + "added_binary"); std::vector projected_columns; projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, + {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "AAEC/w==", + external_primitive_type(TPrimitiveType::VARBINARY, 4), true)})}); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesStringMappedBinaryInitialDefault) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_binary_string_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + const std::string binary_default("\x00\x01\x02\xff", 4); + write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, + "added_binary"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "AAEC/w==", + external_primitive_type(TPrimitiveType::STRING), true)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; io::FileReaderStats file_reader_stats; io::FileCacheStatistics file_cache_stats; auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); ShardedKVCache cache(1); doris::format::iceberg::IcebergTableReader reader; - ASSERT_TRUE(reader.init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = &scan_params, - .io_ctx = io_ctx, - .runtime_state = &state, - .scanner_profile = &profile, - }) - .ok()); + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); auto split_options = build_split_options(file_path); split_options.cache = &cache; split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( - file_path, {make_iceberg_position_delete_file(delete_file_path)})); + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); - EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3, 5})); + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteDecodesBinaryDefaultMappedToString) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_string_binary_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + const std::string binary_default("\x00\x01\x02\xff", 4); + write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, + "added_binary"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "AAEC/w==", + external_primitive_type(TPrimitiveType::STRING), true)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeletePreservesTimestampTzAcrossDstFold) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_timestamp_tz_dst_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + // These instants are the two occurrences of 01:30 during the 2024 New York DST fall-back. + constexpr int64_t FIRST_0130_MICROS = 1'730'611'800'000'000L; + constexpr int64_t SECOND_0130_MICROS = 1'730'615'400'000'000L; + write_timestamp_int_parquet_file(file_path, {FIRST_0130_MICROS, SECOND_0130_MICROS}, {1, 2}); + write_iceberg_timestamp_equality_delete_parquet_file(delete_file_path, 0, FIRST_0130_MICROS, + "event_time", true); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(1, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_enable_mapping_timestamp_tz(true); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, + {external_schema_field("event_time", 0, {}, std::nullopt, + external_primitive_type(TPrimitiveType::TIMESTAMPTZ, -1, 6)), + external_schema_field("id", 1)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("America/New_York"); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesNameMappingWithoutFileFieldIds) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_name_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); + + std::vector projected_columns; + auto id_column = make_table_column(0, "current_id", std::make_shared()); + id_column.name_mapping.push_back("legacy_id"); + projected_columns.push_back(std::move(id_column)); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_stale_field_id_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + // The real key has no id and is mapped by its historical name. A different physical column + // carries the stale id 0, forcing the entire split into BY_NAME mode. + write_two_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt, "stale_key", + {100, 200, 300}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); + + std::vector projected_columns; + auto id_column = make_table_column(0, "current_id", std::make_shared()); + id_column.name_mapping.push_back("legacy_id"); + projected_columns.push_back(std::move(id_column)); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesUnprojectedTableSchemaNameMapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_hidden_name_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_two_int_parquet_file(file_path, "LEGACY_ID", {1, 2, 3}, std::nullopt, "data", + {10, 20, 30}, std::nullopt); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); + + // Model SELECT data: the equality key is deliberately absent from the query projection. + std::vector projected_columns; + projected_columns.push_back(make_table_column(1, "data", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info( + {external_schema(100, {external_schema_field("current_id", 0, {"legacy_id"}), + external_schema_field("data", 1)})}); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({10, 30})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteFileIsReusedAcrossSplits) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_split_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + // Removing the source after the first split proves that the second split consumes the parsed + // delete block from SplitReadOptions.cache instead of reopening the delete file. + ASSERT_TRUE(std::filesystem::remove(delete_file_path)); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteCacheIsScopedByFileSystem) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_filesystem_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_fs_name("filesystem-a"); + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + // The same descriptor path resolves to different content in another filesystem namespace. + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 3); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_fs_name("filesystem-b"); + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 2})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteOnlyMatchesOriginalDataFilePath) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_position_delete_path_match_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto other_file_path = (test_dir / "other.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {other_file_path, file_path}, {0, 1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergRowLineageRemainsFileLocalAfterDeleteFiltering) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_row_lineage_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + TTableFormatFileDesc table_format_params = make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)}); + table_format_params.iceberg_params.__set_first_row_id(1000); + split_options.current_range.__set_table_format_params(table_format_params); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + expect_nullable_int64_column_values(*block.get_by_position(0).column, {1000, 1002}); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 1)); + EXPECT_EQ(id_column.get_element(0), 1); + EXPECT_EQ(id_column.get_element(1), 3); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesPositionDeleteFile) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_position_delete_file_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + write_position_delete_parquet_file(delete_file_path, {file_path, file_path}, {1, 3}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3, 5})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteHonorsPositionBounds) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_position_delete_bounds_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + write_position_delete_parquet_file(delete_file_path, {file_path, file_path, file_path}, + {0, 2, 4}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto delete_file = make_iceberg_position_delete_file(delete_file_path); + delete_file.__set_position_lower_bound(1); + delete_file.__set_position_upper_bound(3); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {delete_file})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 2, 4, 5})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteFileIsReusedAcrossSplits) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_position_delete_split_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_position_delete_parquet_file(delete_file_path, {first_file_path, second_file_path}, + {1, 0}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + // The cached delete file contains entries for every referenced data file, so another split can + // select its own positions without rescanning the shared delete file. + ASSERT_TRUE(std::filesystem::remove(delete_file_path)); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteCacheIsScopedByFileSystem) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_position_delete_filesystem_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_position_delete_parquet_file(delete_file_path, {first_file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_fs_name("filesystem-a"); + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + write_position_delete_parquet_file(delete_file_path, {second_file_path}, {0}); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_fs_name("filesystem-b"); + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2, 3})); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 0d484f60d9e5de..970e4b2153557e 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -4085,6 +4085,63 @@ TEST(TableReaderTest, ReusedBlockClearsProjectedStructWithNullableChild) { std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, ProjectedRenamedStructPreservesParentDeclaredChildNullability) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_struct_parent_child_nullability_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_struct_with_nullable_child_parquet_file(file_path); + + const auto string_type = std::make_shared(); + const auto nullable_string_type = make_nullable(string_type); + auto renamed_note = make_table_column(1, "renamed_note", nullable_string_type); + renamed_note.name_mapping = {"note"}; + // Iceberg nested schema metadata can omit nullability on this child while the parent DataType + // remains authoritative and declares it nullable. + renamed_note.type = string_type; + auto struct_type = std::make_shared(DataTypes {nullable_string_type}, + Strings {"renamed_note"}); + auto struct_column = make_table_column(100, "s", struct_type); + struct_column.children = {renamed_note}; + std::vector projected_columns = {struct_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(eos); + + const auto& struct_result = + assert_cast(expect_not_null_table_column(block, 0)); + ASSERT_EQ(struct_result.get_columns().size(), 1); + const auto& notes = assert_cast(struct_result.get_column(0)); + EXPECT_FALSE(notes.is_null_at(0)); + EXPECT_EQ(notes.get_data_at(0).to_string(), "seven"); + EXPECT_TRUE(notes.is_null_at(1)); + ASSERT_TRUE(block.get_by_position(0).check_type_and_column_match().ok()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, ProjectedPartitionColumnUsesSplitPartitionValue) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_partition_value_test"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 784315fc15c7bd..6a986da45303d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -34,6 +34,7 @@ import org.apache.doris.thrift.schema.external.TStructField; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,6 +46,9 @@ private static TField getExternalSchema(Column column) { root.setId(column.getUniqueId()); root.setIsOptional(column.isAllowNull()); root.setType(column.getType().toColumnTypeThrift()); + if (column.getDefaultValue() != null) { + root.setInitialDefaultValue(column.getDefaultValue()); + } TNestedField nestedField = new TNestedField(); if (column.getType().isStructType()) { @@ -120,7 +124,8 @@ private static TStructField getExternalSchemaForPrunedColumn(List columns, Map> nameMapping) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, Collections.emptyMap()); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, + Map base64InitialDefaults) { params.setCurrentSchemaId(schemaId); TSchema tSchema = new TSchema(); tSchema.setSchemaId(schemaId); - tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping)); + tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping, base64InitialDefaults)); params.addToHistorySchemaInfo(tSchema); } private static TStructField getExternalSchemaForAllColumn(List columns, - Map> nameMapping) { + Map> nameMapping, Map base64InitialDefaults) { TStructField structField = new TStructField(); for (Column child : columns) { TFieldPtr fieldPtr = new TFieldPtr(); - fieldPtr.setFieldPtr(getExternalSchema(child.getType(), child, nameMapping)); + fieldPtr.setFieldPtr(getExternalSchema( + child.getType(), child, nameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } return structField; @@ -149,11 +161,22 @@ private static TStructField getExternalSchemaForAllColumn(List columns, private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping) { + return getExternalSchema(columnType, dorisColumn, nameMapping, Collections.emptyMap()); + } + + private static TField getExternalSchema(Type columnType, Column dorisColumn, + Map> nameMapping, Map base64InitialDefaults) { TField root = new TField(); root.setName(dorisColumn.getName()); root.setId(dorisColumn.getUniqueId()); root.setIsOptional(dorisColumn.isAllowNull()); root.setType(dorisColumn.getType().toColumnTypeThrift()); + if (base64InitialDefaults.containsKey(dorisColumn.getUniqueId())) { + root.setInitialDefaultValue(base64InitialDefaults.get(dorisColumn.getUniqueId())); + root.setInitialDefaultValueIsBase64(true); + } else if (dorisColumn.getDefaultValue() != null) { + root.setInitialDefaultValue(dorisColumn.getDefaultValue()); + } if (nameMapping != null && nameMapping.containsKey(dorisColumn.getUniqueId())) { // for iceberg set name mapping. @@ -175,7 +198,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, for (StructField subField : dorisStructType.getFields()) { TFieldPtr fieldPtr = new TFieldPtr(); Column subColumn = subNameToSubColumn.get(subField.getName()); - fieldPtr.setFieldPtr(getExternalSchema(subField.getType(), subColumn, nameMapping)); + fieldPtr.setFieldPtr(getExternalSchema( + subField.getType(), subColumn, nameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } @@ -187,7 +211,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TArrayField listField = new TArrayField(); TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( - dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping)); + dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping, + base64InitialDefaults)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); root.setNestedField(nestedField); @@ -197,12 +222,14 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TMapField mapField = new TMapField(); TFieldPtr keyPtr = new TFieldPtr(); keyPtr.setFieldPtr(getExternalSchema( - dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping)); + dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping, + base64InitialDefaults)); mapField.setKeyField(keyPtr); TFieldPtr valuePtr = new TFieldPtr(); valuePtr.setFieldPtr(getExternalSchema( - dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping)); + dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping, + base64InitialDefaults)); mapField.setValueField(valuePtr); nestedField.setMapField(mapField); root.setNestedField(nestedField); @@ -210,4 +237,3 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, return root; } } - diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 2cd4f2bcf2f233..12b5dce11395fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -108,6 +108,7 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -135,6 +136,7 @@ import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; +import java.util.Base64; import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashSet; @@ -1182,10 +1184,14 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi List columns = schema.columns(); List resSchema = Lists.newArrayListWithCapacity(columns.size()); for (Types.NestedField field : columns) { + String initialDefault = null; + if (field.initialDefault() != null) { + initialDefault = serializeInitialDefault(field.type(), field.initialDefault(), + enableMappingTimestampTz); + } Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), - true, null, - true, field.doc(), true, -1); + true, null, true, initialDefault, field.doc(), true, -1); updateIcebergColumnUniqueId(column, field); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); @@ -1198,6 +1204,62 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi return resSchema; } + private static String serializeInitialDefault(org.apache.iceberg.types.Type type, Object value, + boolean enableMappingTimestampTz) { + String humanValue = Transforms.identity(type).toHumanString(type, value); + if (type.typeId() == TypeID.TIMESTAMP) { + // Iceberg formats timestamps as ISO-8601 (for example 2024-01-01T00:00:00), while + // Doris' DATETIMEV2 default parser requires a space between the date and time. + String dorisValue = humanValue.replace('T', ' '); + Types.TimestampType timestampType = (Types.TimestampType) type; + if (timestampType.shouldAdjustToUTC() && !enableMappingTimestampTz) { + // Iceberg timestamptz human values carry a trailing offset. DATETIMEV2 has no + // offset carrier, so retain the displayed UTC wall time and remove the suffix. + return dorisValue.replaceFirst("(Z|[+-]\\d{2}:\\d{2})$", ""); + } + return dorisValue; + } + if (isBinaryLike(type)) { + // Always use the lossless Base64 carrier. Binary-like Iceberg fields may map to either + // VARBINARY or STRING/CHAR, and the scan schema marker tells BE to decode both forms + // back to the raw bytes stored in equality-delete files. + return serializeBinaryInitialDefault(type, value); + } + return humanValue; + } + + /** + * Return binary-like initial defaults in a lossless transport representation. These defaults + * cannot be carried as raw Java strings and their Doris type is insufficient to identify them + * when varbinary mapping is disabled, because UUID/BINARY/FIXED then map to STRING/CHAR. + */ + public static Map getBase64EncodedInitialDefaults(Schema schema) { + Map result = Maps.newHashMap(); + for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) { + if (field.initialDefault() == null || !isBinaryLike(field.type())) { + continue; + } + result.put(field.fieldId(), serializeBinaryInitialDefault(field.type(), field.initialDefault())); + } + return result; + } + + private static boolean isBinaryLike(org.apache.iceberg.types.Type type) { + return type.typeId() == TypeID.UUID || type.typeId() == TypeID.BINARY + || type.typeId() == TypeID.FIXED; + } + + private static String serializeBinaryInitialDefault(org.apache.iceberg.types.Type type, Object value) { + if (type.typeId() != TypeID.UUID) { + return Transforms.identity(type).toHumanString(type, value); + } + UUID uuid = (UUID) value; + ByteBuffer bytes = ByteBuffer.allocate(16); + bytes.putLong(uuid.getMostSignificantBits()); + bytes.putLong(uuid.getLeastSignificantBits()); + return Base64.getEncoder().encodeToString(bytes.array()); + } + /** * Estimate iceberg table row count. * Get the row count by adding all task file recordCount. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 2769a1ddb8e43e..a69a1c55d354d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -81,6 +81,7 @@ import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; +import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; @@ -454,20 +455,26 @@ public void createScanRangeLocations() throws UserException { // Extract name mapping from Iceberg table properties Map> nameMapping = extractNameMapping(); - boolean haveTopnLazyMatCol = false; - for (SlotDescriptor slot : desc.getSlots()) { - String colName = slot.getColumn().getName(); - if (colName.startsWith(Column.GLOBAL_ROWID_COL)) { - haveTopnLazyMatCol = true; - break; - } - } - if (haveTopnLazyMatCol) { - ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), nameMapping); - } else { - // Use new initSchemaInfo method that only includes needed columns based on slots and pruned type - ExternalUtil.initSchemaInfoForPrunedColumn(params, -1L, desc.getSlots(), nameMapping); - } + // Equality-delete keys are hidden scan dependencies and need not appear in the query + // projection. Both scanners need the complete current schema to resolve field ids, + // historical names, types, and initial defaults when an old data file lacks such a key. + ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), + nameMapping, getBase64EncodedInitialDefaultsForScan()); + } + + @VisibleForTesting + Map getBase64EncodedInitialDefaultsForScan() throws UserException { + TableScan tableScan = createTableScan(); + Snapshot snapshot = tableScan.snapshot(); + // TableScan.schema() starts from the table's current schema even for useSnapshot/useRef. + // Resolve the selected snapshot's schema id explicitly so this metadata describes the same + // snapshot as source.getTargetTable().getColumns(). Otherwise a later type change can make + // BE decode a historical non-binary default as Base64, or fail to decode a binary default. + Schema scanSchema = snapshot == null + ? tableScan.schema() + : tableScan.table().schemas().get(snapshot.schemaId()); + return IcebergUtils.getBase64EncodedInitialDefaults( + Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan snapshot is null")); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 5ce09be0456e5f..3ecb3a72d6b023 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -219,7 +219,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { TFileScanRangeParams params = new TFileScanRangeParams(); Long schemaId = 500L; - Column col1 = new Column("c1", Type.INT, true); + Column col1 = new Column("c1", Type.INT, false, null, true, "7", ""); col1.setUniqueId(101); Column col2 = new Column("c2", Type.VARCHAR, false); col2.setUniqueId(102); @@ -230,7 +230,10 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { nameMapping.put(col1.getUniqueId(), Arrays.asList("m_c1")); nameMapping.put(col2.getUniqueId(), Arrays.asList("m_c2_a", "m_c2_b")); - ExternalUtil.initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping); + Map base64InitialDefaults = new HashMap<>(); + base64InitialDefaults.put(col2.getUniqueId(), "AAEC/w=="); + ExternalUtil.initSchemaInfoForAllColumn( + params, schemaId, columns, nameMapping, base64InitialDefaults); Assert.assertEquals(schemaId.longValue(), params.getCurrentSchemaId()); List history = params.getHistorySchemaInfo(); @@ -251,12 +254,15 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col1.isAllowNull(), field1.isIsOptional()); Assert.assertEquals(col1.getType().toColumnTypeThrift(), field1.getType()); Assert.assertEquals(Arrays.asList("m_c1"), field1.getNameMapping()); + Assert.assertEquals("7", field1.getInitialDefaultValue()); + Assert.assertFalse(field1.isSetInitialDefaultValueIsBase64()); Assert.assertEquals(col2.getName(), field2.getName()); Assert.assertEquals(col2.getUniqueId(), field2.getId()); Assert.assertEquals(col2.isAllowNull(), field2.isIsOptional()); Assert.assertEquals(col2.getType().toColumnTypeThrift(), field2.getType()); Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); + Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); + Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } } - diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index dc344b08e13812..0a798faff93fef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -198,6 +198,10 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedTableScan).when(mockedTableScan).filter(ArgumentMatchers.any()); Mockito.doReturn(mockedTableScan).when(mockedTableScan).planWith(ArgumentMatchers.any()); Mockito.doReturn(null).when(mockedTableScan).snapshot(); + // Keep the scan schema aligned with the mocked table schema. IcebergScanNode reads the + // selected scan schema when serializing initial defaults, and several tests temporarily + // replace the table schema to exercise partition transforms. + Mockito.doAnswer(invocation -> mockedIcebergTable.schema()).when(mockedTableScan).schema(); Mockito.doReturn(CloseableIterable.withNoopClose(java.util.Collections.emptyList())) .when(mockedTableScan).planFiles(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 8625b36f283ab4..ee34b4bee50b0c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -75,6 +75,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; public class IcebergUtilsTest { @Test @@ -299,6 +300,48 @@ public void testParseSchemaPreservesNonLowercaseColumnNames() { Assert.assertEquals("PART", columns.get(1).getName()); } + @Test + public void testParseSchemaPreservesInitialDefault() { + Schema schema = new Schema( + Types.NestedField.optional("added_column") + .withId(1) + .ofType(Types.IntegerType.get()) + .withInitialDefault(7) + .build(), + Types.NestedField.optional("added_timestamp") + .withId(2) + .ofType(Types.TimestampType.withoutZone()) + .withInitialDefault(1_704_067_200_123_456L) + .build(), + Types.NestedField.optional("added_uuid") + .withId(3) + .ofType(Types.UUIDType.get()) + .withInitialDefault(UUID.fromString("00000000-0000-0000-0000-000000000000")) + .build(), + Types.NestedField.optional("added_binary") + .withId(4) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build(), + Types.NestedField.optional("added_fixed") + .withId(5) + .ofType(Types.FixedType.ofLength(4)) + .withInitialDefault(ByteBuffer.wrap(new byte[] {3, 2, 1, 0})) + .build()); + + List columns = IcebergUtils.parseSchema(schema, true, false); + + Assert.assertEquals("7", columns.get(0).getDefaultValue()); + Assert.assertEquals("2024-01-01 00:00:00.123456", columns.get(1).getDefaultValue()); + Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", columns.get(2).getDefaultValue()); + Assert.assertEquals("AAEC/w==", columns.get(3).getDefaultValue()); + + Map base64Defaults = IcebergUtils.getBase64EncodedInitialDefaults(schema); + Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", base64Defaults.get(3)); + Assert.assertEquals("AAEC/w==", base64Defaults.get(4)); + Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); + } + @Test public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index f55b4f6f8e8027..46abdf08a587e9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -29,6 +29,11 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ScanTaskUtil; import org.junit.Assert; import org.junit.Test; @@ -36,23 +41,64 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; +import java.util.Map; public class IcebergScanNodeTest { private static final long MB = 1024L * 1024L; private static class TestIcebergScanNode extends IcebergScanNode { + private TableScan tableScan; + TestIcebergScanNode(SessionVariable sv) { super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), sv, ScanContext.EMPTY); } + void setTableScan(TableScan tableScan) { + this.tableScan = tableScan; + } + + @Override + public TableScan createTableScan() { + return tableScan; + } + @Override public boolean isBatchMode() { return false; } } + @Test + public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { + Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Schema currentSchema = new Schema(Types.NestedField.optional("current_string") + .withId(7) + .ofType(Types.StringType.get()) + .withInitialDefault("not-base64") + .build()); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Mockito.when(snapshot.schemaId()).thenReturn(11); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + TableScan snapshotScan = Mockito.mock(TableScan.class); + Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); + Mockito.when(snapshotScan.table()).thenReturn(table); + Mockito.when(snapshotScan.schema()).thenReturn(currentSchema); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + node.setTableScan(snapshotScan); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); diff --git a/gensrc/thrift/ExternalTableSchema.thrift b/gensrc/thrift/ExternalTableSchema.thrift index 14972f8d434784..46ae54781ae825 100644 --- a/gensrc/thrift/ExternalTableSchema.thrift +++ b/gensrc/thrift/ExternalTableSchema.thrift @@ -49,7 +49,16 @@ struct TField { 3: optional string name, // Field name 4: optional Types.TColumnType type, // Corresponding Doris column type 5: optional TNestedField nestedField // Nested field definition (for array, struct, or map types) - 6: optional list name_mapping // iceberg : schema.name-mapping.default, for missing column id. + 6: optional list name_mapping, // iceberg : schema.name-mapping.default, for missing column id. + // Iceberg initial default normalized for transport to BE. Binary-like Iceberg values use + // Base64 because Thrift's Java string carrier cannot preserve arbitrary bytes; other primitive + // values use Doris' FE string representation. An old data file that predates this field + // logically contains this value rather than NULL. + 7: optional string initial_default_value, + // True when initial_default_value is Base64 and must be decoded before constructing the Doris + // STRING/CHAR/VARBINARY value. This cannot be inferred from the Doris type because Iceberg + // UUID/BINARY/FIXED may map either to VARBINARY or to STRING/CHAR. + 8: optional bool initial_default_value_is_base64 } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy index b4394755c9aa97..f24b075443ab4d 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy @@ -42,9 +42,17 @@ suite("test_iceberg_equality_delete_with_schema_change", "p0,external,doris,exte sql """switch ${catalog_name};""" sql """ use multi_catalog;""" - + def enableFileScannerV2Rows = sql """show variables like 'enable_file_scanner_v2'""" + assertTrue(enableFileScannerV2Rows.size() > 0, + "Session variable enable_file_scanner_v2 is not found") + String originalEnableFileScannerV2 = enableFileScannerV2Rows[0][1].toString() + // The existing output was generated by the legacy scanner and is kept as the differential + // baseline. Run the complete schema-evolution matrix through V2 so missing/renamed equality + // delete keys must produce byte-for-byte identical query results. + sql """set enable_file_scanner_v2=true""" - for (String format: ["par","orc"]) { + try { + for (String format: ["par","orc"]) { // Basic full table scan order_qt_q1 """ SELECT * FROM equality_delete_${format}_1 ORDER BY new_new_id; """ @@ -190,5 +198,9 @@ suite("test_iceberg_equality_delete_with_schema_change", "p0,external,doris,exte order_qt_q70 """ SELECT new_new_id, new_name, data FROM equality_delete_${format}_1 WHERE new_new_id = 1 UNION ALL SELECT new_new_id, new_name, data FROM equality_delete_${format}_2 WHERE new_new_id = 1; """ + + } + } finally { + sql """set enable_file_scanner_v2=${originalEnableFileScannerV2}""" } }