diff --git a/be/src/format/table/iceberg_reader.cpp b/be/src/format/table/iceberg_reader.cpp index 110a10f74c0c77..38bb395120df24 100644 --- a/be/src/format/table/iceberg_reader.cpp +++ b/be/src/format/table/iceberg_reader.cpp @@ -221,7 +221,8 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { _all_required_col_names = ctx->column_names; // Create column IDs from field descriptor - auto column_id_result = _create_column_ids(field_desc, ctx->tuple_descriptor); + auto column_id_result = + _create_column_ids(field_desc, ctx->tuple_descriptor, ctx->table_info_node); ctx->column_ids = std::move(column_id_result.column_ids); ctx->filter_column_ids = std::move(column_id_result.filter_column_ids); @@ -240,13 +241,20 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { // - Prefix with __equality_delete_column__ to avoid name conflicts // - Correctly map table_col_name → file_col_name in table_info_node const static std::string EQ_DELETE_PRE = "__equality_delete_column__"; - std::unordered_map field_id_to_file_col_name; + std::unordered_map field_id_to_file_column; + bool all_file_columns_have_field_ids = true; for (int i = 0; i < field_desc->size(); ++i) { - auto field_schema = field_desc->get_column(i); + const auto* field_schema = field_desc->get_column(i); if (field_schema) { - field_id_to_file_col_name[field_schema->field_id] = field_schema->name; + field_id_to_file_column[field_schema->field_id] = field_schema; + if (field_schema->field_id < 0) { + all_file_columns_have_field_ids = false; + } } } + const auto struct_node = + std::dynamic_pointer_cast(ctx->table_info_node); + DORIS_CHECK(struct_node != nullptr); // Rebuild _expand_col_names with proper file-column-based names std::vector new_expand_col_names; @@ -261,12 +269,30 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { } } - std::string file_col_name = old_name; - auto it = field_id_to_file_col_name.find(field_id); - if (it != field_id_to_file_col_name.end()) { - file_col_name = it->second; + const FieldSchema* file_column = nullptr; + if (!all_file_columns_have_field_ids && struct_node->get_children().contains(old_name) && + struct_node->children_column_exists(old_name)) { + // Iceberg files written without field ids must use schema.name-mapping.default. The + // root schema mapper deliberately switches the whole file to BY_NAME when even one + // top-level field id is absent. Hidden equality keys must make the same choice: a + // different physical column may still carry this key's stale id after migration. + const auto& mapped_name = struct_node->children_file_column_name(old_name); + for (int j = 0; j < field_desc->size(); ++j) { + const auto* candidate = field_desc->get_column(j); + if (candidate != nullptr && candidate->name == mapped_name) { + file_column = candidate; + break; + } + } + DORIS_CHECK(file_column != nullptr); + } else if (all_file_columns_have_field_ids) { + auto id_it = field_id_to_file_column.find(field_id); + if (id_it != field_id_to_file_column.end()) { + file_column = id_it->second; + } } + const std::string file_col_name = file_column == nullptr ? old_name : file_column->name; std::string table_col_name = EQ_DELETE_PRE + file_col_name; // Update _id_to_block_column_name @@ -279,18 +305,21 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { _expand_columns[i].name = table_col_name; } + if (file_column == nullptr) { + DORIS_CHECK(i < _expand_columns.size()); + RETURN_IF_ERROR(_register_missing_equality_delete_column(field_id, table_col_name, + _expand_columns[i].type)); + // The old data file predates this equality key. Keep it in the expand block so the + // synthesized-column hook can materialize its logical initial default before reader + // filtering, but do not advertise it to Parquet as a physical child. + new_expand_col_names.push_back(table_col_name); + continue; + } + new_expand_col_names.push_back(table_col_name); // Add column IDs - if (it != field_id_to_file_col_name.end()) { - for (int j = 0; j < field_desc->size(); ++j) { - auto field_schema = field_desc->get_column(j); - if (field_schema && field_schema->field_id == field_id) { - ctx->column_ids.insert(field_schema->get_column_id()); - break; - } - } - } + ctx->column_ids.insert(file_column->get_column_id()); // Register in table_info_node: table_col_name → file_col_name ctx->column_names.push_back(table_col_name); @@ -308,8 +337,9 @@ Status IcebergParquetReader::on_before_init_reader(ReaderInitContext* ctx) { // ============================================================================ // IcebergParquetReader: _create_column_ids // ============================================================================ -ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* field_desc, - const TupleDescriptor* tuple_descriptor) { +ColumnIdResult IcebergParquetReader::_create_column_ids( + const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor, + const std::shared_ptr& table_info_node) { auto* mutable_field_desc = const_cast(field_desc); mutable_field_desc->assign_ids(); @@ -335,11 +365,32 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f }; for (const auto* slot : tuple_descriptor->slots()) { - auto it = iceberg_id_to_field_schema_map.find(slot->col_unique_id()); - if (it == iceberg_id_to_field_schema_map.end()) { + const FieldSchema* field_schema = nullptr; + if (table_info_node != nullptr) { + if (table_info_node->children_column_exists(slot->col_name())) { + // Use the physical child selected by the schema-mapping pass. This keeps partial-id + // files in BY_NAME mode from binding a projected column through an unrelated stale + // field id. + const auto& file_column_name = + table_info_node->children_file_column_name(slot->col_name()); + for (int i = 0; i < field_desc->size(); ++i) { + const auto* candidate = field_desc->get_column(i); + if (candidate != nullptr && candidate->name == file_column_name) { + field_schema = candidate; + break; + } + } + DORIS_CHECK(field_schema != nullptr); + } + } else { + auto it = iceberg_id_to_field_schema_map.find(slot->col_unique_id()); + if (it != iceberg_id_to_field_schema_map.end()) { + field_schema = it->second; + } + } + if (field_schema == nullptr) { continue; } - auto field_schema = it->second; if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY && slot->col_type() != TYPE_MAP)) { @@ -358,7 +409,7 @@ ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* f process_access_paths(field_schema, predicate_access_paths, filter_column_ids); } } - return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids)); + return {std::move(column_ids), std::move(filter_column_ids)}; } // ============================================================================ @@ -503,7 +554,8 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { _all_required_col_names = ctx->column_names; // Create column IDs from ORC type - auto column_id_result = _create_column_ids(orc_type_ptr, ctx->tuple_descriptor); + auto column_id_result = + _create_column_ids(orc_type_ptr, ctx->tuple_descriptor, ctx->table_info_node); ctx->column_ids = std::move(column_id_result.column_ids); ctx->filter_column_ids = std::move(column_id_result.filter_column_ids); @@ -518,15 +570,20 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { // Add expand column IDs for equality delete and remap expand column names // (matching master's behavior with __equality_delete_column__ prefix) const static std::string EQ_DELETE_PRE = "__equality_delete_column__"; - std::unordered_map field_id_to_file_col_name; + std::unordered_map field_id_to_file_column; + bool all_file_columns_have_field_ids = true; for (uint64_t i = 0; i < orc_type_ptr->getSubtypeCount(); ++i) { - std::string col_name = orc_type_ptr->getFieldName(i); const orc::Type* sub_type = orc_type_ptr->getSubtype(i); if (sub_type->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) { int fid = std::stoi(sub_type->getAttributeValue(ICEBERG_ORC_ATTRIBUTE)); - field_id_to_file_col_name[fid] = col_name; + field_id_to_file_column[fid] = sub_type; + } else { + all_file_columns_have_field_ids = false; } } + const auto struct_node = + std::dynamic_pointer_cast(ctx->table_info_node); + DORIS_CHECK(struct_node != nullptr); std::vector new_expand_col_names; for (size_t i = 0; i < _expand_col_names.size(); ++i) { @@ -539,12 +596,36 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { } } - std::string file_col_name = old_name; - auto it = field_id_to_file_col_name.find(field_id); - if (it != field_id_to_file_col_name.end()) { - file_col_name = it->second; + const orc::Type* file_column = nullptr; + if (!all_file_columns_have_field_ids && struct_node->get_children().contains(old_name) && + struct_node->children_column_exists(old_name)) { + // Match the root ORC schema mapper's all-or-nothing BY_NAME decision. Accepting a + // matching id in a partial-id file could bind this hidden key to an unrelated stale + // column instead of the current name or historical alias selected by table_info_node. + const auto& mapped_name = struct_node->children_file_column_name(old_name); + for (uint64_t j = 0; j < orc_type_ptr->getSubtypeCount(); ++j) { + if (orc_type_ptr->getFieldName(j) == mapped_name) { + file_column = orc_type_ptr->getSubtype(j); + break; + } + } + DORIS_CHECK(file_column != nullptr); + } else if (all_file_columns_have_field_ids) { + auto id_it = field_id_to_file_column.find(field_id); + if (id_it != field_id_to_file_column.end()) { + file_column = id_it->second; + } } + std::string file_col_name = old_name; + if (file_column != nullptr) { + for (uint64_t j = 0; j < orc_type_ptr->getSubtypeCount(); ++j) { + if (orc_type_ptr->getSubtype(j) == file_column) { + file_col_name = orc_type_ptr->getFieldName(j); + break; + } + } + } std::string table_col_name = EQ_DELETE_PRE + file_col_name; if (field_id >= 0) { @@ -553,18 +634,21 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { if (i < _expand_columns.size()) { _expand_columns[i].name = table_col_name; } + if (file_column == nullptr) { + DORIS_CHECK(i < _expand_columns.size()); + RETURN_IF_ERROR(_register_missing_equality_delete_column(field_id, table_col_name, + _expand_columns[i].type)); + // The old data file predates this equality key. Keep it in the expand block so the + // synthesized-column hook can materialize its logical initial default before ORC's + // block-size checks. Adding it to column_names/table_info_node would mark it as an + // existing ORC child and make OrcReader read a column that is not present in the file. + new_expand_col_names.push_back(table_col_name); + continue; + } new_expand_col_names.push_back(table_col_name); // Add column IDs - if (it != field_id_to_file_col_name.end()) { - for (uint64_t j = 0; j < orc_type_ptr->getSubtypeCount(); ++j) { - const orc::Type* sub_type = orc_type_ptr->getSubtype(j); - if (orc_type_ptr->getFieldName(j) == file_col_name) { - ctx->column_ids.insert(sub_type->getColumnId()); - break; - } - } - } + ctx->column_ids.insert(file_column->getColumnId()); ctx->column_names.push_back(table_col_name); ctx->table_info_node->add_children(table_col_name, file_col_name, @@ -578,12 +662,15 @@ Status IcebergOrcReader::on_before_init_reader(ReaderInitContext* ctx) { // ============================================================================ // IcebergOrcReader: _create_column_ids // ============================================================================ -ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type, - const TupleDescriptor* tuple_descriptor) { +ColumnIdResult IcebergOrcReader::_create_column_ids( + const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor, + const std::shared_ptr& table_info_node) { std::unordered_map iceberg_id_to_orc_type_map; for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) { - auto orc_sub_type = orc_type->getSubtype(i); - if (!orc_sub_type) continue; + const auto* orc_sub_type = orc_type->getSubtype(i); + if (!orc_sub_type) { + continue; + } if (!orc_sub_type->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) { continue; } @@ -605,11 +692,31 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type, }; for (const auto* slot : tuple_descriptor->slots()) { - auto it = iceberg_id_to_orc_type_map.find(slot->col_unique_id()); - if (it == iceberg_id_to_orc_type_map.end()) { + const orc::Type* orc_field = nullptr; + if (table_info_node != nullptr) { + if (table_info_node->children_column_exists(slot->col_name())) { + // Select the physical child resolved by the shared schema-mapping pass. Hidden + // equality keys and projected columns must obey the same BY_NAME decision for + // partial-id ORC files. + const auto& file_column_name = + table_info_node->children_file_column_name(slot->col_name()); + for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) { + if (orc_type->getFieldName(i) == file_column_name) { + orc_field = orc_type->getSubtype(i); + break; + } + } + DORIS_CHECK(orc_field != nullptr); + } + } else { + auto it = iceberg_id_to_orc_type_map.find(slot->col_unique_id()); + if (it != iceberg_id_to_orc_type_map.end()) { + orc_field = it->second; + } + } + if (orc_field == nullptr) { continue; } - const orc::Type* orc_field = it->second; if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY && slot->col_type() != TYPE_MAP)) { @@ -629,7 +736,7 @@ ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type, } } - return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids)); + return {std::move(column_ids), std::move(filter_column_ids)}; } // ============================================================================ diff --git a/be/src/format/table/iceberg_reader.h b/be/src/format/table/iceberg_reader.h index 205e46a3a1bf7b..9a237848b522f7 100644 --- a/be/src/format/table/iceberg_reader.h +++ b/be/src/format/table/iceberg_reader.h @@ -105,8 +105,9 @@ class IcebergParquetReader final : public IcebergReaderMixin { this->get_state(), this->_meta_cache); } - static ColumnIdResult _create_column_ids(const FieldDescriptor* field_desc, - const TupleDescriptor* tuple_descriptor); + static ColumnIdResult _create_column_ids( + const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor, + const std::shared_ptr& table_info_node = nullptr); private: Status _read_position_delete_file(const TFileRangeDesc* delete_range, @@ -151,8 +152,9 @@ class IcebergOrcReader final : public IcebergReaderMixin { this->get_io_ctx(), this->_meta_cache); } - static ColumnIdResult _create_column_ids(const orc::Type* orc_type, - const TupleDescriptor* tuple_descriptor); + static ColumnIdResult _create_column_ids( + const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor, + const std::shared_ptr& table_info_node = nullptr); static const std::string ICEBERG_ORC_ATTRIBUTE; diff --git a/be/src/format/table/iceberg_reader_mixin.h b/be/src/format/table/iceberg_reader_mixin.h index 2bc7be1741f756..aee211aee019cd 100644 --- a/be/src/format/table/iceberg_reader_mixin.h +++ b/be/src/format/table/iceberg_reader_mixin.h @@ -17,9 +17,12 @@ #pragma once +#include + #include #include #include +#include #include #include #include @@ -33,6 +36,7 @@ #include "core/column/column_struct.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/primitive_type.h" #include "format/generic_reader.h" #include "format/table/equality_delete.h" #include "format/table/iceberg_delete_file_reader_helper.h" @@ -40,6 +44,7 @@ #include "runtime/runtime_profile.h" #include "runtime/runtime_state.h" #include "storage/olap_common.h" +#include "util/url_coding.h" namespace doris { class TIcebergDeleteFileDesc; @@ -135,6 +140,20 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { return _position_delete_base(data_file_path, delete_files); } + void TEST_set_column_name_to_block_index( + std::unordered_map* column_name_to_block_index) { + this->col_name_to_block_idx_ref() = column_name_to_block_index; + } + + Status TEST_register_missing_equality_delete_column(int32_t field_id, const std::string& name, + const DataTypePtr& delete_key_type) { + return _register_missing_equality_delete_column(field_id, name, delete_key_type); + } + + Status TEST_materialize_missing_equality_delete_columns(Block* block, size_t rows) { + return _materialize_missing_equality_delete_columns(block, rows); + } + protected: // ---- Hook implementations ---- @@ -251,6 +270,11 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { Status _expand_block_if_need(Block* block); Status _shrink_block_if_need(Block* block); + Status _register_missing_equality_delete_column(int32_t field_id, const std::string& name, + const DataTypePtr& delete_key_type); + Status _materialize_missing_equality_delete_column(Block* block, const std::string& name, + const ColumnPtr& value, size_t rows); + Status _materialize_missing_equality_delete_columns(Block* block, size_t rows); // Type aliases — must be defined before member function declarations that use them. using DeleteRows = std::vector; @@ -352,6 +376,7 @@ class IcebergReaderMixin : public BaseReader, public TableSchemaChangeHelper { const DeletionVector* _iceberg_deletion_vector = nullptr; std::vector _expand_col_names; std::vector _expand_columns; + std::unordered_map _missing_equality_delete_values; std::vector _all_required_col_names; Fileformat _file_format = Fileformat::NONE; @@ -624,6 +649,11 @@ Status IcebergReaderMixin::_expand_block_if_need(Block* block) { auto block_names = block->get_names(); names.insert(block_names.begin(), block_names.end()); for (auto& col : _expand_columns) { + if (_missing_equality_delete_values.contains(col.name)) { + // Missing equality keys are logical columns, not physical file columns. Add them only + // after the base reader has established the batch row count. + continue; + } if (names.contains(col.name)) { return Status::InternalError("Wrong expand column '{}'", col.name); } @@ -634,6 +664,118 @@ Status IcebergReaderMixin::_expand_block_if_need(Block* block) { return Status::OK(); } +template +Status IcebergReaderMixin::_register_missing_equality_delete_column( + int32_t field_id, const std::string& name, const DataTypePtr& delete_key_type) { + DORIS_CHECK(delete_key_type != nullptr); + const schema::external::TField* table_field = nullptr; + const auto& scan_params = this->get_scan_params(); + const schema::external::TSchema* current_schema = nullptr; + if (scan_params.__isset.history_schema_info && !scan_params.history_schema_info.empty()) { + current_schema = &scan_params.history_schema_info.front(); + } + if (current_schema != nullptr && scan_params.__isset.current_schema_id) { + const auto schema_it = std::ranges::find_if( + scan_params.history_schema_info, [&](const schema::external::TSchema& schema) { + return schema.__isset.schema_id && + schema.schema_id == scan_params.current_schema_id; + }); + if (schema_it != scan_params.history_schema_info.end()) { + current_schema = &*schema_it; + } else { + current_schema = nullptr; + } + } + if (current_schema != nullptr && current_schema->__isset.root_field) { + for (const auto& field_ptr : current_schema->root_field.fields) { + if (field_ptr.__isset.field_ptr && field_ptr.field_ptr != nullptr && + field_ptr.field_ptr->__isset.id && field_ptr.field_ptr->id == field_id) { + table_field = field_ptr.field_ptr.get(); + break; + } + } + } + if (table_field == nullptr) { + // A projected descriptor from an older FE may omit a hidden equality key. Without the + // current Iceberg field metadata BE cannot distinguish a true NULL initial default from a + // non-NULL default, so continuing would risk silently keeping or deleting wrong rows. + return Status::InternalError( + "Missing Iceberg schema metadata for equality-delete field id {}", field_id); + } + + Field value; + if (table_field->__isset.initial_default_value) { + const auto nested_type = remove_nullable(delete_key_type); + const bool default_is_base64 = (table_field->__isset.initial_default_value_is_base64 && + table_field->initial_default_value_is_base64) || + (table_field->__isset.type && + thrift_to_type(table_field->type.type) == TYPE_VARBINARY); + if (default_is_base64) { + std::string decoded_default; + if (!base64_decode(table_field->initial_default_value, &decoded_default)) { + return Status::InvalidArgument( + "Invalid Base64 Iceberg initial default for field {}", table_field->name); + } + if (nested_type->get_primitive_type() == TYPE_VARBINARY) { + value = Field::create_field(StringView(decoded_default)); + } else { + DORIS_CHECK(is_string_type(nested_type->get_primitive_type())); + value = Field::create_field(decoded_default); + } + } else { + RETURN_IF_ERROR(nested_type->get_serde()->from_fe_string( + table_field->initial_default_value, value)); + } + } + const bool inserted = _missing_equality_delete_values + .emplace(name, delete_key_type->create_column_const(1, value)) + .second; + DORIS_CHECK(inserted); + this->register_synthesized_column_handler( + name, [this, name](Block* block, size_t rows) -> Status { + DORIS_CHECK(_missing_equality_delete_values.contains(name)); + return _materialize_missing_equality_delete_column( + block, name, _missing_equality_delete_values.at(name), rows); + }); + return Status::OK(); +} + +template +Status IcebergReaderMixin::_materialize_missing_equality_delete_column( + Block* block, const std::string& name, const ColumnPtr& value, size_t rows) { + if (!this->col_name_to_block_idx_ref()->contains(name)) { + // ORC must not register a key that is absent from the file as a physical child. In that + // case the reader block has no slot for the synthesized key, so append one here before + // equality-delete filtering. MultiEqualityDelete requires a full, batch-sized column. + const auto expand_col = std::ranges::find_if( + _expand_columns, + [&](const ColumnWithTypeAndName& col) { return col.name == name; }); + DORIS_CHECK(expand_col != _expand_columns.end()); + (*this->col_name_to_block_idx_ref())[name] = block->columns(); + block->insert({value->clone_resized(rows)->convert_to_full_column_if_const(), + expand_col->type, name}); + return Status::OK(); + } + const auto position = this->col_name_to_block_idx_ref()->at(name); + DORIS_CHECK(position < block->columns()); + DORIS_CHECK(block->get_by_position(position).column->empty()); + // MultiEqualityDelete hashes each key column directly. Materialize the repeated default so + // every key has the batch row count; a ColumnConst keeps only one nested value and therefore + // cannot participate in the row-wise multi-column hash contract. + block->get_by_position(position).column = + value->clone_resized(rows)->convert_to_full_column_if_const(); + return Status::OK(); +} + +template +Status IcebergReaderMixin::_materialize_missing_equality_delete_columns(Block* block, + size_t rows) { + for (const auto& [name, value] : _missing_equality_delete_values) { + RETURN_IF_ERROR(_materialize_missing_equality_delete_column(block, name, value, rows)); + } + return Status::OK(); +} + template Status IcebergReaderMixin::_shrink_block_if_need(Block* block) { std::set positions_to_erase; diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index 7816ea8263cb42..ac510caaf07ac4 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -258,6 +258,12 @@ struct ColumnDefinition { // Expression used to materialize missing/default/generated values when the column is not read // directly from the file. VExprContextSPtr default_expr = nullptr; + // Table-format initial default normalized for transport from FE. Binary-like values use Base64 + // and set initial_default_value_is_base64 because they can map to STRING/CHAR or VARBINARY. + // Unlike default_expr, this metadata is also available for hidden delete-predicate columns + // that are absent from the query projection. + std::optional initial_default_value = std::nullopt; + bool initial_default_value_is_base64 = false; // Partition columns are constants from split metadata and should not be matched against file // schema unless table-format logic explicitly asks for it. bool is_partition_key = false; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 572e76280a0e37..6a0daf903f01d4 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -230,6 +230,11 @@ std::string join_debug_strings(const std::vector& values, Formatter formatter } // namespace +const ColumnDefinition* find_column_by_name(const ColumnDefinition& table_column, + const std::vector& file_schema) { + return matcher_for_mode(TableColumnMappingMode::BY_NAME).find(table_column, file_schema); +} + const Field* find_partition_value(const ColumnDefinition& table_column, const std::map& partition_values) { const auto find_by_name = [&](const std::string& name) -> const Field* { @@ -1237,6 +1242,26 @@ static std::vector synthesize_complex_children_from_type( return children; } +static void align_struct_child_types_with_parent(const DataTypePtr& parent_type, + std::vector& children) { + const auto nested_parent_type = remove_nullable(parent_type); + DORIS_CHECK(nested_parent_type->get_primitive_type() == TYPE_STRUCT); + const auto type_children = synthesize_complex_children_from_type(parent_type); + for (auto& child : children) { + const auto type_child = std::ranges::find_if( + type_children, [&](const auto& candidate) { return candidate.name == child.name; }); + DORIS_CHECK(type_child != type_children.end()) + << "Complex child '" << child.name + << "' is absent from its parent table type: " << parent_type->get_name(); + // The parent DataType is the authoritative output contract. Nested schema descriptors can + // omit child nullability even though the parent struct still declares Nullable(String). + // For example, the Iceberg full-schema-change case maps nullable `location` to `city`, but + // its child descriptor carries String. Keeping String here makes rematerialization strip + // the child's null map and creates Struct(String) under a Struct(Nullable(String)) type. + child.type = type_child->type; + } +} + static bool has_table_child_named(const std::vector& children, std::string_view name) { return std::ranges::any_of(children, [&](const ColumnDefinition& child) { @@ -1245,8 +1270,7 @@ static bool has_table_child_named(const std::vector& children, } static void complete_required_complex_children_from_type(const DataTypePtr& type, - std::vector* children) { - DORIS_CHECK(children != nullptr); + std::vector& children) { if (type == nullptr) { return; } @@ -1261,8 +1285,8 @@ static void complete_required_complex_children_from_type(const DataTypePtr& type // In that shape the scanner keeps the value stream readable, but the table projection can // carry only the key child. Add the missing value child so recursive mapping can evolve the // value type instead of letting TableReader cast old/new value structs directly. - if (has_table_child_named(*children, "key") && !has_table_child_named(*children, "value")) { - children->push_back(synthetic_child_definition("value", map_type->get_value_type(), 1)); + if (has_table_child_named(children, "key") && !has_table_child_named(children, "value")) { + children.push_back(synthetic_child_definition("value", map_type->get_value_type(), 1)); } break; } @@ -1279,6 +1303,37 @@ static void complete_required_complex_children_from_type(const DataTypePtr& type } } +struct PreparedTableChildren { + std::vector children; + bool synthesized_from_type = false; +}; + +static PreparedTableChildren prepare_table_children_for_mapping( + const ColumnDefinition& table_column, const DataTypePtr& file_type) { + PreparedTableChildren prepared {.children = table_column.children}; + const auto nested_table_type = remove_nullable(table_column.type); + + // Some scan paths, especially SELECT *, only carry the complete complex DataType for a table + // column and leave ColumnDefinition::children empty. Synthesize the hierarchy so recursive + // mapping can evolve nested fields instead of falling back to an invalid whole-column cast. + prepared.synthesized_from_type = prepared.children.empty() && + is_complex_type(nested_table_type->get_primitive_type()) && + !table_column.type->equals(*file_type); + if (prepared.synthesized_from_type) { + prepared.children = synthesize_complex_children_from_type(table_column.type); + } else if (!prepared.children.empty() && !table_column.type->equals(*file_type)) { + complete_required_complex_children_from_type(table_column.type, prepared.children); + } + + if (!prepared.children.empty() && nested_table_type->get_primitive_type() == TYPE_STRUCT) { + // Struct children are table fields, so the parent Struct type is authoritative for their + // nullability. ARRAY and MAP children are format-level structural wrappers and keep the + // descriptor types used by their recursive mappings. + align_struct_child_types_with_parent(table_column.type, prepared.children); + } + return prepared; +} + static Status validate_file_schema_children(const ColumnDefinition& file_field) { if (file_field.type == nullptr) { return Status::InternalError("File column '{}' has null type", file_field.name); @@ -2016,29 +2071,8 @@ Status TableColumnMapper::_create_direct_mapping(const ColumnDefinition& table_c mapping->filter_conversion = direct_filter_conversion(*mapping); mapping->child_mappings.clear(); - auto table_children = table_column.children; - const auto nested_table_type = remove_nullable(mapping->table_type); - // Some scan paths, especially SELECT *, only carry the complete complex DataType for a table - // column and leave ColumnDefinition::children empty. If the file type is an older complex - // schema, treating this as a leaf mapping would make TableReader fall back to a plain CAST. - // That is invalid for evolved structs with different field counts. - // - // Example: - // table column type: Map(String, Struct(age, full_name, gender)) - // old file type: Map(String, Struct(age, name)) - // table children: empty - // - // Synthesize key/value/struct-field children from the table type so the normal recursive - // mapping path can rematerialize `name -> full_name` and fill missing `gender` with defaults, - // instead of trying to CAST Struct(age, name) to Struct(age, full_name, gender). - const bool synthesized_table_children = - table_children.empty() && is_complex_type(nested_table_type->get_primitive_type()) && - !mapping->table_type->equals(*mapping->file_type); - if (synthesized_table_children) { - table_children = synthesize_complex_children_from_type(mapping->table_type); - } else if (!table_children.empty() && !mapping->table_type->equals(*mapping->file_type)) { - complete_required_complex_children_from_type(mapping->table_type, &table_children); - } + auto [table_children, synthesized_table_children] = + prepare_table_children_for_mapping(table_column, mapping->file_type); if (!table_children.empty()) { if (!is_complex_type(remove_nullable(mapping->file_type)->get_primitive_type())) { diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 28e31a12639936..4f417a680aad60 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -163,6 +163,10 @@ struct TableColumnMapperOptions { Status clone_table_expr_tree(const VExprSPtr& expr, VExprSPtr* cloned_expr); const Field* find_partition_value(const ColumnDefinition& table_column, const std::map& partition_values); +// Apply the same case-insensitive logical name, string identifier, and bidirectional alias rules +// used by TableColumnMapper's BY_NAME mode. +const ColumnDefinition* find_column_by_name(const ColumnDefinition& table_column, + const std::vector& file_schema); // Generic mapping layer from table schema to file schema. // Iceberg uses BY_FIELD_ID. Plain by-name formats can reuse this component as well, so keep this diff --git a/be/src/format_v2/expr/equality_delete_predicate.cpp b/be/src/format_v2/expr/equality_delete_predicate.cpp index 13454e3b22f116..4d5c511abdb181 100644 --- a/be/src/format_v2/expr/equality_delete_predicate.cpp +++ b/be/src/format_v2/expr/equality_delete_predicate.cpp @@ -19,6 +19,7 @@ #include +#include #include #include "common/status.h" @@ -92,29 +93,52 @@ void EqualityDeletePredicate::close(VExprContext* context, Status EqualityDeletePredicate::execute(VExprContext* context, Block* block, int* result_column_id) const { + size_t rows = 0; + for (const auto& column : block->get_columns()) { + rows = std::max(rows, column->size()); + } + // Lazy readers may leave an unread projected column at block position zero while predicate + // columns (or the all-missing-key row-count carrier) are populated later. Block::rows() only + // inspects the first column, so derive the explicit expression count from all materialized + // columns and let execute_column() enforce that every result has exactly this batch size. + ColumnPtr result_column; + RETURN_IF_ERROR(execute_column(context, block, nullptr, rows, result_column)); + block->insert({std::move(result_column), std::make_shared(), expr_name()}); + *result_column_id = static_cast(block->columns() - 1); + return Status::OK(); +} + +Status EqualityDeletePredicate::execute_column_impl(VExprContext* context, const Block* block, + const Selector* selector, size_t count, + ColumnPtr& result_column) const { if (_children.size() != _field_ids.size()) { return Status::InternalError( "EqualityDeletePredicate should have {} child exprs, but got {}", _field_ids.size(), _children.size()); } + // This path is required when every equality key is a literal, notably when all key columns are + // absent from an older Iceberg data file and are represented by typed NULL literals. Preserve + // `count` so the constant result can be expanded to the caller's batch size when necessary. Block data_key_block; for (const auto& child : _children) { - Block eval_block = *block; - int slot = -1; - RETURN_IF_ERROR(child->execute(context, &eval_block, &slot)); - const auto& key_column = eval_block.get_by_position(slot); - data_key_block.insert({key_column.column, key_column.type, key_column.name}); + ColumnPtr key_column; + RETURN_IF_ERROR(child->execute_column(context, block, selector, count, key_column)); + // Equality comparison operates on row-addressable columns. Materialize literal constants + // so nullable NULL keys and regular slot columns share the same compare_at contract. + data_key_block.insert({key_column->convert_to_full_column_if_const(), + child->execute_type(block), child->expr_name()}); } + result_column = _evaluate_key_block(data_key_block); + return Status::OK(); +} +ColumnPtr EqualityDeletePredicate::_evaluate_key_block(const Block& data_key_block) const { const auto rows = data_key_block.rows(); auto res_col = ColumnBool::create(rows, 0); if (_delete_hash_map.empty() || rows == 0) { - block->insert({std::move(res_col), std::make_shared(), expr_name()}); - *result_column_id = static_cast(block->columns() - 1); - return Status::OK(); + return res_col; } - auto data_hashes = _build_hashes(data_key_block); auto& result_data = res_col->get_data(); for (size_t row = 0; row < rows; ++row) { @@ -126,10 +150,7 @@ Status EqualityDeletePredicate::execute(VExprContext* context, Block* block, } } } - - block->insert({std::move(res_col), std::make_shared(), expr_name()}); - *result_column_id = static_cast(block->columns() - 1); - return Status::OK(); + return res_col; } std::vector EqualityDeletePredicate::_build_hashes(const Block& block) { diff --git a/be/src/format_v2/expr/equality_delete_predicate.h b/be/src/format_v2/expr/equality_delete_predicate.h index cad16ca387ccd8..0e6f127cee23c1 100644 --- a/be/src/format_v2/expr/equality_delete_predicate.h +++ b/be/src/format_v2/expr/equality_delete_predicate.h @@ -46,9 +46,7 @@ class EqualityDeletePredicate final : public VExpr { Status execute(VExprContext* context, Block* block, int* result_column_id) const override; Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, - size_t count, ColumnPtr& result_column) const override { - return Status::InternalError("Not implement EqualityDeletePredicate::execute_column_impl"); - } + size_t count, ColumnPtr& result_column) const override; Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; Status open(RuntimeState* state, VExprContext* context, FunctionContext::FunctionStateScope scope) override; @@ -59,6 +57,7 @@ class EqualityDeletePredicate final : public VExpr { private: static std::vector _build_hashes(const Block& block); + ColumnPtr _evaluate_key_block(const Block& data_key_block) const; bool _equal(const Block& data_block, size_t data_row, size_t delete_row) const; std::string _expr_name; diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 135df2820145e3..59f684121ce1e3 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -74,7 +74,8 @@ struct FileScanRequest { std::map local_positions; // Row-level filters converted to file-local expressions from table-level predicates. VExprContextSPtrs conjuncts; - // Delete predicates converted to file-local expressions. + // Delete predicates converted to file-local expressions. A TRUE result means that the row is + // deleted, so readers must invert each result when building their keep filter. VExprContextSPtrs delete_conjuncts; }; diff --git a/be/src/format_v2/materialized_reader_util.cpp b/be/src/format_v2/materialized_reader_util.cpp index a7e533633510c4..d27f066b30bc2b 100644 --- a/be/src/format_v2/materialized_reader_util.cpp +++ b/be/src/format_v2/materialized_reader_util.cpp @@ -20,7 +20,9 @@ #include #include "core/block/block.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" +#include "core/types.h" #include "exprs/vexpr_context.h" #include "format_v2/file_reader.h" #include "io/io_common.h" @@ -54,8 +56,28 @@ Status apply_materialized_reader_filters(const FileScanRequest* request, io::IOC if (request != nullptr && rows_before_filter > 0 && !request->delete_conjuncts.empty()) { { SCOPED_TIMER(profile == nullptr ? nullptr : profile->delete_conjunct_filter_time); - RETURN_IF_ERROR(VExprContext::filter_block(request->delete_conjuncts, file_block, - file_block->columns())); + // Delete conjuncts use the opposite polarity from ordinary predicates: TRUE marks a + // row for deletion. Multiple delete files are combined as a union, so a row remains + // visible only when every delete conjunct returns FALSE. + IColumn::Filter keep_filter(rows_before_filter, 1); + for (const auto& delete_conjunct : request->delete_conjuncts) { + DORIS_CHECK(delete_conjunct != nullptr); + int result_column_id = -1; + RETURN_IF_ERROR(delete_conjunct->root()->execute(delete_conjunct.get(), file_block, + &result_column_id)); + DORIS_CHECK(result_column_id >= 0 && + result_column_id < static_cast(file_block->columns())); + const auto& delete_filter = + assert_cast( + *file_block->get_by_position(result_column_id).column) + .get_data(); + DORIS_CHECK(delete_filter.size() == rows_before_filter); + for (size_t row = 0; row < rows_before_filter; ++row) { + keep_filter[row] &= !delete_filter[row]; + } + file_block->erase(result_column_id); + } + RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(file_block, keep_filter)); } rows_after_delete_filter = file_block->columns() == 0 ? rows_before_filter : file_block->rows(); diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index 1598fa783cda44..42f78d38c46bd7 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -34,6 +34,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/define_primitive_type.h" #include "core/field.h" +#include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" #include "format_v2/expr/cast.h" @@ -43,6 +44,7 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "io/file_factory.h" +#include "util/url_coding.h" namespace doris::format::iceberg { @@ -77,6 +79,54 @@ static bool is_projected_iceberg_rowid(const format::ColumnDefinition& column) { return column.name == BeConsts::ICEBERG_ROWID_COL; } +static Status build_missing_equality_delete_key_expr(const format::ColumnDefinition& table_field, + const DataTypePtr& delete_key_type, + VExprSPtr* key_expr) { + DORIS_CHECK(delete_key_type != nullptr); + DORIS_CHECK(key_expr != nullptr); + if (!table_field.initial_default_value.has_value()) { + // A newly added optional field without an initial default is logically NULL in older + // files. EqualityDeletePredicate treats NULL == NULL as a match. + *key_expr = VLiteral::create_shared(make_nullable(delete_key_type), Field()); + return Status::OK(); + } + + Field initial_default; + if (table_field.initial_default_value_is_base64 || + table_field.type->get_primitive_type() == TYPE_VARBINARY) { + // New FE versions mark every Iceberg UUID/BINARY/FIXED default as Base64 regardless of its + // Doris mapping. Keep the VARBINARY fallback for scan descriptors produced before that + // marker existed. Decode before parsing so STRING/CHAR and VARBINARY all compare against + // the raw bytes stored in equality-delete files. + std::string decoded_default; + if (!base64_decode(*table_field.initial_default_value, &decoded_default)) { + return Status::InvalidArgument("Invalid Base64 Iceberg initial default for field {}", + table_field.name); + } + if (table_field.type->get_primitive_type() == TYPE_VARBINARY) { + initial_default = Field::create_field(StringView(decoded_default)); + } else { + DORIS_CHECK(is_string_type(table_field.type->get_primitive_type())); + initial_default = Field::create_field(decoded_default); + } + } else { + // An added field's initial default is its logical value in every older data file that lacks + // the physical column. FE normalizes the string for the current Doris table type. + RETURN_IF_ERROR(table_field.type->get_serde()->from_fe_string( + *table_field.initial_default_value, initial_default)); + } + + auto literal = VLiteral::create_shared(table_field.type, initial_default); + if (table_field.type->equals(*delete_key_type)) { + *key_expr = std::move(literal); + return Status::OK(); + } + auto cast_expr = Cast::create_shared(delete_key_type); + cast_expr->add_child(std::move(literal)); + *key_expr = std::move(cast_expr); + return Status::OK(); +} + static std::string iceberg_delete_file_debug_string(const TIcebergDeleteFileDesc& delete_file) { std::ostringstream out; out << "TIcebergDeleteFileDesc{path=" << (delete_file.__isset.path ? delete_file.path : "null") @@ -143,8 +193,10 @@ static std::string iceberg_params_debug_string(const std::optional(*remove_nullable(pos_column_ptr)); for (size_t row = 0; row < read_rows; ++row) { const auto file_path = file_path_column.get_data_at(row).to_string(); - if (file_path == _data_file_path) { - _rows->push_back(pos_column.get_element(row)); - } + (*_rows_by_data_file)[file_path].push_back(pos_column.get_element(row)); } return Status::OK(); } @@ -180,6 +230,7 @@ Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options _delete_predicates_initialized = false; _position_delete_rows_storage.clear(); _equality_delete_filters.clear(); + _split_cache = options.cache; if (options.current_range.__isset.table_format_params && options.current_range.table_format_params.__isset.iceberg_params) { const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; @@ -438,31 +489,114 @@ Status IcebergTableReader::_append_row_position_output_column(format::FileScanRe return Status::OK(); } +const format::ColumnDefinition* IcebergTableReader::_find_equality_delete_data_field( + const EqualityDeleteFilter& filter, size_t key_idx) const { + DORIS_CHECK(key_idx < filter.field_ids.size()); + DORIS_CHECK(key_idx < filter.field_names.size()); + if (mapping_mode() != format::TableColumnMappingMode::BY_NAME) { + const int field_id = filter.field_ids[key_idx]; + const auto field_it = std::ranges::find_if( + _data_reader.file_schema, [field_id](const format::ColumnDefinition& field) { + return field.has_identifier_field_id() && + field.get_identifier_field_id() == field_id; + }); + return field_it == _data_reader.file_schema.end() ? nullptr : &*field_it; + } + + // Equality keys are hidden scan dependencies and need not appear in the query projection. + // Resolve their current name and aliases from the full table schema supplied by FE, falling + // back to the delete-file name when history metadata is unavailable. Reuse ColumnMapper's + // exact BY_NAME rules so case, string identifiers, and aliases on either side stay consistent. + auto table_field = _find_equality_delete_table_field(filter, key_idx); + return format::find_column_by_name(*table_field, _data_reader.file_schema); +} + +std::optional IcebergTableReader::_find_equality_delete_table_field( + const EqualityDeleteFilter& filter, size_t key_idx) const { + DORIS_CHECK(key_idx < filter.field_ids.size()); + DORIS_CHECK(key_idx < filter.field_names.size()); + const int field_id = filter.field_ids[key_idx]; + auto table_field = _find_current_table_column_by_field_id(field_id, filter.key_types[key_idx]); + if (!table_field.has_value()) { + const auto projected_field = std::ranges::find_if( + _projected_columns, [field_id](const format::ColumnDefinition& field) { + return field.has_identifier_field_id() && + field.get_identifier_field_id() == field_id; + }); + if (projected_field != _projected_columns.end()) { + // Older scan descriptors and focused unit tests may omit history_schema_info. Keep the + // projected metadata as a compatibility fallback, but never require projection when + // the complete current schema is available. + table_field = *projected_field; + } + } + if (!table_field.has_value()) { + table_field = format::ColumnDefinition { + .identifier = {}, + .name = filter.field_names[key_idx], + .type = filter.key_types[key_idx], + }; + } + return table_field; +} + +std::string IcebergTableReader::_delete_file_cache_key(const char* prefix, + const std::string& path) const { + DORIS_CHECK(prefix != nullptr); + std::string fs_name; + if (_current_task != nullptr && _current_task->data_file != nullptr) { + fs_name = _current_task->data_file->fs_name; + } + // Delete descriptors can reuse the same path text in different filesystem namespaces. Encode + // both variable-length strings so neither an fs/path boundary nor equality field-id suffixes + // can be reinterpreted as path content; scan-level credentials/properties are shared here. + std::ostringstream key; + key << prefix << fs_name.size() << ':' << fs_name << ':' << path.size() << ':' << path; + return key.str(); +} + +void IcebergTableReader::_append_equality_delete_row_count_carrier( + format::FileScanRequest* request) { + DORIS_CHECK(request != nullptr); + // Columnar readers establish a filter batch's row count from predicate columns. If all + // equality keys are missing, the predicate consists only of NULL literals and the filter block + // would otherwise have zero rows. Read one physical column eagerly as a row-count carrier; + // normal final materialization ignores this hidden dependency. + const auto carrier_it = std::ranges::find_if( + _data_reader.file_schema, [](const format::ColumnDefinition& field) { + return field.column_type == format::ColumnType::DATA_COLUMN; + }); + DORIS_CHECK(carrier_it != _data_reader.file_schema.end()); + _append_file_scan_column(request, format::LocalColumnId(carrier_it->file_local_id()), + &request->predicate_columns); +} + Status IcebergTableReader::_append_equality_delete_predicates(format::FileScanRequest* request) { DORIS_CHECK(request != nullptr); for (const auto& filter : _equality_delete_filters) { auto delete_predicate = std::make_shared(filter.delete_block, filter.field_ids); DCHECK_EQ(filter.field_ids.size(), filter.key_types.size()); + bool has_missing_key = false; for (size_t idx = 0; idx < filter.field_ids.size(); ++idx) { - const int field_id = filter.field_ids[idx]; - auto field_it = std::ranges::find_if( - _data_reader.file_schema, [field_id](const format::ColumnDefinition& field) { - return field.has_identifier_field_id() && - field.get_identifier_field_id() == field_id; - }); - if (field_it == _data_reader.file_schema.end()) { - return Status::InternalError( - "Can not find equality delete column field id {} in data file schema", - field_id); + const auto* field = _find_equality_delete_data_field(filter, idx); + if (field == nullptr) { + auto table_field = _find_equality_delete_table_field(filter, idx); + DORIS_CHECK(table_field.has_value()); + VExprSPtr key_expr; + RETURN_IF_ERROR(build_missing_equality_delete_key_expr( + *table_field, filter.key_types[idx], &key_expr)); + delete_predicate->add_child(key_expr); + has_missing_key = true; + continue; } - const auto field_column_id = format::LocalColumnId(field_it->file_local_id()); + const auto field_column_id = format::LocalColumnId(field->file_local_id()); _append_file_scan_column(request, field_column_id, &request->predicate_columns); const auto block_position = request->local_positions.at(field_column_id).value(); auto slot = VSlotRef::create_shared(cast_set(block_position), - cast_set(block_position), -1, field_it->type, - field_it->name); - if (field_it->type->equals(*filter.key_types[idx])) { + cast_set(block_position), -1, field->type, + field->name); + if (field->type->equals(*filter.key_types[idx])) { delete_predicate->add_child(std::move(slot)); } else { auto cast_expr = Cast::create_shared(filter.key_types[idx]); @@ -470,6 +604,9 @@ Status IcebergTableReader::_append_equality_delete_predicates(format::FileScanRe delete_predicate->add_child(std::move(cast_expr)); } } + if (has_missing_key && request->predicate_columns.empty()) { + _append_equality_delete_row_count_carrier(request); + } request->delete_conjuncts.push_back( VExprContext::create_shared(std::move(delete_predicate))); } @@ -498,12 +635,16 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe auto system_properties = _delete_file_system_properties(scan_params); auto file_description = _delete_file_description(delete_range); std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); + const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && + scan_params.enable_mapping_timestamp_tz; if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { *reader = std::make_unique( - system_properties, file_description, io_ctx, _scanner_profile); + system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz); } else { *reader = std::make_unique(system_properties, file_description, - io_ctx, _scanner_profile); + io_ctx, _scanner_profile, std::nullopt, + enable_mapping_timestamp_tz); } RETURN_IF_ERROR((*reader)->init(_runtime_state)); return Status::OK(); @@ -566,14 +707,50 @@ Status IcebergTableReader::_read_position_delete_file(const TIcebergDeleteFileDe Status IcebergTableReader::_init_position_delete_rows( const std::vector& delete_files) { + DORIS_CHECK(_split_cache != nullptr); TFileScanRangeParams delete_scan_params = _scan_params == nullptr ? TFileScanRangeParams() : *_scan_params; format::DeleteRows position_delete_rows; IcebergDeleteFileIOContext delete_io_ctx(_runtime_state); - PositionDeleteRowsCollector collector(_data_file_path(), &position_delete_rows); for (const auto& delete_file : delete_files) { - RETURN_IF_ERROR(_read_position_delete_file(delete_file, delete_scan_params, &delete_io_ctx, - &collector)); + Status read_status = Status::OK(); + // A position delete file normally references many data files. Cache the complete + // path-to-position map once; caching only the current data file would still rescan the + // shared delete file for every subsequent split. + auto* rows_by_data_file = + _split_cache->get( + _delete_file_cache_key("iceberg_v2_position_delete_", delete_file.path), + [&]() -> PositionDeleteRowsCollector::PositionDeleteFile* { + auto result = std::make_unique< + PositionDeleteRowsCollector::PositionDeleteFile>(); + PositionDeleteRowsCollector collector(result.get()); + read_status = _read_position_delete_file( + delete_file, delete_scan_params, &delete_io_ctx, &collector); + if (!read_status.ok()) { + return nullptr; + } + for (auto& [_, rows] : *result) { + std::ranges::sort(rows); + } + return result.release(); + }); + RETURN_IF_ERROR(read_status); + DORIS_CHECK(rows_by_data_file != nullptr); + const auto rows_it = rows_by_data_file->find(_data_file_path()); + if (rows_it == rows_by_data_file->end()) { + continue; + } + auto first = rows_it->second.begin(); + auto last = rows_it->second.end(); + // Bounds are inclusive Iceberg position statistics supplied by FE. Apply them after the + // cached per-data-file vector is sorted so irrelevant positions are sliced without a scan. + if (delete_file.__isset.position_lower_bound) { + first = std::lower_bound(first, last, delete_file.position_lower_bound); + } + if (delete_file.__isset.position_upper_bound) { + last = std::upper_bound(first, last, delete_file.position_upper_bound); + } + position_delete_rows.insert(position_delete_rows.end(), first, last); } if (position_delete_rows.empty()) { return Status::OK(); @@ -593,6 +770,7 @@ Status IcebergTableReader::_init_position_delete_rows( Status IcebergTableReader::_init_equality_delete_predicates( const std::vector& delete_files) { + DORIS_CHECK(_split_cache != nullptr); TFileScanRangeParams delete_scan_params = _scan_params == nullptr ? TFileScanRangeParams() : *_scan_params; IcebergDeleteFileIOContext delete_io_ctx(_runtime_state); @@ -603,27 +781,18 @@ Status IcebergTableReader::_init_equality_delete_predicates( return Status::OK(); } -Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, - const TFileScanRangeParams& scan_params, - IcebergDeleteFileIOContext* delete_io_ctx) { - if (!delete_file.__isset.field_ids || delete_file.field_ids.empty()) { - return Status::InternalError("Iceberg equality delete file is missing field ids"); - } - std::unique_ptr reader; - RETURN_IF_ERROR(_create_delete_file_reader(delete_file, scan_params, delete_io_ctx, &reader)); - DORIS_CHECK(reader != nullptr); - - std::vector schema; - RETURN_IF_ERROR(reader->get_schema(&schema)); - std::vector delete_fields; - std::vector delete_field_ids; - std::vector delete_key_types; +Status IcebergTableReader::_resolve_equality_delete_fields( + const TIcebergDeleteFileDesc& delete_file, + const std::vector& schema, + std::vector* delete_fields, EqualityDeleteFilter* result) const { + DORIS_CHECK(delete_fields != nullptr); + DORIS_CHECK(result != nullptr); for (const auto field_id : delete_file.field_ids) { - auto field_it = std::find_if(schema.begin(), schema.end(), - [field_id](const format::ColumnDefinition& field) { - return field.has_identifier_field_id() && - field_id == field.get_identifier_field_id(); - }); + const auto field_it = + std::ranges::find_if(schema, [field_id](const format::ColumnDefinition& field) { + return field.has_identifier_field_id() && + field_id == field.get_identifier_field_id(); + }); if (field_it == schema.end()) { return Status::InternalError("Can not find field id {} in equality delete file {}", field_id, delete_file.path); @@ -632,12 +801,36 @@ Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDe return Status::NotSupported( "Iceberg equality delete does not support complex column {}", field_it->name); } - delete_fields.push_back(*field_it); - delete_field_ids.push_back(field_id); - delete_key_types.push_back(field_it->type); + delete_fields->push_back(*field_it); + result->field_ids.push_back(field_id); + result->field_names.push_back(field_it->name); + result->key_types.push_back(field_it->type); } + return Status::OK(); +} + +Status IcebergTableReader::_load_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + EqualityDeleteFilter* result) { + DORIS_CHECK(result != nullptr); + std::unique_ptr reader; + RETURN_IF_ERROR(_create_delete_file_reader(delete_file, scan_params, delete_io_ctx, &reader)); + DORIS_CHECK(reader != nullptr); + + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + std::vector delete_fields; + RETURN_IF_ERROR(_resolve_equality_delete_fields(delete_file, schema, &delete_fields, result)); auto request = std::make_shared(); + auto build_block = [](const std::vector& fields) -> Block { + Block block; + for (const auto& field : fields) { + block.insert({field.type->create_column(), field.type, field.name}); + } + return block; + }; for (size_t idx = 0; idx < delete_fields.size(); ++idx) { const auto local_column_id = format::LocalColumnId(delete_fields[idx].file_local_id()); request->non_predicate_columns.push_back( @@ -646,19 +839,10 @@ Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDe } RETURN_IF_ERROR(reader->open(request)); - auto build_equality_delete_block = - [](const std::vector fields) -> Block { - Block block; - for (const auto& field : fields) { - block.insert({field.type->create_column(), field.type, field.name}); - } - return block; - }; - Block delete_block = build_equality_delete_block(delete_fields); - MutableBlock mutable_delete_block(std::move(delete_block)); + MutableBlock mutable_delete_block(build_block(delete_fields)); bool eof = false; while (!eof) { - Block block = build_equality_delete_block(delete_fields); + Block block = build_block(delete_fields); size_t read_rows = 0; RETURN_IF_ERROR(reader->get_block(&block, &read_rows, &eof)); if (read_rows > 0) { @@ -666,11 +850,39 @@ Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDe } } RETURN_IF_ERROR(reader->close()); - delete_block = mutable_delete_block.to_block(); - _equality_delete_filters.push_back( - EqualityDeleteFilter {.field_ids = std::move(delete_field_ids), - .key_types = std::move(delete_key_types), - .delete_block = std::move(delete_block)}); + result->delete_block = mutable_delete_block.to_block(); + return Status::OK(); +} + +Status IcebergTableReader::_read_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx) { + if (!delete_file.__isset.field_ids || delete_file.field_ids.empty()) { + return Status::InternalError("Iceberg equality delete file is missing field ids"); + } + std::ostringstream cache_key; + cache_key << _delete_file_cache_key("iceberg_v2_equality_delete_", delete_file.path); + cache_key << ':' << delete_file.field_ids.size(); + for (const auto field_id : delete_file.field_ids) { + cache_key << ':' << field_id; + } + Status read_status = Status::OK(); + // Include the ordered equality ids in the key because the same physical delete file can be + // projected with different key layouts. The cached block and its key metadata are immutable + // after construction and therefore safe to copy into each split-local predicate. + auto* cached_filter = _split_cache->get( + cache_key.str(), [&]() -> EqualityDeleteFilter* { + auto result = std::make_unique(); + read_status = _load_equality_delete_file(delete_file, scan_params, delete_io_ctx, + result.get()); + if (!read_status.ok()) { + return nullptr; + } + return result.release(); + }); + RETURN_IF_ERROR(read_status); + DORIS_CHECK(cached_filter != nullptr); + _equality_delete_filters.push_back(*cached_filter); return Status::OK(); } diff --git a/be/src/format_v2/table/iceberg_reader.h b/be/src/format_v2/table/iceberg_reader.h index 6903310bb6e34c..d328f574f2b22b 100644 --- a/be/src/format_v2/table/iceberg_reader.h +++ b/be/src/format_v2/table/iceberg_reader.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "common/status.h" @@ -74,6 +75,8 @@ class IcebergTableReader : public format::TableReader { Status _init_delete_predicates(const TTableFormatFileDesc& t_desc); private: + struct EqualityDeleteFilter; + bool _has_field_id(const std::vector& schema) const { for (const auto& field : schema) { // TopN lazy materialization asks the file reader to synthesize GLOBAL_ROWID in the @@ -110,13 +113,14 @@ class IcebergTableReader : public format::TableReader { class PositionDeleteRowsCollector final { public: - PositionDeleteRowsCollector(std::string data_file_path, format::DeleteRows* rows); + using PositionDeleteFile = std::unordered_map; + + explicit PositionDeleteRowsCollector(PositionDeleteFile* rows_by_data_file); Status collect(const Block& block, size_t read_rows); private: - std::string _data_file_path; - format::DeleteRows* _rows = nullptr; + PositionDeleteFile* _rows_by_data_file = nullptr; }; static std::shared_ptr _delete_file_system_properties( @@ -132,6 +136,12 @@ class IcebergTableReader : public format::TableReader { // Append equality delete predicates to file scan request based on the delete files in iceberg // params. DeleteVector and position delete files use the common DeleteRows path in TableReader. Status _append_equality_delete_predicates(format::FileScanRequest* request); + const format::ColumnDefinition* _find_equality_delete_data_field( + const EqualityDeleteFilter& filter, size_t key_idx) const; + std::optional _find_equality_delete_table_field( + const EqualityDeleteFilter& filter, size_t key_idx) const; + void _append_equality_delete_row_count_carrier(format::FileScanRequest* request); + std::string _delete_file_cache_key(const char* prefix, const std::string& path) const; Status _init_equality_delete_predicates( const std::vector& delete_files); @@ -144,6 +154,14 @@ class IcebergTableReader : public format::TableReader { Status _read_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, const TFileScanRangeParams& scan_params, IcebergDeleteFileIOContext* delete_io_ctx); + Status _load_equality_delete_file(const TIcebergDeleteFileDesc& delete_file, + const TFileScanRangeParams& scan_params, + IcebergDeleteFileIOContext* delete_io_ctx, + EqualityDeleteFilter* result); + Status _resolve_equality_delete_fields(const TIcebergDeleteFileDesc& delete_file, + const std::vector& schema, + std::vector* delete_fields, + EqualityDeleteFilter* result) const; Status _read_position_delete_file(const TIcebergDeleteFileDesc& delete_file, const TFileScanRangeParams& scan_params, IcebergDeleteFileIOContext* delete_io_ctx, @@ -165,10 +183,16 @@ class IcebergTableReader : public format::TableReader { format::DeleteRows _position_delete_rows_storage; struct EqualityDeleteFilter { std::vector field_ids; + // Delete-file names are retained for Iceberg tables imported from formats that did not + // persist field ids. In BY_NAME mode they are the fallback binding key. + std::vector field_names; std::vector key_types; Block delete_block; }; std::vector _equality_delete_filters; + // Scanner-shared cache supplied in SplitReadOptions. Parsed delete files outlive one data-file + // split and can be reused by every split referencing the same delete file. + ShardedKVCache* _split_cache = nullptr; bool _need_row_lineage_row_id() const; bool _need_iceberg_rowid() const; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 09c5d65e82b7fd..5fb32b84d0ab52 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -33,8 +33,10 @@ #include "common/status.h" #include "core/assert_cast.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/primitive_type.h" #include "exprs/vexpr_context.h" #include "exprs/vslot_ref.h" #include "format/table/deletion_vector_reader.h" @@ -160,8 +162,27 @@ DataTypePtr find_struct_child_type_by_external_field(const DataTypeStruct& struc return nullptr; } +DataTypePtr restore_current_primitive_type(const schema::external::TField& field, + DataTypePtr fallback_type) { + if (!field.__isset.type) { + return fallback_type; + } + const auto primitive_type = thrift_to_type(field.type.type); + if (is_complex_type(primitive_type)) { + return fallback_type; + } + // The delete file can expose an older physical type, but initial defaults belong to the + // current table field. Restore that type from FE before parsing the default and let the table + // reader apply the normal promotion cast to the delete-key type. + return DataTypeFactory::instance().create_data_type( + primitive_type, false, field.type.__isset.precision ? field.type.precision : 0, + field.type.__isset.scale ? field.type.scale : 0, + field.type.__isset.len ? field.type.len : -1); +} + ColumnDefinition build_schema_column_from_external_field(const schema::external::TField& field, DataTypePtr type) { + type = restore_current_primitive_type(field, std::move(type)); ColumnDefinition column { .identifier = field.__isset.id ? Field::create_field(field.id) : Field {}, .name = field.__isset.name ? field.name : "", @@ -170,6 +191,11 @@ ColumnDefinition build_schema_column_from_external_field(const schema::external: .type = std::move(type), .children = {}, .default_expr = nullptr, + .initial_default_value = field.__isset.initial_default_value + ? std::make_optional(field.initial_default_value) + : std::nullopt, + .initial_default_value_is_base64 = field.__isset.initial_default_value_is_base64 && + field.initial_default_value_is_base64, .is_partition_key = false, }; if (column.type == nullptr || !field.__isset.nestedField) { @@ -464,6 +490,34 @@ Status TableReader::annotate_projected_column(const TFileScanSlotInfo& slot_info return Status::OK(); } +std::optional TableReader::_find_current_table_column_by_field_id( + int32_t field_id, DataTypePtr type) const { + if (_scan_params == nullptr || !_scan_params->__isset.history_schema_info || + _scan_params->history_schema_info.empty()) { + return std::nullopt; + } + const auto* schema = &_scan_params->history_schema_info.front(); + if (_scan_params->__isset.current_schema_id) { + for (const auto& candidate_schema : _scan_params->history_schema_info) { + if (candidate_schema.__isset.schema_id && + candidate_schema.schema_id == _scan_params->current_schema_id) { + schema = &candidate_schema; + break; + } + } + } + if (!schema->__isset.root_field || !schema->root_field.__isset.fields) { + return std::nullopt; + } + for (const auto& field_ptr : schema->root_field.fields) { + const auto* field = get_field_ptr(field_ptr); + if (field != nullptr && field->__isset.id && field->id == field_id) { + return build_schema_column_from_external_field(*field, std::move(type)); + } + } + return std::nullopt; +} + Status TableReader::init(TableReadOptions&& options) { _scan_params = options.scan_params; _format = options.format; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index b8457ee9b4a31e..a875058103c145 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -154,7 +154,7 @@ struct SplitReadOptions { // filter could arrive after synthetic rows have already been returned and those rows cannot be // retracted. Standalone TableReader callers have no scanner runtime-filter lifecycle. bool all_runtime_filters_applied = true; - ShardedKVCache* cache; + ShardedKVCache* cache = nullptr; TFileRangeDesc current_range; FileFormat current_split_format = FileFormat::PARQUET; std::optional global_rowid_context; @@ -318,6 +318,9 @@ class TableReader { } protected: + std::optional _find_current_table_column_by_field_id(int32_t field_id, + DataTypePtr type) const; + // Parse deletion vector information from table format specific file description. virtual Status _parse_deletion_vector_file(const TTableFormatFileDesc& t_desc, DeleteFileDesc* desc, bool* has_delete_file) { diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp b/be/test/format/table/iceberg/iceberg_reader_test.cpp index 89bf367d06b053..6c504e0b49a46b 100644 --- a/be/test/format/table/iceberg/iceberg_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp @@ -17,15 +17,22 @@ #include "format/table/iceberg_reader.h" +#include +#include #include #include #include #include #include #include +#include +#include +#include +#include #include #include +#include #include #include #include @@ -40,13 +47,16 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_varbinary.h" #include "format/column_descriptor.h" #include "format/format_common.h" +#include "format/orc/orc_memory_stream_test.h" #include "format/orc/vorc_reader.h" #include "format/parquet/vparquet_column_chunk_reader.h" #include "format/parquet/vparquet_reader.h" @@ -62,6 +72,200 @@ namespace doris { +void write_iceberg_int_orc_file(const std::string& file_path, const std::string& field_name, + int32_t field_id, const std::vector& values) { + auto type = std::unique_ptr<::orc::Type>( + ::orc::Type::buildTypeFromString("struct<" + field_name + ":int>")); + type->getSubtype(0)->setAttribute("iceberg.id", std::to_string(field_id)); + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(values.size()); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + auto& value_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[0]); + for (size_t row = 0; row < values.size(); ++row) { + value_batch.data[row] = values[row]; + } + struct_batch.numElements = values.size(); + value_batch.numElements = values.size(); + writer->add(*batch); + writer->close(); + + std::ofstream output(file_path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +void write_iceberg_three_int_orc_file( + const std::string& file_path, const std::string& first_name, + std::optional first_id, const std::vector& first_values, + const std::string& second_name, std::optional second_id, + const std::vector& second_values, const std::string& third_name, + std::optional third_id, const std::vector& third_values) { + DORIS_CHECK(first_values.size() == second_values.size()); + DORIS_CHECK(first_values.size() == third_values.size()); + auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( + "struct<" + first_name + ":int," + second_name + ":int," + third_name + ":int>")); + const std::array field_ids = {first_id, second_id, third_id}; + for (size_t idx = 0; idx < field_ids.size(); ++idx) { + if (field_ids[idx].has_value()) { + type->getSubtype(idx)->setAttribute("iceberg.id", std::to_string(*field_ids[idx])); + } + } + + MemoryOutputStream memory_stream(1024 * 1024); + ::orc::WriterOptions options; + options.setCompression(::orc::CompressionKind_NONE); + options.setMemoryPool(::orc::getDefaultPool()); + auto writer = ::orc::createWriter(*type, &memory_stream, options); + auto batch = writer->createRowBatch(first_values.size()); + auto& struct_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch); + const std::array value_sets = {&first_values, &second_values, &third_values}; + for (size_t field_idx = 0; field_idx < value_sets.size(); ++field_idx) { + auto& value_batch = dynamic_cast<::orc::LongVectorBatch&>(*struct_batch.fields[field_idx]); + for (size_t row = 0; row < first_values.size(); ++row) { + value_batch.data[row] = (*value_sets[field_idx])[row]; + } + value_batch.numElements = first_values.size(); + } + struct_batch.numElements = first_values.size(); + writer->add(*batch); + writer->close(); + + std::ofstream output(file_path, std::ios::binary); + output.write(memory_stream.getData(), static_cast(memory_stream.getLength())); +} + +std::shared_ptr build_iceberg_int32_array(const std::vector& values) { + arrow::Int32Builder builder; + for (const auto value : values) { + DORIS_CHECK(builder.Append(value).ok()); + } + std::shared_ptr result; + DORIS_CHECK(builder.Finish(&result).ok()); + return result; +} + +void write_iceberg_three_int_parquet_file( + const std::string& file_path, const std::string& first_name, + std::optional first_id, const std::vector& first_values, + const std::string& second_name, std::optional second_id, + const std::vector& second_values, const std::string& third_name, + std::optional third_id, const std::vector& third_values) { + DORIS_CHECK(first_values.size() == second_values.size()); + DORIS_CHECK(first_values.size() == third_values.size()); + std::vector> fields = { + arrow::field(first_name, arrow::int32(), false), + arrow::field(second_name, arrow::int32(), false), + arrow::field(third_name, arrow::int32(), false)}; + const std::array field_ids = {first_id, second_id, third_id}; + for (size_t idx = 0; idx < fields.size(); ++idx) { + if (field_ids[idx].has_value()) { + fields[idx] = fields[idx]->WithMetadata(arrow::key_value_metadata( + {"PARQUET:field_id"}, {std::to_string(*field_ids[idx])})); + } + } + auto schema = arrow::schema(fields); + auto table = arrow::Table::Make(schema, {build_iceberg_int32_array(first_values), + build_iceberg_int32_array(second_values), + build_iceberg_int32_array(third_values)}); + + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + static_cast(first_values.size()), + properties.build())); +} + +void write_iceberg_int_equality_delete_parquet_file(const std::string& file_path, + const std::string& field_name, int32_t field_id, + int32_t value) { + auto field = arrow::field(field_name, arrow::int32(), false) + ->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, + {std::to_string(field_id)})); + auto table = arrow::Table::Make(arrow::schema({field}), {build_iceberg_int32_array({value})}); + auto output = arrow::io::FileOutputStream::Open(file_path); + DORIS_CHECK(output.ok()); + ::parquet::WriterProperties::Builder properties; + properties.version(::parquet::ParquetVersion::PARQUET_2_6); + properties.data_page_version(::parquet::ParquetDataPageVersion::V2); + properties.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *output, + 1, properties.build())); +} + +Status create_single_int_tuple_descriptor(ObjectPool* object_pool, const std::string& column_name, + int32_t field_id, DescriptorTbl** descriptor_table, + const TupleDescriptor** tuple_descriptor) { + TDescriptorTable thrift_table; + TTableDescriptor table_descriptor; + table_descriptor.__set_id(0); + table_descriptor.__set_tableType(TTableType::OLAP_TABLE); + table_descriptor.__set_numCols(1); + table_descriptor.__set_numClusteringCols(0); + thrift_table.__set_tableDescriptors({table_descriptor}); + + TTypeNode type_node; + type_node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::INT); + type_node.__set_scalar_type(scalar_type); + TTypeDesc type; + type.__set_types({type_node}); + + TSlotDescriptor slot_descriptor; + slot_descriptor.__set_id(0); + slot_descriptor.__set_parent(0); + slot_descriptor.__set_col_unique_id(field_id); + slot_descriptor.__set_slotType(type); + slot_descriptor.__set_columnPos(0); + slot_descriptor.__set_byteOffset(0); + slot_descriptor.__set_nullIndicatorByte(0); + slot_descriptor.__set_nullIndicatorBit(-1); + slot_descriptor.__set_colName(column_name); + slot_descriptor.__set_slotIdx(0); + slot_descriptor.__set_isMaterialized(true); + thrift_table.__set_slotDescriptors({slot_descriptor}); + + TTupleDescriptor thrift_tuple; + thrift_tuple.__set_id(0); + thrift_tuple.__set_byteSize(16); + thrift_tuple.__set_numNullBytes(0); + thrift_tuple.__set_tableId(0); + thrift_table.__set_tupleDescriptors({thrift_tuple}); + + RETURN_IF_ERROR(DescriptorTbl::create(object_pool, thrift_table, descriptor_table)); + *tuple_descriptor = (*descriptor_table)->get_tuple_descriptor(0); + return Status::OK(); +} + +schema::external::TFieldPtr make_external_int_field(const std::string& name, int32_t field_id, + std::optional initial_default, + std::vector aliases = {}) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(field_id); + TColumnType type; + type.__set_type(TPrimitiveType::INT); + field->__set_type(type); + if (initial_default.has_value()) { + field->__set_initial_default_value(*initial_default); + } + if (!aliases.empty()) { + field->__set_name_mapping(aliases); + } + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; +} + class IcebergReaderTestHelper : public IcebergTableReader { public: using IcebergTableReader::_is_fully_dictionary_encoded; @@ -845,6 +1049,443 @@ TEST_F(IcebergReaderTest, v1_position_delete_read_error_releases_cache_entry) { EXPECT_NE(status.to_string().find(delete_file.path), std::string::npos); } +TEST_F(IcebergReaderTest, v1_materializes_missing_equality_delete_initial_defaults) { + auto make_field = [](const std::string& name, int32_t id, TPrimitiveType::type primitive_type, + const std::string& default_value, int32_t len, int32_t scale, + bool default_is_base64) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + field->__set_initial_default_value(default_value); + if (default_is_base64) { + field->__set_initial_default_value_is_base64(true); + } + TColumnType type; + type.__set_type(primitive_type); + if (len >= 0) { + type.__set_len(len); + } + if (scale >= 0) { + type.__set_scale(scale); + } + field->__set_type(type); + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; + }; + + schema::external::TStructField root_field; + root_field.__set_fields( + {make_field("added_timestamp", 1, TPrimitiveType::DATETIMEV2, + "2024-01-01 00:00:00.123456", -1, 6, false), + make_field("added_binary", 2, TPrimitiveType::VARBINARY, "AAEC/w==", 4, -1, true), + make_field("added_string_binary", 3, TPrimitiveType::STRING, "AAEC/w==", -1, -1, + true)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({std::move(current_schema)}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state = RuntimeState(TQueryOptions(), TQueryGlobals()); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + + const auto timestamp_type = make_nullable(std::make_shared(6)); + const auto varbinary_type = make_nullable(std::make_shared(4)); + const auto string_type = make_nullable(std::make_shared()); + Block block; + block.insert({timestamp_type->create_column(), timestamp_type, "added_timestamp"}); + block.insert({varbinary_type->create_column(), varbinary_type, "added_binary"}); + block.insert({string_type->create_column(), string_type, "added_string_binary"}); + std::unordered_map positions = { + {"added_timestamp", 0}, {"added_binary", 1}, {"added_string_binary", 2}}; + reader.TEST_set_column_name_to_block_index(&positions); + ASSERT_TRUE(reader.TEST_register_missing_equality_delete_column(1, "added_timestamp", + timestamp_type) + .ok()); + ASSERT_TRUE( + reader.TEST_register_missing_equality_delete_column(2, "added_binary", varbinary_type) + .ok()); + ASSERT_TRUE(reader.TEST_register_missing_equality_delete_column(3, "added_string_binary", + string_type) + .ok()); + ASSERT_TRUE(reader.TEST_materialize_missing_equality_delete_columns(&block, 3).ok()); + + ASSERT_EQ(block.rows(), 3); + EXPECT_EQ(timestamp_type->to_string(*block.get_by_position(0).column, 0), + "2024-01-01 00:00:00.123456"); + EXPECT_EQ(varbinary_type->to_string(*block.get_by_position(1).column, 0), + std::string("\x00\x01\x02\xff", 4)); + EXPECT_EQ(string_type->to_string(*block.get_by_position(2).column, 0), + std::string("\x00\x01\x02\xff", 4)); + EXPECT_FALSE(is_column_const(*block.get_by_position(0).column)); + EXPECT_FALSE(is_column_const(*block.get_by_position(1).column)); + EXPECT_FALSE(is_column_const(*block.get_by_position(2).column)); +} + +TEST_F(IcebergReaderTest, v1_multi_equality_delete_hashes_materialized_missing_default) { + schema::external::TStructField root_field; + root_field.__set_fields({make_external_int_field("id", 0, std::nullopt), + make_external_int_field("added_column", 1, "7")}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + + const auto int_type = make_nullable(std::make_shared()); + auto id_column = int_type->create_column(); + id_column->insert(Field::create_field(1)); + id_column->insert(Field::create_field(2)); + id_column->insert(Field::create_field(3)); + Block data_block; + data_block.insert({std::move(id_column), int_type, "id"}); + data_block.insert({int_type->create_column(), int_type, "added_column"}); + std::unordered_map positions = {{"id", 0}, {"added_column", 1}}; + reader.TEST_set_column_name_to_block_index(&positions); + ASSERT_TRUE( + reader.TEST_register_missing_equality_delete_column(1, "added_column", int_type).ok()); + ASSERT_TRUE(reader.TEST_materialize_missing_equality_delete_columns(&data_block, 3).ok()); + ASSERT_EQ(data_block.rows(), 3); + ASSERT_FALSE(is_column_const(*data_block.get_by_position(1).column)); + + auto delete_id_column = int_type->create_column(); + delete_id_column->insert(Field::create_field(2)); + auto delete_default_column = int_type->create_column(); + delete_default_column->insert(Field::create_field(7)); + Block delete_block; + delete_block.insert({std::move(delete_id_column), int_type, "id"}); + delete_block.insert({std::move(delete_default_column), int_type, "added_column"}); + MultiEqualityDelete equality_delete(&delete_block, {0, 1}); + ASSERT_TRUE(equality_delete.init(&profile).ok()); + + std::unordered_map id_to_name = {{0, "id"}, {1, "added_column"}}; + IColumn::Filter filter(3, 1); + ASSERT_TRUE( + equality_delete.filter_data_block(&data_block, &positions, id_to_name, filter).ok()); + EXPECT_EQ(filter, IColumn::Filter({1, 0, 1})); +} + +TEST_F(IcebergReaderTest, v1_missing_equality_key_returns_error_for_pruned_descriptor) { + schema::external::TStructField root_field; + root_field.__set_fields({make_external_int_field("id", 0, std::nullopt)}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + TFileRangeDesc scan_range; + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + + const auto int_type = make_nullable(std::make_shared()); + const auto status = + reader.TEST_register_missing_equality_delete_column(1, "hidden_key", int_type); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("field id 1"), std::string::npos); +} + +TEST_F(IcebergReaderTest, v1_orc_equality_delete_matches_missing_initial_default) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_orc_missing_equality_delete_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.orc").string(); + const auto delete_file = (test_dir / "equality-delete.orc").string(); + write_iceberg_int_orc_file(data_file, "id", 0, {1, 2, 3}); + write_iceberg_int_orc_file(delete_file, "added_column", 1, {7}); + + schema::external::TStructField root_field; + root_field.__set_fields({make_external_int_field("id", 0, std::nullopt), + make_external_int_field("added_column", 1, "7")}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + + TIcebergDeleteFileDesc delete_descriptor; + delete_descriptor.__set_content(IcebergReaderMixin::EQUALITY_DELETE); + delete_descriptor.__set_path(delete_file); + delete_descriptor.__set_field_ids({1}); + delete_descriptor.__set_file_format(TFileFormatType::FORMAT_ORC); + TIcebergFileDesc iceberg_descriptor; + iceberg_descriptor.__set_format_version(2); + iceberg_descriptor.__set_original_file_path(data_file); + iceberg_descriptor.__set_delete_files({delete_descriptor}); + TTableFormatFileDesc table_format_descriptor; + table_format_descriptor.__set_iceberg_params(iceberg_descriptor); + + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_table_format_params(table_format_descriptor); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_int_tuple_descriptor(&object_pool, "id", 0, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergOrcReader reader(&kv_cache, &profile, &runtime_state, scan_params, scan_range, 1024, + "UTC", &io_ctx, cache.get()); + + std::vector column_descriptors(1); + column_descriptors[0].name = "id"; + std::unordered_map block_positions = {{"id", 0}}; + OrcInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto id_type = make_nullable(std::make_shared()); + Block block; + block.insert({id_type->create_column(), id_type, "id"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(read_rows, 0); + EXPECT_EQ(block.rows(), 0); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_parquet_partial_id_equality_delete_ignores_stale_field_id) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_v1_parquet_partial_id_equality_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.parquet").string(); + const auto delete_file = (test_dir / "equality-delete.parquet").string(); + // The id-less columns put the complete Parquet file in BY_NAME mode. The unrelated + // stale_added column deliberately retains field id 1, which must not override the historical + // alias selected for the hidden added_column equality key. + write_iceberg_three_int_parquet_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", + std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); + write_iceberg_int_equality_delete_parquet_file(delete_file, "added_column", 1, 7); + + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 0, std::nullopt), + make_external_int_field("added_column", 1, std::nullopt, {"legacy_added"})}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + + TIcebergDeleteFileDesc delete_descriptor; + delete_descriptor.__set_content(IcebergReaderMixin::EQUALITY_DELETE); + delete_descriptor.__set_path(delete_file); + delete_descriptor.__set_field_ids({1}); + delete_descriptor.__set_file_format(TFileFormatType::FORMAT_PARQUET); + TIcebergFileDesc iceberg_descriptor; + iceberg_descriptor.__set_format_version(2); + iceberg_descriptor.__set_original_file_path(data_file); + iceberg_descriptor.__set_delete_files({delete_descriptor}); + TTableFormatFileDesc table_format_descriptor; + table_format_descriptor.__set_iceberg_params(iceberg_descriptor); + + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_table_format_params(table_format_descriptor); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_int_tuple_descriptor(&object_pool, "id", 0, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + cctz::time_zone ctz; + TimezoneUtils::find_cctz_time_zone("UTC", ctz); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergParquetReader reader(&kv_cache, &profile, scan_params, scan_range, 1024, &ctz, &io_ctx, + &runtime_state, cache.get()); + io::FileReaderSPtr file_reader; + ASSERT_TRUE(io::global_local_filesystem()->open_file(data_file, &file_reader).ok()); + reader.set_file_reader(file_reader); + + std::vector column_descriptors(1); + column_descriptors[0].name = "id"; + std::unordered_map block_positions = {{"id", 0}}; + ParquetInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto id_type = make_nullable(std::make_shared()); + Block block; + block.insert({id_type->create_column(), id_type, "id"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 2); + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 0), "1"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "3"); + + std::filesystem::remove_all(test_dir); +} + +TEST_F(IcebergReaderTest, v1_orc_partial_id_equality_delete_ignores_stale_field_id) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_v1_orc_partial_id_equality_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + const auto data_file = (test_dir / "data.orc").string(); + const auto delete_file = (test_dir / "equality-delete.orc").string(); + // One id-less column switches the whole file to BY_NAME. The hidden current key is physically + // stored as its id-less historical alias, while an unrelated migrated column still carries + // the key's stale field id. Equality-delete binding must select legacy_added, not stale_added. + write_iceberg_three_int_orc_file(data_file, "id", std::nullopt, {1, 2, 3}, "legacy_added", + std::nullopt, {5, 7, 9}, "stale_added", 1, {70, 70, 70}); + write_iceberg_int_orc_file(delete_file, "added_column", 1, {7}); + + schema::external::TStructField root_field; + root_field.__set_fields( + {make_external_int_field("id", 0, std::nullopt), + make_external_int_field("added_column", 1, std::nullopt, {"legacy_added"})}); + schema::external::TSchema current_schema; + current_schema.__set_schema_id(-1); + current_schema.__set_root_field(root_field); + + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + scan_params.__set_format_type(TFileFormatType::FORMAT_ORC); + scan_params.__set_current_schema_id(-1); + scan_params.__set_history_schema_info({current_schema}); + + TIcebergDeleteFileDesc delete_descriptor; + delete_descriptor.__set_content(IcebergReaderMixin::EQUALITY_DELETE); + delete_descriptor.__set_path(delete_file); + delete_descriptor.__set_field_ids({1}); + delete_descriptor.__set_file_format(TFileFormatType::FORMAT_ORC); + TIcebergFileDesc iceberg_descriptor; + iceberg_descriptor.__set_format_version(2); + iceberg_descriptor.__set_original_file_path(data_file); + iceberg_descriptor.__set_delete_files({delete_descriptor}); + TTableFormatFileDesc table_format_descriptor; + table_format_descriptor.__set_iceberg_params(iceberg_descriptor); + + TFileRangeDesc scan_range; + scan_range.__set_fs_name(""); + scan_range.__set_path(data_file); + scan_range.__set_start_offset(0); + scan_range.__set_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_file_size(static_cast(std::filesystem::file_size(data_file))); + scan_range.__set_table_format_params(table_format_descriptor); + + ObjectPool object_pool; + DescriptorTbl* descriptor_table = nullptr; + const TupleDescriptor* tuple_descriptor = nullptr; + ASSERT_TRUE(create_single_int_tuple_descriptor(&object_pool, "id", 0, &descriptor_table, + &tuple_descriptor) + .ok()); + ASSERT_NE(tuple_descriptor, nullptr); + + RuntimeProfile profile("test_profile"); + RuntimeState runtime_state {TQueryOptions(), TQueryGlobals()}; + runtime_state.set_timezone("UTC"); + io::IOContext io_ctx; + ShardedKVCache kv_cache(8); + IcebergOrcReader reader(&kv_cache, &profile, &runtime_state, scan_params, scan_range, 1024, + "UTC", &io_ctx, cache.get()); + + std::vector column_descriptors(1); + column_descriptors[0].name = "id"; + std::unordered_map block_positions = {{"id", 0}}; + OrcInitContext context; + context.column_descs = &column_descriptors; + context.col_name_to_block_idx = &block_positions; + context.tuple_descriptor = tuple_descriptor; + context.params = &scan_params; + context.range = &scan_range; + const auto init_status = reader.init_reader(&context); + ASSERT_TRUE(init_status.ok()) << init_status; + + const auto id_type = make_nullable(std::make_shared()); + Block block; + block.insert({id_type->create_column(), id_type, "id"}); + size_t read_rows = 0; + bool eof = false; + const auto status = reader.get_next_block(&block, &read_rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(read_rows, 2); + ASSERT_EQ(block.rows(), 2); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 0), "1"); + EXPECT_EQ(id_type->to_string(*block.get_by_position(0).column, 1), "3"); + + std::filesystem::remove_all(test_dir); +} + // Test reading real Iceberg Orc file using IcebergTableReader TEST_F(IcebergReaderTest, read_iceberg_orc_file) { // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email diff --git a/be/test/format_v2/expr/equality_delete_predicate_test.cpp b/be/test/format_v2/expr/equality_delete_predicate_test.cpp index 886a86713fe8da..0f63e915f28a53 100644 --- a/be/test/format_v2/expr/equality_delete_predicate_test.cpp +++ b/be/test/format_v2/expr/equality_delete_predicate_test.cpp @@ -112,6 +112,27 @@ TEST_F(EqualityDeletePredicateTest, MatchSingleColumn) { EXPECT_EQ(result_column_data(data_block, result_column_id), std::vector({1, 0, 0, 1})); } +TEST_F(EqualityDeletePredicateTest, UsesPopulatedPredicateColumnForLazyBatchSize) { + Block delete_block; + delete_block.insert(make_nullable_int_column("id", {1, 4})); + Block data_block; + // A lazy reader can leave the first projected column unread while decoding a later predicate + // column. Block::rows() is therefore zero even though the equality key has four rows. + data_block.insert(make_nullable_string_column("unread", {})); + data_block.insert(make_nullable_int_column("id", {1, 2, 3, 4})); + + auto predicate = std::make_shared(std::move(delete_block), + std::vector {1}); + predicate->_open_finished = true; + predicate->add_child(std::make_shared(1, data_block.get_by_position(1).type)); + VExprContext context(predicate); + + int result_column_id = -1; + auto status = predicate->execute(&context, &data_block, &result_column_id); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(result_column_data(data_block, result_column_id), std::vector({1, 0, 0, 1})); +} + TEST_F(EqualityDeletePredicateTest, MatchMultipleColumns) { Block delete_block; delete_block.insert(make_nullable_int_column("id", {1, 2})); diff --git a/be/test/format_v2/json/json_reader_test.cpp b/be/test/format_v2/json/json_reader_test.cpp index 31c77501ce67c6..683a47dee299cc 100644 --- a/be/test/format_v2/json/json_reader_test.cpp +++ b/be/test/format_v2/json/json_reader_test.cpp @@ -590,9 +590,9 @@ TEST(JsonReaderTest, AppliesDeleteAndNormalConjunctsWithPredicateFilterAccountin request->local_positions.emplace(LocalColumnId(0), LocalIndex(0)); request->local_positions.emplace(LocalColumnId(1), LocalIndex(1)); request->delete_conjuncts = { - prepared_conjunct(&state, std::make_shared(0, 1))}; - request->conjuncts = { prepared_conjunct(&state, std::make_shared(0, 2))}; + request->conjuncts = { + prepared_conjunct(&state, std::make_shared(0, 1))}; ASSERT_TRUE(reader.open(request).ok()); auto block = make_block(schema, {0, 1}); @@ -600,8 +600,8 @@ TEST(JsonReaderTest, AppliesDeleteAndNormalConjunctsWithPredicateFilterAccountin bool eof = false; ASSERT_TRUE(reader.get_block(&block, &rows, &eof).ok()); ASSERT_EQ(rows, 1); - EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 3); - EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "carol"); + EXPECT_EQ(nullable_int_at(*block.get_by_position(0).column, 0), 2); + EXPECT_EQ(nullable_string_at(*block.get_by_position(1).column, 0), "bob"); EXPECT_EQ(io_ctx->predicate_filtered_rows, 1); } diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 8a10fc06a41db3..597816cab51126 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -46,11 +46,14 @@ #include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_varbinary.h" #include "exec/common/endian.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vectorized_fn_call.h" @@ -61,6 +64,7 @@ #include "format/table/deletion_vector_reader.h" #include "format_v2/table_reader.h" #include "gen_cpp/Exprs_types.h" +#include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" #include "io/io_common.h" #include "roaring/roaring64map.hh" @@ -281,12 +285,61 @@ std::shared_ptr build_string_array(const std::vector& return finish_array(&builder); } +schema::external::TFieldPtr external_schema_field( + std::string name, int32_t id, std::vector aliases = {}, + std::optional initial_default = std::nullopt, + std::optional type = std::nullopt, bool initial_default_is_base64 = false) { + auto field = std::make_shared(); + field->__set_name(name); + field->__set_id(id); + if (!aliases.empty()) { + field->__set_name_mapping(aliases); + } + if (initial_default.has_value()) { + field->__set_initial_default_value(*initial_default); + if (initial_default_is_base64) { + field->__set_initial_default_value_is_base64(true); + } + } + if (type.has_value()) { + field->__set_type(*type); + } + schema::external::TFieldPtr field_ptr; + field_ptr.field_ptr = std::move(field); + field_ptr.__isset.field_ptr = true; + return field_ptr; +} + +TColumnType external_primitive_type(TPrimitiveType::type type, int32_t len = -1, + int32_t scale = -1) { + TColumnType result; + result.__set_type(type); + if (len >= 0) { + result.__set_len(len); + } + if (scale >= 0) { + result.__set_scale(scale); + } + return result; +} + +schema::external::TSchema external_schema(int64_t schema_id, + std::vector fields) { + schema::external::TStructField root_field; + root_field.__set_fields(fields); + schema::external::TSchema schema; + schema.__set_schema_id(schema_id); + schema.__set_root_field(root_field); + return schema; +} + void write_iceberg_equality_delete_parquet_file(const std::string& file_path, int32_t field_id, - int32_t value) { + int32_t value, + const std::string& field_name = "id") { const auto metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), false)->WithMetadata(metadata), + arrow::field(field_name, arrow::int32(), false)->WithMetadata(metadata), }); auto table = arrow::Table::Make(schema, {build_int32_array({value})}); @@ -302,6 +355,168 @@ void write_iceberg_equality_delete_parquet_file(const std::string& file_path, in builder.build())); } +void write_iceberg_timestamp_equality_delete_parquet_file(const std::string& file_path, + int32_t field_id, int64_t value, + const std::string& field_name, + bool adjusted_to_utc = false) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + const auto timestamp_type = adjusted_to_utc ? arrow::timestamp(arrow::TimeUnit::MICRO, "UTC") + : arrow::timestamp(arrow::TimeUnit::MICRO); + arrow::TimestampBuilder value_builder(timestamp_type, arrow::default_memory_pool()); + ASSERT_TRUE(value_builder.Append(value).ok()); + auto value_result = value_builder.Finish(); + ASSERT_TRUE(value_result.ok()) << value_result.status(); + auto schema = arrow::schema({ + arrow::field(field_name, timestamp_type, false)->WithMetadata(metadata), + }); + auto table = arrow::Table::Make(schema, {*value_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_iceberg_binary_equality_delete_parquet_file(const std::string& file_path, + int32_t field_id, const std::string& value, + const std::string& field_name) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + arrow::BinaryBuilder value_builder; + ASSERT_TRUE(value_builder.Append(value).ok()); + auto value_result = value_builder.Finish(); + ASSERT_TRUE(value_result.ok()) << value_result.status(); + auto schema = arrow::schema({ + arrow::field(field_name, arrow::binary(), false)->WithMetadata(metadata), + }); + auto table = arrow::Table::Make(schema, {*value_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_iceberg_null_equality_delete_parquet_file(const std::string& file_path, int32_t field_id, + const std::string& field_name) { + const auto metadata = + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(field_id)}); + auto schema = arrow::schema({ + arrow::field(field_name, arrow::int32(), true)->WithMetadata(metadata), + }); + arrow::Int32Builder value_builder; + ASSERT_TRUE(value_builder.AppendNull().ok()); + auto value_result = value_builder.Finish(); + ASSERT_TRUE(value_result.ok()) << value_result.status(); + auto table = arrow::Table::Make(schema, {*value_result}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + +void write_single_int_parquet_file(const std::string& file_path, const std::string& field_name, + const std::vector& values, + std::optional field_id) { + auto field = arrow::field(field_name, arrow::int32(), false); + if (field_id.has_value()) { + field = field->WithMetadata( + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(*field_id)})); + } + auto table = arrow::Table::Make(arrow::schema({field}), {build_int32_array(values)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(values.size()), + builder.build())); +} + +void write_two_int_parquet_file(const std::string& file_path, const std::string& first_name, + const std::vector& first_values, + std::optional first_field_id, + const std::string& second_name, + const std::vector& second_values, + std::optional second_field_id) { + ASSERT_EQ(first_values.size(), second_values.size()); + auto first_field = arrow::field(first_name, arrow::int32(), false); + if (first_field_id.has_value()) { + first_field = first_field->WithMetadata( + arrow::key_value_metadata({"PARQUET:field_id"}, {std::to_string(*first_field_id)})); + } + auto second_field = arrow::field(second_name, arrow::int32(), false); + if (second_field_id.has_value()) { + second_field = second_field->WithMetadata(arrow::key_value_metadata( + {"PARQUET:field_id"}, {std::to_string(*second_field_id)})); + } + auto schema = arrow::schema({first_field, second_field}); + auto table = arrow::Table::Make( + schema, {build_int32_array(first_values), build_int32_array(second_values)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(first_values.size()), + builder.build())); +} + +void write_timestamp_int_parquet_file(const std::string& file_path, + const std::vector& timestamps, + const std::vector& ids) { + ASSERT_EQ(timestamps.size(), ids.size()); + const auto timestamp_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"0"}); + const auto id_metadata = arrow::key_value_metadata({"PARQUET:field_id"}, {"1"}); + const auto timestamp_type = arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); + arrow::TimestampBuilder timestamp_builder(timestamp_type, arrow::default_memory_pool()); + ASSERT_TRUE(timestamp_builder.AppendValues(timestamps).ok()); + auto timestamp_result = timestamp_builder.Finish(); + ASSERT_TRUE(timestamp_result.ok()) << timestamp_result.status(); + auto schema = arrow::schema( + {arrow::field("event_time", timestamp_type, false)->WithMetadata(timestamp_metadata), + arrow::field("id", arrow::int32(), false)->WithMetadata(id_metadata)}); + auto table = arrow::Table::Make(schema, {*timestamp_result, build_int32_array(ids)}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + static_cast(timestamps.size()), + builder.build())); +} + void write_iceberg_equality_delete_bigint_parquet_file(const std::string& file_path, int32_t field_id, int64_t value) { const auto metadata = @@ -597,6 +812,7 @@ void expect_int32_column_values(const IColumn& column, SplitReadOptions build_split_options(const std::string& file_path) { SplitReadOptions options; + EXPECT_EQ(options.cache, nullptr); options.current_range.__set_path(file_path); options.current_range.__set_file_size( static_cast(std::filesystem::file_size(file_path))); @@ -709,6 +925,23 @@ std::vector read_iceberg_ids(doris::format::iceberg::IcebergTableReader return ids; } +void init_iceberg_reader(doris::format::iceberg::IcebergTableReader* reader, + const std::vector& projected_columns, + TFileScanRangeParams* scan_params, + const std::shared_ptr& io_ctx, RuntimeState* state, + RuntimeProfile* profile) { + ASSERT_TRUE(reader->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = scan_params, + .io_ctx = io_ctx, + .runtime_state = state, + .scanner_profile = profile, + }) + .ok()); +} + DataTypePtr make_table_test_type(const DataTypePtr& type, bool nullable_root = true) { DORIS_CHECK(type != nullptr); const auto nested_type = remove_nullable(type); @@ -1870,17 +2103,16 @@ TEST(IcebergV2ReaderTest, IcebergEqualityDeleteCastsDataColumnToDeleteKeyType) { std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergPositionDeleteOnlyMatchesOriginalDataFilePath) { +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesNullForMissingDataColumn) { const auto test_dir = std::filesystem::temp_directory_path() / - "doris_iceberg_position_delete_path_match_test"; + "doris_iceberg_equality_delete_missing_column_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto file_path = (test_dir / "split.parquet").string(); - const auto other_file_path = (test_dir / "other.parquet").string(); - const auto delete_file_path = (test_dir / "position-delete.parquet").string(); - write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); - write_position_delete_parquet_file(delete_file_path, {other_file_path, file_path}, {0, 1}); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + write_iceberg_null_equality_delete_parquet_file(delete_file_path, 1, "added_column"); std::vector projected_columns; projected_columns.push_back(make_table_column(0, "id", std::make_shared())); @@ -1893,126 +2125,772 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteOnlyMatchesOriginalDataFilePath) auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); ShardedKVCache cache(1); doris::format::iceberg::IcebergTableReader reader; - ASSERT_TRUE(reader.init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = &scan_params, - .io_ctx = io_ctx, - .runtime_state = &state, - .scanner_profile = &profile, - }) - .ok()); + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); auto split_options = build_split_options(file_path); split_options.cache = &cache; split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( - file_path, {make_iceberg_position_delete_file(delete_file_path)})); + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); - EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + // Iceberg treats a field missing from an older data file as NULL. The NULL equality-delete + // key therefore matches every row in this file. + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergRowLineageRemainsFileLocalAfterDeleteFiltering) { - const auto test_dir = - std::filesystem::temp_directory_path() / "doris_iceberg_row_lineage_delete_test"; +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesInitialDefaultForMissingDataColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_default_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto file_path = (test_dir / "split.parquet").string(); - const auto delete_file_path = (test_dir / "position-delete.parquet").string(); - write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); - write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 1, 7, "added_column"); std::vector projected_columns; - projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); projected_columns.push_back(make_table_column(0, "id", std::make_shared())); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info( + {external_schema(100, {external_schema_field("id", 0), + external_schema_field("added_column", 1, {}, "7")})}); io::FileReaderStats file_reader_stats; io::FileCacheStatistics file_cache_stats; auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); ShardedKVCache cache(1); doris::format::iceberg::IcebergTableReader reader; - ASSERT_TRUE(reader.init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = &scan_params, - .io_ctx = io_ctx, - .runtime_state = &state, - .scanner_profile = &profile, - }) - .ok()); + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); auto split_options = build_split_options(file_path); split_options.cache = &cache; - TTableFormatFileDesc table_format_params = make_iceberg_table_format_desc( - file_path, {make_iceberg_position_delete_file(delete_file_path)}); - table_format_params.iceberg_params.__set_first_row_id(1000); - split_options.current_range.__set_table_format_params(table_format_params); + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); - Block block = build_table_block(projected_columns); - bool eos = false; - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 2); - expect_nullable_int64_column_values(*block.get_by_position(0).column, {1000, 1002}); - const auto& id_column = assert_cast(expect_not_null_table_column(block, 1)); - EXPECT_EQ(id_column.get_element(0), 1); - EXPECT_EQ(id_column.get_element(1), 3); + // The old file has no physical added_column, but Iceberg defines every row as the initial + // default 7. The equality delete key 7 therefore removes the entire file. + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); } -TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesPositionDeleteFile) { - const auto test_dir = - std::filesystem::temp_directory_path() / "doris_iceberg_position_delete_file_test"; +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesTimestampInitialDefaultForMissingColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_timestamp_default_test"; std::filesystem::remove_all(test_dir); std::filesystem::create_directories(test_dir); const auto file_path = (test_dir / "split.parquet").string(); - const auto delete_file_path = (test_dir / "position-delete.parquet").string(); - write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, - {"one", "two", "three", "four", "five"}); - write_position_delete_parquet_file(delete_file_path, {file_path, file_path}, {1, 3}); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + write_iceberg_timestamp_equality_delete_parquet_file(delete_file_path, 1, + 1'704'067'200'123'456L, "added_timestamp"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, + {external_schema_field("id", 0), + external_schema_field("added_timestamp", 1, {}, "2024-01-01 00:00:00.123456", + external_primitive_type(TPrimitiveType::DATETIMEV2, -1, 6))})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesVarbinaryInitialDefaultForMissingColumn) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_varbinary_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + const std::string binary_default("\x00\x01\x02\xff", 4); + write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, + "added_binary"); std::vector projected_columns; projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, + {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "AAEC/w==", + external_primitive_type(TPrimitiveType::VARBINARY, 4), true)})}); RuntimeProfile profile("test_profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteMatchesStringMappedBinaryInitialDefault) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_binary_string_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + const std::string binary_default("\x00\x01\x02\xff", 4); + write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, + "added_binary"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "AAEC/w==", + external_primitive_type(TPrimitiveType::STRING), true)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; io::FileReaderStats file_reader_stats; io::FileCacheStatistics file_cache_stats; auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); ShardedKVCache cache(1); doris::format::iceberg::IcebergTableReader reader; - ASSERT_TRUE(reader.init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = &scan_params, - .io_ctx = io_ctx, - .runtime_state = &state, - .scanner_profile = &profile, - }) - .ok()); + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); auto split_options = build_split_options(file_path); split_options.cache = &cache; split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( - file_path, {make_iceberg_position_delete_file(delete_file_path)})); + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); - EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3, 5})); + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteDecodesBinaryDefaultMappedToString) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_missing_string_binary_default_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "id", {1, 2, 3}, 0); + const std::string binary_default("\x00\x01\x02\xff", 4); + write_iceberg_binary_equality_delete_parquet_file(delete_file_path, 1, binary_default, + "added_binary"); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, {external_schema_field("id", 0), + external_schema_field("added_binary", 1, {}, "AAEC/w==", + external_primitive_type(TPrimitiveType::STRING), true)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {1})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_TRUE(read_iceberg_ids(&reader, projected_columns).empty()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeletePreservesTimestampTzAcrossDstFold) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_timestamp_tz_dst_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + // These instants are the two occurrences of 01:30 during the 2024 New York DST fall-back. + constexpr int64_t FIRST_0130_MICROS = 1'730'611'800'000'000L; + constexpr int64_t SECOND_0130_MICROS = 1'730'615'400'000'000L; + write_timestamp_int_parquet_file(file_path, {FIRST_0130_MICROS, SECOND_0130_MICROS}, {1, 2}); + write_iceberg_timestamp_equality_delete_parquet_file(delete_file_path, 0, FIRST_0130_MICROS, + "event_time", true); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(1, "id", std::make_shared())); + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_enable_mapping_timestamp_tz(true); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info({external_schema( + 100, + {external_schema_field("event_time", 0, {}, std::nullopt, + external_primitive_type(TPrimitiveType::TIMESTAMPTZ, -1, 6)), + external_schema_field("id", 1)})}); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state.set_timezone("America/New_York"); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesNameMappingWithoutFileFieldIds) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_name_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); + + std::vector projected_columns; + auto id_column = make_table_column(0, "current_id", std::make_shared()); + id_column.name_mapping.push_back("legacy_id"); + projected_columns.push_back(std::move(id_column)); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteByNameIgnoresStaleFileFieldId) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_stale_field_id_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + // The real key has no id and is mapped by its historical name. A different physical column + // carries the stale id 0, forcing the entire split into BY_NAME mode. + write_two_int_parquet_file(file_path, "legacy_id", {1, 2, 3}, std::nullopt, "stale_key", + {100, 200, 300}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); + + std::vector projected_columns; + auto id_column = make_table_column(0, "current_id", std::make_shared()); + id_column.name_mapping.push_back("legacy_id"); + projected_columns.push_back(std::move(id_column)); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteUsesUnprojectedTableSchemaNameMapping) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_hidden_name_mapping_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_two_int_parquet_file(file_path, "LEGACY_ID", {1, 2, 3}, std::nullopt, "data", + {10, 20, 30}, std::nullopt); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2, "current_id"); + + // Model SELECT data: the equality key is deliberately absent from the query projection. + std::vector projected_columns; + projected_columns.push_back(make_table_column(1, "data", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + scan_params.__set_current_schema_id(100); + scan_params.__set_history_schema_info( + {external_schema(100, {external_schema_field("current_id", 0, {"legacy_id"}), + external_schema_field("data", 1)})}); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({10, 30})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteFileIsReusedAcrossSplits) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_split_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + // Removing the source after the first split proves that the second split consumes the parsed + // delete block from SplitReadOptions.cache instead of reopening the delete file. + ASSERT_TRUE(std::filesystem::remove(delete_file_path)); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergEqualityDeleteCacheIsScopedByFileSystem) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_equality_delete_filesystem_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "equality-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 2); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_fs_name("filesystem-a"); + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + // The same descriptor path resolves to different content in another filesystem namespace. + write_iceberg_equality_delete_parquet_file(delete_file_path, 0, 3); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_fs_name("filesystem-b"); + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_equality_delete_file(delete_file_path, {0})})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 2})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteOnlyMatchesOriginalDataFilePath) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_position_delete_path_match_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto other_file_path = (test_dir / "other.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {other_file_path, file_path}, {0, 1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergRowLineageRemainsFileLocalAfterDeleteFiltering) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_row_lineage_delete_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3}, {10, 20, 30}, {"one", "two", "three"}); + write_position_delete_parquet_file(delete_file_path, {file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_iceberg_row_lineage_row_id_column()); + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + TTableFormatFileDesc table_format_params = make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)}); + table_format_params.iceberg_params.__set_first_row_id(1000); + split_options.current_range.__set_table_format_params(table_format_params); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + ASSERT_EQ(block.rows(), 2); + expect_nullable_int64_column_values(*block.get_by_position(0).column, {1000, 1002}); + const auto& id_column = assert_cast(expect_not_null_table_column(block, 1)); + EXPECT_EQ(id_column.get_element(0), 1); + EXPECT_EQ(id_column.get_element(1), 3); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergTableReaderAppliesPositionDeleteFile) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_position_delete_file_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + write_position_delete_parquet_file(delete_file_path, {file_path, file_path}, {1, 3}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = io_ctx, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc( + file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3, 5})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteHonorsPositionBounds) { + const auto test_dir = + std::filesystem::temp_directory_path() / "doris_iceberg_position_delete_bounds_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 50}, + {"one", "two", "three", "four", "five"}); + write_position_delete_parquet_file(delete_file_path, {file_path, file_path, file_path}, + {0, 2, 4}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto delete_file = make_iceberg_position_delete_file(delete_file_path); + delete_file.__set_position_lower_bound(1); + delete_file.__set_position_upper_bound(3); + auto split_options = build_split_options(file_path); + split_options.cache = &cache; + split_options.current_range.__set_table_format_params( + make_iceberg_table_format_desc(file_path, {delete_file})); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 2, 4, 5})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteFileIsReusedAcrossSplits) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_position_delete_split_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_position_delete_parquet_file(delete_file_path, {first_file_path, second_file_path}, + {1, 0}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + // The cached delete file contains entries for every referenced data file, so another split can + // select its own positions without rescanning the shared delete file. + ASSERT_TRUE(std::filesystem::remove(delete_file_path)); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2, 3})); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + +TEST(IcebergV2ReaderTest, IcebergPositionDeleteCacheIsScopedByFileSystem) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_iceberg_position_delete_filesystem_cache_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto first_file_path = (test_dir / "first.parquet").string(); + const auto second_file_path = (test_dir / "second.parquet").string(); + const auto delete_file_path = (test_dir / "position-delete.parquet").string(); + write_single_int_parquet_file(first_file_path, "id", {1, 2, 3}, 0); + write_single_int_parquet_file(second_file_path, "id", {1, 2, 3}, 0); + write_position_delete_parquet_file(delete_file_path, {first_file_path}, {1}); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + io::FileReaderStats file_reader_stats; + io::FileCacheStatistics file_cache_stats; + auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats); + ShardedKVCache cache(1); + doris::format::iceberg::IcebergTableReader reader; + init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx, &state, &profile); + + auto first_split = build_split_options(first_file_path); + first_split.cache = &cache; + first_split.current_range.__set_fs_name("filesystem-a"); + first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); + + write_position_delete_parquet_file(delete_file_path, {second_file_path}, {0}); + auto second_split = build_split_options(second_file_path); + second_split.cache = &cache; + second_split.current_range.__set_fs_name("filesystem-b"); + second_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( + second_file_path, {make_iceberg_position_delete_file(delete_file_path)})); + ASSERT_TRUE(reader.prepare_split(second_split).ok()); + EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({2, 3})); ASSERT_TRUE(reader.close().ok()); std::filesystem::remove_all(test_dir); diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 0d484f60d9e5de..970e4b2153557e 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -4085,6 +4085,63 @@ TEST(TableReaderTest, ReusedBlockClearsProjectedStructWithNullableChild) { std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, ProjectedRenamedStructPreservesParentDeclaredChildNullability) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_struct_parent_child_nullability_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto file_path = (test_dir / "split.parquet").string(); + write_struct_with_nullable_child_parquet_file(file_path); + + const auto string_type = std::make_shared(); + const auto nullable_string_type = make_nullable(string_type); + auto renamed_note = make_table_column(1, "renamed_note", nullable_string_type); + renamed_note.name_mapping = {"note"}; + // Iceberg nested schema metadata can omit nullability on this child while the parent DataType + // remains authoritative and declares it nullable. + renamed_note.type = string_type; + auto struct_type = std::make_shared(DataTypes {nullable_string_type}, + Strings {"renamed_note"}); + auto struct_column = make_table_column(100, "s", struct_type); + struct_column.children = {renamed_note}; + std::vector projected_columns = {struct_column}; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + set_name_identifiers(&projected_columns); + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(file_path)).ok()); + + Block block = build_table_block(projected_columns); + bool eos = false; + const auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_FALSE(eos); + + const auto& struct_result = + assert_cast(expect_not_null_table_column(block, 0)); + ASSERT_EQ(struct_result.get_columns().size(), 1); + const auto& notes = assert_cast(struct_result.get_column(0)); + EXPECT_FALSE(notes.is_null_at(0)); + EXPECT_EQ(notes.get_data_at(0).to_string(), "seven"); + EXPECT_TRUE(notes.is_null_at(1)); + ASSERT_TRUE(block.get_by_position(0).check_type_and_column_match().ok()); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, ProjectedPartitionColumnUsesSplitPartitionValue) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_table_reader_partition_value_test"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java index 784315fc15c7bd..6a986da45303d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java @@ -34,6 +34,7 @@ import org.apache.doris.thrift.schema.external.TStructField; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,6 +46,9 @@ private static TField getExternalSchema(Column column) { root.setId(column.getUniqueId()); root.setIsOptional(column.isAllowNull()); root.setType(column.getType().toColumnTypeThrift()); + if (column.getDefaultValue() != null) { + root.setInitialDefaultValue(column.getDefaultValue()); + } TNestedField nestedField = new TNestedField(); if (column.getType().isStructType()) { @@ -120,7 +124,8 @@ private static TStructField getExternalSchemaForPrunedColumn(List columns, Map> nameMapping) { + initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping, Collections.emptyMap()); + } + + public static void initSchemaInfoForAllColumn(TFileScanRangeParams params, Long schemaId, + List columns, Map> nameMapping, + Map base64InitialDefaults) { params.setCurrentSchemaId(schemaId); TSchema tSchema = new TSchema(); tSchema.setSchemaId(schemaId); - tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping)); + tSchema.setRootField(getExternalSchemaForAllColumn(columns, nameMapping, base64InitialDefaults)); params.addToHistorySchemaInfo(tSchema); } private static TStructField getExternalSchemaForAllColumn(List columns, - Map> nameMapping) { + Map> nameMapping, Map base64InitialDefaults) { TStructField structField = new TStructField(); for (Column child : columns) { TFieldPtr fieldPtr = new TFieldPtr(); - fieldPtr.setFieldPtr(getExternalSchema(child.getType(), child, nameMapping)); + fieldPtr.setFieldPtr(getExternalSchema( + child.getType(), child, nameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } return structField; @@ -149,11 +161,22 @@ private static TStructField getExternalSchemaForAllColumn(List columns, private static TField getExternalSchema(Type columnType, Column dorisColumn, Map> nameMapping) { + return getExternalSchema(columnType, dorisColumn, nameMapping, Collections.emptyMap()); + } + + private static TField getExternalSchema(Type columnType, Column dorisColumn, + Map> nameMapping, Map base64InitialDefaults) { TField root = new TField(); root.setName(dorisColumn.getName()); root.setId(dorisColumn.getUniqueId()); root.setIsOptional(dorisColumn.isAllowNull()); root.setType(dorisColumn.getType().toColumnTypeThrift()); + if (base64InitialDefaults.containsKey(dorisColumn.getUniqueId())) { + root.setInitialDefaultValue(base64InitialDefaults.get(dorisColumn.getUniqueId())); + root.setInitialDefaultValueIsBase64(true); + } else if (dorisColumn.getDefaultValue() != null) { + root.setInitialDefaultValue(dorisColumn.getDefaultValue()); + } if (nameMapping != null && nameMapping.containsKey(dorisColumn.getUniqueId())) { // for iceberg set name mapping. @@ -175,7 +198,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, for (StructField subField : dorisStructType.getFields()) { TFieldPtr fieldPtr = new TFieldPtr(); Column subColumn = subNameToSubColumn.get(subField.getName()); - fieldPtr.setFieldPtr(getExternalSchema(subField.getType(), subColumn, nameMapping)); + fieldPtr.setFieldPtr(getExternalSchema( + subField.getType(), subColumn, nameMapping, base64InitialDefaults)); structField.addToFields(fieldPtr); } @@ -187,7 +211,8 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TArrayField listField = new TArrayField(); TFieldPtr fieldPtr = new TFieldPtr(); fieldPtr.setFieldPtr(getExternalSchema( - dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping)); + dorisArrayType.getItemType(), dorisColumn.getChildren().get(0), nameMapping, + base64InitialDefaults)); listField.setItemField(fieldPtr); nestedField.setArrayField(listField); root.setNestedField(nestedField); @@ -197,12 +222,14 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, TMapField mapField = new TMapField(); TFieldPtr keyPtr = new TFieldPtr(); keyPtr.setFieldPtr(getExternalSchema( - dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping)); + dorisMapType.getKeyType(), dorisColumn.getChildren().get(0), nameMapping, + base64InitialDefaults)); mapField.setKeyField(keyPtr); TFieldPtr valuePtr = new TFieldPtr(); valuePtr.setFieldPtr(getExternalSchema( - dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping)); + dorisMapType.getValueType(), dorisColumn.getChildren().get(1), nameMapping, + base64InitialDefaults)); mapField.setValueField(valuePtr); nestedField.setMapField(mapField); root.setNestedField(nestedField); @@ -210,4 +237,3 @@ private static TField getExternalSchema(Type columnType, Column dorisColumn, return root; } } - diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 2cd4f2bcf2f233..12b5dce11395fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -108,6 +108,7 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -135,6 +136,7 @@ import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; +import java.util.Base64; import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashSet; @@ -1182,10 +1184,14 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi List columns = schema.columns(); List resSchema = Lists.newArrayListWithCapacity(columns.size()); for (Types.NestedField field : columns) { + String initialDefault = null; + if (field.initialDefault() != null) { + initialDefault = serializeInitialDefault(field.type(), field.initialDefault(), + enableMappingTimestampTz); + } Column column = new Column(field.name(), IcebergUtils.icebergTypeToDorisType(field.type(), enableMappingVarbinary, enableMappingTimestampTz), - true, null, - true, field.doc(), true, -1); + true, null, true, initialDefault, field.doc(), true, -1); updateIcebergColumnUniqueId(column, field); if (field.type().isPrimitiveType() && field.type().typeId() == TypeID.TIMESTAMP) { Types.TimestampType timestampType = (Types.TimestampType) field.type(); @@ -1198,6 +1204,62 @@ public static List parseSchema(Schema schema, boolean enableMappingVarbi return resSchema; } + private static String serializeInitialDefault(org.apache.iceberg.types.Type type, Object value, + boolean enableMappingTimestampTz) { + String humanValue = Transforms.identity(type).toHumanString(type, value); + if (type.typeId() == TypeID.TIMESTAMP) { + // Iceberg formats timestamps as ISO-8601 (for example 2024-01-01T00:00:00), while + // Doris' DATETIMEV2 default parser requires a space between the date and time. + String dorisValue = humanValue.replace('T', ' '); + Types.TimestampType timestampType = (Types.TimestampType) type; + if (timestampType.shouldAdjustToUTC() && !enableMappingTimestampTz) { + // Iceberg timestamptz human values carry a trailing offset. DATETIMEV2 has no + // offset carrier, so retain the displayed UTC wall time and remove the suffix. + return dorisValue.replaceFirst("(Z|[+-]\\d{2}:\\d{2})$", ""); + } + return dorisValue; + } + if (isBinaryLike(type)) { + // Always use the lossless Base64 carrier. Binary-like Iceberg fields may map to either + // VARBINARY or STRING/CHAR, and the scan schema marker tells BE to decode both forms + // back to the raw bytes stored in equality-delete files. + return serializeBinaryInitialDefault(type, value); + } + return humanValue; + } + + /** + * Return binary-like initial defaults in a lossless transport representation. These defaults + * cannot be carried as raw Java strings and their Doris type is insufficient to identify them + * when varbinary mapping is disabled, because UUID/BINARY/FIXED then map to STRING/CHAR. + */ + public static Map getBase64EncodedInitialDefaults(Schema schema) { + Map result = Maps.newHashMap(); + for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) { + if (field.initialDefault() == null || !isBinaryLike(field.type())) { + continue; + } + result.put(field.fieldId(), serializeBinaryInitialDefault(field.type(), field.initialDefault())); + } + return result; + } + + private static boolean isBinaryLike(org.apache.iceberg.types.Type type) { + return type.typeId() == TypeID.UUID || type.typeId() == TypeID.BINARY + || type.typeId() == TypeID.FIXED; + } + + private static String serializeBinaryInitialDefault(org.apache.iceberg.types.Type type, Object value) { + if (type.typeId() != TypeID.UUID) { + return Transforms.identity(type).toHumanString(type, value); + } + UUID uuid = (UUID) value; + ByteBuffer bytes = ByteBuffer.allocate(16); + bytes.putLong(uuid.getMostSignificantBits()); + bytes.putLong(uuid.getLeastSignificantBits()); + return Base64.getEncoder().encodeToString(bytes.array()); + } + /** * Estimate iceberg table row count. * Get the row count by adding all task file recordCount. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 2769a1ddb8e43e..a69a1c55d354d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -81,6 +81,7 @@ import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; +import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; @@ -454,20 +455,26 @@ public void createScanRangeLocations() throws UserException { // Extract name mapping from Iceberg table properties Map> nameMapping = extractNameMapping(); - boolean haveTopnLazyMatCol = false; - for (SlotDescriptor slot : desc.getSlots()) { - String colName = slot.getColumn().getName(); - if (colName.startsWith(Column.GLOBAL_ROWID_COL)) { - haveTopnLazyMatCol = true; - break; - } - } - if (haveTopnLazyMatCol) { - ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), nameMapping); - } else { - // Use new initSchemaInfo method that only includes needed columns based on slots and pruned type - ExternalUtil.initSchemaInfoForPrunedColumn(params, -1L, desc.getSlots(), nameMapping); - } + // Equality-delete keys are hidden scan dependencies and need not appear in the query + // projection. Both scanners need the complete current schema to resolve field ids, + // historical names, types, and initial defaults when an old data file lacks such a key. + ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), + nameMapping, getBase64EncodedInitialDefaultsForScan()); + } + + @VisibleForTesting + Map getBase64EncodedInitialDefaultsForScan() throws UserException { + TableScan tableScan = createTableScan(); + Snapshot snapshot = tableScan.snapshot(); + // TableScan.schema() starts from the table's current schema even for useSnapshot/useRef. + // Resolve the selected snapshot's schema id explicitly so this metadata describes the same + // snapshot as source.getTargetTable().getColumns(). Otherwise a later type change can make + // BE decode a historical non-binary default as Base64, or fail to decode a binary default. + Schema scanSchema = snapshot == null + ? tableScan.schema() + : tableScan.table().schemas().get(snapshot.schemaId()); + return IcebergUtils.getBase64EncodedInitialDefaults( + Preconditions.checkNotNull(scanSchema, "Schema for Iceberg scan snapshot is null")); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java index 5ce09be0456e5f..3ecb3a72d6b023 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalUtilTest.java @@ -219,7 +219,7 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { TFileScanRangeParams params = new TFileScanRangeParams(); Long schemaId = 500L; - Column col1 = new Column("c1", Type.INT, true); + Column col1 = new Column("c1", Type.INT, false, null, true, "7", ""); col1.setUniqueId(101); Column col2 = new Column("c2", Type.VARCHAR, false); col2.setUniqueId(102); @@ -230,7 +230,10 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { nameMapping.put(col1.getUniqueId(), Arrays.asList("m_c1")); nameMapping.put(col2.getUniqueId(), Arrays.asList("m_c2_a", "m_c2_b")); - ExternalUtil.initSchemaInfoForAllColumn(params, schemaId, columns, nameMapping); + Map base64InitialDefaults = new HashMap<>(); + base64InitialDefaults.put(col2.getUniqueId(), "AAEC/w=="); + ExternalUtil.initSchemaInfoForAllColumn( + params, schemaId, columns, nameMapping, base64InitialDefaults); Assert.assertEquals(schemaId.longValue(), params.getCurrentSchemaId()); List history = params.getHistorySchemaInfo(); @@ -251,12 +254,15 @@ public void testInitSchemaInfoForAllColumnMultipleColumnsAndNameMapping() { Assert.assertEquals(col1.isAllowNull(), field1.isIsOptional()); Assert.assertEquals(col1.getType().toColumnTypeThrift(), field1.getType()); Assert.assertEquals(Arrays.asList("m_c1"), field1.getNameMapping()); + Assert.assertEquals("7", field1.getInitialDefaultValue()); + Assert.assertFalse(field1.isSetInitialDefaultValueIsBase64()); Assert.assertEquals(col2.getName(), field2.getName()); Assert.assertEquals(col2.getUniqueId(), field2.getId()); Assert.assertEquals(col2.isAllowNull(), field2.isIsOptional()); Assert.assertEquals(col2.getType().toColumnTypeThrift(), field2.getType()); Assert.assertEquals(Arrays.asList("m_c2_a", "m_c2_b"), field2.getNameMapping()); + Assert.assertEquals("AAEC/w==", field2.getInitialDefaultValue()); + Assert.assertTrue(field2.isInitialDefaultValueIsBase64()); } } - diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index dc344b08e13812..0a798faff93fef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -198,6 +198,10 @@ protected void runBeforeAll() throws Exception { Mockito.doReturn(mockedTableScan).when(mockedTableScan).filter(ArgumentMatchers.any()); Mockito.doReturn(mockedTableScan).when(mockedTableScan).planWith(ArgumentMatchers.any()); Mockito.doReturn(null).when(mockedTableScan).snapshot(); + // Keep the scan schema aligned with the mocked table schema. IcebergScanNode reads the + // selected scan schema when serializing initial defaults, and several tests temporarily + // replace the table schema to exercise partition transforms. + Mockito.doAnswer(invocation -> mockedIcebergTable.schema()).when(mockedTableScan).schema(); Mockito.doReturn(CloseableIterable.withNoopClose(java.util.Collections.emptyList())) .when(mockedTableScan).planFiles(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java index 8625b36f283ab4..ee34b4bee50b0c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java @@ -75,6 +75,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; public class IcebergUtilsTest { @Test @@ -299,6 +300,48 @@ public void testParseSchemaPreservesNonLowercaseColumnNames() { Assert.assertEquals("PART", columns.get(1).getName()); } + @Test + public void testParseSchemaPreservesInitialDefault() { + Schema schema = new Schema( + Types.NestedField.optional("added_column") + .withId(1) + .ofType(Types.IntegerType.get()) + .withInitialDefault(7) + .build(), + Types.NestedField.optional("added_timestamp") + .withId(2) + .ofType(Types.TimestampType.withoutZone()) + .withInitialDefault(1_704_067_200_123_456L) + .build(), + Types.NestedField.optional("added_uuid") + .withId(3) + .ofType(Types.UUIDType.get()) + .withInitialDefault(UUID.fromString("00000000-0000-0000-0000-000000000000")) + .build(), + Types.NestedField.optional("added_binary") + .withId(4) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build(), + Types.NestedField.optional("added_fixed") + .withId(5) + .ofType(Types.FixedType.ofLength(4)) + .withInitialDefault(ByteBuffer.wrap(new byte[] {3, 2, 1, 0})) + .build()); + + List columns = IcebergUtils.parseSchema(schema, true, false); + + Assert.assertEquals("7", columns.get(0).getDefaultValue()); + Assert.assertEquals("2024-01-01 00:00:00.123456", columns.get(1).getDefaultValue()); + Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", columns.get(2).getDefaultValue()); + Assert.assertEquals("AAEC/w==", columns.get(3).getDefaultValue()); + + Map base64Defaults = IcebergUtils.getBase64EncodedInitialDefaults(schema); + Assert.assertEquals("AAAAAAAAAAAAAAAAAAAAAA==", base64Defaults.get(3)); + Assert.assertEquals("AAEC/w==", base64Defaults.get(4)); + Assert.assertEquals("AwIBAA==", base64Defaults.get(5)); + } + @Test public void testGetPartitionInfoMapSkipBinaryIdentityPartition() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index f55b4f6f8e8027..46abdf08a587e9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -29,6 +29,11 @@ import org.apache.iceberg.DataFile; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ScanTaskUtil; import org.junit.Assert; import org.junit.Test; @@ -36,23 +41,64 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; +import java.util.Map; public class IcebergScanNodeTest { private static final long MB = 1024L * 1024L; private static class TestIcebergScanNode extends IcebergScanNode { + private TableScan tableScan; + TestIcebergScanNode(SessionVariable sv) { super(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), sv, ScanContext.EMPTY); } + void setTableScan(TableScan tableScan) { + this.tableScan = tableScan; + } + + @Override + public TableScan createTableScan() { + return tableScan; + } + @Override public boolean isBatchMode() { return false; } } + @Test + public void testInitialDefaultMetadataUsesSnapshotSchema() throws Exception { + Schema snapshotSchema = new Schema(Types.NestedField.optional("historical_binary") + .withId(7) + .ofType(Types.BinaryType.get()) + .withInitialDefault(ByteBuffer.wrap(new byte[] {0, 1, 2, (byte) 0xFF})) + .build()); + Schema currentSchema = new Schema(Types.NestedField.optional("current_string") + .withId(7) + .ofType(Types.StringType.get()) + .withInitialDefault("not-base64") + .build()); + Snapshot snapshot = Mockito.mock(Snapshot.class); + Mockito.when(snapshot.schemaId()).thenReturn(11); + Table table = Mockito.mock(Table.class); + Mockito.when(table.schemas()).thenReturn(Collections.singletonMap(11, snapshotSchema)); + TableScan snapshotScan = Mockito.mock(TableScan.class); + Mockito.when(snapshotScan.snapshot()).thenReturn(snapshot); + Mockito.when(snapshotScan.table()).thenReturn(table); + Mockito.when(snapshotScan.schema()).thenReturn(currentSchema); + + TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); + node.setTableScan(snapshotScan); + + Map defaults = node.getBase64EncodedInitialDefaultsForScan(); + Assert.assertEquals(Collections.singletonMap(7, "AAEC/w=="), defaults); + } + @Test public void testDetermineTargetFileSplitSizeHonorsMaxFileSplitNum() throws Exception { SessionVariable sv = new SessionVariable(); diff --git a/gensrc/thrift/ExternalTableSchema.thrift b/gensrc/thrift/ExternalTableSchema.thrift index 14972f8d434784..46ae54781ae825 100644 --- a/gensrc/thrift/ExternalTableSchema.thrift +++ b/gensrc/thrift/ExternalTableSchema.thrift @@ -49,7 +49,16 @@ struct TField { 3: optional string name, // Field name 4: optional Types.TColumnType type, // Corresponding Doris column type 5: optional TNestedField nestedField // Nested field definition (for array, struct, or map types) - 6: optional list name_mapping // iceberg : schema.name-mapping.default, for missing column id. + 6: optional list name_mapping, // iceberg : schema.name-mapping.default, for missing column id. + // Iceberg initial default normalized for transport to BE. Binary-like Iceberg values use + // Base64 because Thrift's Java string carrier cannot preserve arbitrary bytes; other primitive + // values use Doris' FE string representation. An old data file that predates this field + // logically contains this value rather than NULL. + 7: optional string initial_default_value, + // True when initial_default_value is Base64 and must be decoded before constructing the Doris + // STRING/CHAR/VARBINARY value. This cannot be inferred from the Doris type because Iceberg + // UUID/BINARY/FIXED may map either to VARBINARY or to STRING/CHAR. + 8: optional bool initial_default_value_is_base64 } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy index b4394755c9aa97..f24b075443ab4d 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete_with_schema_change.groovy @@ -42,9 +42,17 @@ suite("test_iceberg_equality_delete_with_schema_change", "p0,external,doris,exte sql """switch ${catalog_name};""" sql """ use multi_catalog;""" - + def enableFileScannerV2Rows = sql """show variables like 'enable_file_scanner_v2'""" + assertTrue(enableFileScannerV2Rows.size() > 0, + "Session variable enable_file_scanner_v2 is not found") + String originalEnableFileScannerV2 = enableFileScannerV2Rows[0][1].toString() + // The existing output was generated by the legacy scanner and is kept as the differential + // baseline. Run the complete schema-evolution matrix through V2 so missing/renamed equality + // delete keys must produce byte-for-byte identical query results. + sql """set enable_file_scanner_v2=true""" - for (String format: ["par","orc"]) { + try { + for (String format: ["par","orc"]) { // Basic full table scan order_qt_q1 """ SELECT * FROM equality_delete_${format}_1 ORDER BY new_new_id; """ @@ -190,5 +198,9 @@ suite("test_iceberg_equality_delete_with_schema_change", "p0,external,doris,exte order_qt_q70 """ SELECT new_new_id, new_name, data FROM equality_delete_${format}_1 WHERE new_new_id = 1 UNION ALL SELECT new_new_id, new_name, data FROM equality_delete_${format}_2 WHERE new_new_id = 1; """ + + } + } finally { + sql """set enable_file_scanner_v2=${originalEnableFileScannerV2}""" } }