Skip to content

fix(schema): use Arrow concrete decimal type IDs in schema validation #347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lxy-9602 merged 2 commits into from
Jun 9, 2026
Merged

fix(schema): use Arrow concrete decimal type IDs in schema validation #347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e39c13a
fix: replace arrow type DECIMAL to DECIMAL128
lucasfang Jun 8, 2026
56f157a
Merge branch 'main' into benchmark
lucasfang Jun 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/paimon/common/data/binary_row_writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ Result<BinaryRowWriter::FieldSetterFunc> BinaryRowWriter::CreateFieldSetter(
};
return field_setter;
}
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
auto* decimal_type =
arrow::internal::checked_cast<arrow::Decimal128Type*>(field_type.get());
assert(decimal_type);
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/common/data/internal_row.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ Result<InternalRow::FieldGetterFunc> InternalRow::CreateFieldGetter(
};
break;
}
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
auto* decimal_type =
arrow::internal::checked_cast<arrow::Decimal128Type*>(field_type.get());
assert(decimal_type);
Expand Down
4 changes: 2 additions & 2 deletions src/paimon/common/data/serializer/binary_serializer_utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ Status BinarySerializerUtils::WriteBinaryData(const std::shared_ptr<arrow::DataT
if (array_writer) {
array_writer->SetNullAt(pos, type_id);
return Status::OK();
} else if (type_id != arrow::Type::type::DECIMAL &&
} else if (type_id != arrow::Type::type::DECIMAL128 &&
type_id != arrow::Type::type::TIMESTAMP) {
// if row writer, exclude decimal and timestamp when set null
writer->SetNullAt(pos);
Expand Down Expand Up @@ -167,7 +167,7 @@ Status BinarySerializerUtils::WriteBinaryData(const std::shared_ptr<arrow::DataT
}
break;
}
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
auto* decimal_type = arrow::internal::checked_cast<arrow::Decimal128Type*>(type.get());
assert(decimal_type);
auto precision = decimal_type->precision();
Expand Down
8 changes: 4 additions & 4 deletions src/paimon/common/data/serializer/row_compacted_serializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ Result<int32_t> RowCompactedSerializer::CompareField(const FieldInfo& field_info
PAIMON_ASSIGN_OR_RAISE(Timestamp val2, reader2->ReadTimestamp(field_info.precision));
return val1 == val2 ? 0 : (val1 < val2 ? -1 : 1);
}
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
PAIMON_ASSIGN_OR_RAISE(Decimal val1,
reader1->ReadDecimal(field_info.precision, field_info.scale));
PAIMON_ASSIGN_OR_RAISE(Decimal val2,
Expand Down Expand Up @@ -124,7 +124,7 @@ Result<MemorySlice::SliceComparator> RowCompactedSerializer::CreateSliceComparat
arrow::internal::checked_pointer_cast<arrow::TimestampType>(field_type);
assert(timestamp_type);
field_infos[i].precision = DateTimeUtils::GetPrecisionFromType(timestamp_type);
} else if (field_type->id() == arrow::Type::type::DECIMAL) {
} else if (field_type->id() == arrow::Type::type::DECIMAL128) {
auto decimal_type =
arrow::internal::checked_pointer_cast<arrow::Decimal128Type>(field_type);
assert(decimal_type);
Expand Down Expand Up @@ -275,7 +275,7 @@ Result<RowCompactedSerializer::FieldReader> RowCompactedSerializer::CreateFieldR
};
break;
}
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
auto* decimal_type =
arrow::internal::checked_cast<arrow::Decimal128Type*>(field_type.get());
assert(decimal_type);
Expand Down Expand Up @@ -410,7 +410,7 @@ Result<RowCompactedSerializer::FieldWriter> RowCompactedSerializer::CreateFieldW
};
break;
}
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
auto* decimal_type =
arrow::internal::checked_cast<arrow::Decimal128Type*>(field_type.get());
assert(decimal_type);
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/common/utils/fields_comparator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ Result<FieldsComparator::FieldComparatorFunc> FieldsComparator::CompareField(
return lvalue == rvalue ? 0 : (lvalue < rvalue ? -1 : 1);
});
}
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
auto* decimal_type =
arrow::internal::checked_cast<arrow::Decimal128Type*>(input_type.get());
assert(decimal_type);
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/core/bucket/bucket_id_calculator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ static Result<WriteFunction> WriteBucketRow(int32_t col_id,
};
return writer_func;
}
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
const auto* decimal_type =
arrow::internal::checked_cast<const arrow::Decimal128Type*>(field->type().get());
assert(decimal_type);
Expand Down
6 changes: 3 additions & 3 deletions src/paimon/core/io/row_to_arrow_array_converter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ Status RowToArrowArrayConverter<T, R>::Reserve(arrow::ArrayBuilder* array_builde
case arrow::Type::type::FLOAT:
case arrow::Type::type::DOUBLE:
case arrow::Type::type::TIMESTAMP:
case arrow::Type::type::DECIMAL:
case arrow::Type::type::DECIMAL128:
break;
case arrow::Type::type::STRING: {
// reserve string data buffer
Expand Down Expand Up @@ -201,7 +201,7 @@ Status RowToArrowArrayConverter<T, R>::Accumulate(const arrow::Array* array, int
case arrow::Type::type::FLOAT:
case arrow::Type::type::DOUBLE:
case arrow::Type::type::TIMESTAMP:
case arrow::Type::type::DECIMAL:
case arrow::Type::type::DECIMAL128:
break;
case arrow::Type::type::STRING: {
auto string_array = arrow::internal::checked_cast<const arrow::StringArray*>(array);
Expand Down Expand Up @@ -403,7 +403,7 @@ RowToArrowArrayConverter<T, R>::AppendField(bool use_view, arrow::ArrayBuilder*
DateTimeUtils::TimestampToInteger(timestamp, time_type));
});
}
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
PAIMON_ASSIGN_OR_RAISE(auto* field_builder,
CastToTypedBuilder<arrow::Decimal128Builder>(array_builder));
auto decimal_type =
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/core/mergetree/compact/aggregate/field_max_agg.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class FieldMaxAgg : public FieldAggregator {
case arrow::Type::type::FLOAT:
case arrow::Type::type::DOUBLE:
case arrow::Type::type::TIMESTAMP:
case arrow::Type::type::DECIMAL:
case arrow::Type::type::DECIMAL128:
case arrow::Type::type::STRING:
case arrow::Type::type::BINARY:
return FieldMaxFunc([](const VariantType& accumulator,
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/core/mergetree/compact/aggregate/field_min_agg.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class FieldMinAgg : public FieldAggregator {
case arrow::Type::type::FLOAT:
case arrow::Type::type::DOUBLE:
case arrow::Type::type::TIMESTAMP:
case arrow::Type::type::DECIMAL:
case arrow::Type::type::DECIMAL128:
case arrow::Type::type::STRING:
case arrow::Type::type::BINARY:
return FieldMinFunc([](const VariantType& accumulator,
Expand Down
4 changes: 2 additions & 2 deletions src/paimon/core/mergetree/compact/aggregate/field_sum_agg.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ Result<FieldSumAgg::FieldSumFunc> FieldSumAgg::CreateSumFunc(
DataDefine::GetVariantValue<double>(input_field);
return sum;
});
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
return FieldSumFunc(
[](const VariantType& accumulator, const VariantType& input_field) -> VariantType {
auto v1 = DataDefine::GetVariantValue<Decimal>(accumulator);
Expand Down Expand Up @@ -120,7 +120,7 @@ Result<FieldSumAgg::FieldNegFunc> FieldSumAgg::CreateNegFunc(
auto value = DataDefine::GetVariantValue<double>(input_field);
return (-value);
});
case arrow::Type::type::DECIMAL: {
case arrow::Type::type::DECIMAL128: {
return FieldNegFunc([](const VariantType& input_field) -> VariantType {
auto value = DataDefine::GetVariantValue<Decimal>(input_field);
return Decimal(value.Precision(), value.Scale(), -value.Value());
Expand Down
2 changes: 1 addition & 1 deletion src/paimon/core/mergetree/in_memory_sort_buffer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ Result<int64_t> InMemorySortBuffer::EstimateMemoryUse(const std::shared_ptr<arro
return null_bits_size_in_bytes + array->length() * sizeof(double);
case arrow::Type::type::TIMESTAMP:
return null_bits_size_in_bytes + array->length() * sizeof(int64_t);
case arrow::Type::type::DECIMAL:
case arrow::Type::type::DECIMAL128:
return null_bits_size_in_bytes + array->length() * sizeof(Decimal::int128_t);
case arrow::Type::type::STRING:
case arrow::Type::type::BINARY: {
Expand Down
5 changes: 3 additions & 2 deletions src/paimon/core/schema/schema_validation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,9 @@
namespace paimon {

bool SchemaValidation::IsComplexType(const std::shared_ptr<arrow::Field>& field) {
return (field->type()->id() == arrow::Type::TIMESTAMP ||
field->type()->id() == arrow::Type::DECIMAL || BlobUtils::IsBlobField(field));
arrow::Type::type arrow_type_id = field->type()->id();
return (arrow_type_id == arrow::Type::TIMESTAMP || arrow_type_id == arrow::Type::DECIMAL128 ||
BlobUtils::IsBlobField(field));
}

Status SchemaValidation::ValidateTableSchema(const TableSchema& schema) {
Expand Down
Loading