From c66a4ca9d944e81769b4939c9acddc2d2c0f1b0b Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 12 Jul 2026 14:36:48 +0800 Subject: [PATCH 1/2] [fix](be) Prevent DST timestamp statistics mispruning ### What problem does this PR solve? Issue Number: N/A Related PR: #64456 Problem Summary: FileScannerV2 converts UTC timestamp statistics into session-local DATETIMEV2 values before ZoneMap evaluation. Across a daylight-saving time rollback, that conversion is not monotonic, so converted metadata endpoints may exclude local values that occur inside the UTC interval. Parquet row-group and page-index pruning, and ORC statistics pruning, could therefore skip matching data. Detect backward timezone transitions within UTC statistics ranges and conservatively disable min/max pruning for those ranges while retaining statistics for monotonic ranges and TIMESTAMPTZ mappings. ### Release note Fix incorrect Parquet and ORC file scan pruning for UTC timestamps across daylight-saving time rollbacks. ### Check List (For Author) - Test: Unit Test - Remote ASAN BE UT: TimestampStatisticsTest.*, ParquetStatisticsTransformTest.*, and NewOrcReaderTest.AggregatePushdownTimestamp* (7 passed) - Behavior changed: Yes. Non-monotonic local timestamp statistics no longer prune rows or pages. - Does this need documentation: No --- be/src/format_v2/orc/orc_reader.cpp | 6 ++ .../format_v2/parquet/parquet_statistics.cpp | 43 ++++++++++++++ be/src/format_v2/timestamp_statistics.h | 59 +++++++++++++++++++ .../parquet/parquet_statistics_test.cpp | 28 +++++++++ .../format_v2/timestamp_statistics_test.cpp | 44 ++++++++++++++ 5 files changed, 180 insertions(+) create mode 100644 be/src/format_v2/timestamp_statistics.h create mode 100644 be/test/format_v2/timestamp_statistics_test.cpp diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 32b8aed2a888db..6f9ea2e10aaaca 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -69,6 +69,7 @@ #include "exprs/vslot_ref.h" #include "format_v2/column_mapper.h" #include "format_v2/orc/orc_search_argument.h" +#include "format_v2/timestamp_statistics.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" #include "runtime/runtime_profile.h" @@ -531,6 +532,11 @@ bool set_timestamp_zone_map(const ::orc::ColumnStatistics& statistics, timestamp_statistics->getMaximum(), timestamp_statistics->getMaximumNanos())); return true; } + if (!format::utc_timestamp_range_is_monotonic( + format::floor_epoch_seconds(timestamp_statistics->getMinimum(), 1000), + format::floor_epoch_seconds(timestamp_statistics->getMaximum(), 1000), timezone)) { + return false; + } zone_map->min_value = Field::create_field(datetime_v2_from_orc_millis( timestamp_statistics->getMinimum(), timestamp_statistics->getMinimumNanos(), timezone)); zone_map->max_value = Field::create_field(datetime_v2_from_orc_millis( diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index d0692a80205f3f..e188a6bf157e31 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,7 @@ #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr_context.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/timestamp_statistics.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/index/zone_map/zonemap_eval_context.h" @@ -118,6 +120,35 @@ bool set_decoded_field(const ParquetColumnSchema& column_schema, DecodedValueKin return read_decoded_field(column_schema, view, field, timezone).ok(); } +int64_t floor_timestamp_seconds(int64_t value, ParquetTimeUnit time_unit) { + int64_t units_per_second = 1; + switch (time_unit) { + case ParquetTimeUnit::MILLIS: + units_per_second = 1000; + break; + case ParquetTimeUnit::MICROS: + units_per_second = 1000000; + break; + case ParquetTimeUnit::NANOS: + units_per_second = 1000000000; + break; + default: + DORIS_CHECK(false); + } + return format::floor_epoch_seconds(value, units_per_second); +} + +bool timestamp_min_max_is_safe(const ParquetColumnSchema& column_schema, int64_t min_value, + int64_t max_value, const cctz::time_zone* timezone) { + if (!column_schema.type_descriptor.is_timestamp || + !column_schema.type_descriptor.timestamp_is_adjusted_to_utc || timezone == nullptr) { + return true; + } + return format::utc_timestamp_range_is_monotonic( + floor_timestamp_seconds(min_value, column_schema.type_descriptor.time_unit), + floor_timestamp_seconds(max_value, column_schema.type_descriptor.time_unit), *timezone); +} + template bool set_decoded_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, @@ -125,6 +156,12 @@ bool set_decoded_min_max(const std::shared_ptr<::parquet::Statistics>& statistic const cctz::time_zone* timezone) { auto typed_statistics = std::static_pointer_cast<::parquet::TypedStatistics>(statistics); + if constexpr (std::is_same_v) { + if (!timestamp_min_max_is_safe(column_schema, typed_statistics->min(), + typed_statistics->max(), timezone)) { + return false; + } + } if (!set_decoded_field(column_schema, value_kind, typed_statistics->min(), &column_statistics->min_value, timezone) || !set_decoded_field(column_schema, value_kind, typed_statistics->max(), @@ -969,6 +1006,12 @@ bool set_page_decoded_min_max(const std::shared_ptr<::parquet::ColumnIndex>& col page_idx >= typed_index->max_values().size()) { return false; } + if constexpr (std::is_same_v) { + if (!timestamp_min_max_is_safe(column_schema, typed_index->min_values()[page_idx], + typed_index->max_values()[page_idx], timezone)) { + return false; + } + } if (!set_decoded_field(column_schema, value_kind, typed_index->min_values()[page_idx], &page_statistics->min_value, timezone) || !set_decoded_field(column_schema, value_kind, typed_index->max_values()[page_idx], diff --git a/be/src/format_v2/timestamp_statistics.h b/be/src/format_v2/timestamp_statistics.h new file mode 100644 index 00000000000000..6bfd861c05a9d7 --- /dev/null +++ b/be/src/format_v2/timestamp_statistics.h @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/check.h" + +namespace doris::format { + +inline int64_t floor_epoch_seconds(int64_t value, int64_t units_per_second) { + DORIS_CHECK(units_per_second > 0); + auto seconds = value / units_per_second; + if (value < 0 && value % units_per_second != 0) { + --seconds; + } + return seconds; +} + +// UTC instants are ordered, but converting them to civil DATETIMEV2 values is not monotonic when +// a timezone moves its clock backward. Metadata min/max converted across such a transition cannot +// safely form a ZoneMap: the true civil minimum or maximum may occur inside the UTC interval. +inline bool utc_timestamp_range_is_monotonic(int64_t min_seconds, int64_t max_seconds, + const cctz::time_zone& timezone) { + DORIS_CHECK(min_seconds <= max_seconds); + auto current = cctz::time_point(cctz::seconds(min_seconds)); + const auto range_end = cctz::time_point(cctz::seconds(max_seconds)); + cctz::time_zone::civil_transition transition; + while (timezone.next_transition(current, &transition)) { + const auto transition_time = timezone.lookup(transition.to).trans; + if (transition_time > range_end) { + return true; + } + if (transition.to < transition.from) { + return false; + } + current = transition_time; + } + return true; +} + +} // namespace doris::format diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index e5cdd8a87457e0..6ef18ca37ce612 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -392,6 +392,34 @@ TEST(ParquetStatisticsTransformTest, ConvertsMinMaxNullCountUnsignedStringAndTim EXPECT_LT(timestamp_stats.min_value, timestamp_stats.max_value); } +TEST(ParquetStatisticsTransformTest, DisablesUtcTimestampMinMaxAcrossDstRollback) { + constexpr int64_t MICROS_PER_SECOND = 1000000; + // America/New_York moved from UTC-04:00 to UTC-05:00 at 2021-11-07 06:00:00 UTC. + // Both UTC endpoints below map to 01:30 local time, while values inside the interval cover + // 01:00 through 01:59. Endpoint conversion therefore cannot represent the true local range. + auto table = arrow::Table::Make( + arrow::schema( + {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), + {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636263900 * MICROS_PER_SECOND, + 1636266600 * MICROS_PER_SECOND})}); + auto reader = make_reader(table, 3, false, true); + auto schema = build_file_schema(*reader); + auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); + + cctz::time_zone new_york; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); + const auto local_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], statistics, &new_york); + EXPECT_TRUE(local_stats.has_not_null); + EXPECT_FALSE(local_stats.has_min_max); + + auto utc = cctz::utc_time_zone(); + const auto utc_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], statistics, &utc); + EXPECT_TRUE(utc_stats.has_min_max); + EXPECT_LT(utc_stats.min_value, utc_stats.max_value); +} + TEST(ParquetStatisticsTransformTest, HandlesMissingStatisticsAndAllNullChunks) { auto no_stats_table = arrow::Table::Make( arrow::schema({arrow::field("i", arrow::int32(), true)}), {int32_array({1, 2, 3})}); diff --git a/be/test/format_v2/timestamp_statistics_test.cpp b/be/test/format_v2/timestamp_statistics_test.cpp new file mode 100644 index 00000000000000..1eae798a13cbe8 --- /dev/null +++ b/be/test/format_v2/timestamp_statistics_test.cpp @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/timestamp_statistics.h" + +#include +#include + +namespace doris::format { + +TEST(TimestampStatisticsTest, DetectsBackwardTimezoneTransitionsInUtcRange) { + cctz::time_zone new_york; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); + + // The 2021 fall transition at 06:00 UTC makes local civil time move backward by one hour. + EXPECT_FALSE(utc_timestamp_range_is_monotonic(1636263000, 1636266600, new_york)); + EXPECT_FALSE(utc_timestamp_range_is_monotonic(1636263000, 1636264800, new_york)); + EXPECT_TRUE(utc_timestamp_range_is_monotonic(1636264800, 1636266600, new_york)); + + // The 2021 spring transition at 07:00 UTC skips civil values but preserves ordering. + EXPECT_TRUE(utc_timestamp_range_is_monotonic(1615703400, 1615707000, new_york)); +} + +TEST(TimestampStatisticsTest, FloorsNegativeEpochFractions) { + EXPECT_EQ(floor_epoch_seconds(1001, 1000), 1); + EXPECT_EQ(floor_epoch_seconds(-1, 1000), -1); + EXPECT_EQ(floor_epoch_seconds(-1001, 1000), -2); +} + +} // namespace doris::format From 833fbdf2bd6a670c09857a3a99704f8e9cc25915 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 12 Jul 2026 16:02:12 +0800 Subject: [PATCH 2/2] [fix](be) guard timestamp pruning across DST rollback Disable unsafe ORC timestamp SARG pushdown in civil years with backward timezone transitions, while preserving Parquet min/max pruning for TIMESTAMPTZ values that retain UTC ordering. Add ORC, Parquet, and timezone transition regression coverage. --- be/src/format_v2/orc/orc_search_argument.cpp | 32 ++++++- .../format_v2/parquet/parquet_statistics.cpp | 5 +- be/src/format_v2/timestamp_statistics.h | 5 +- be/test/format_v2/orc/orc_reader_test.cpp | 95 ++++++++++++++++++- .../parquet/parquet_statistics_test.cpp | 27 ++++++ .../format_v2/timestamp_statistics_test.cpp | 4 + 6 files changed, 161 insertions(+), 7 deletions(-) diff --git a/be/src/format_v2/orc/orc_search_argument.cpp b/be/src/format_v2/orc/orc_search_argument.cpp index 9c40a60ae85fa6..4766fa8304587a 100644 --- a/be/src/format_v2/orc/orc_search_argument.cpp +++ b/be/src/format_v2/orc/orc_search_argument.cpp @@ -45,6 +45,7 @@ #include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "exprs/vtopn_pred.h" +#include "format_v2/timestamp_statistics.h" namespace doris::format::orc { namespace { @@ -597,20 +598,47 @@ std::optional date_time_literal_parts(const Field& field) std::optional<::orc::Literal> make_timestamp_literal(const Field& field, const cctz::time_zone& timezone) { + const auto civil_year_is_monotonic = [&](const cctz::civil_second& civil_seconds) { + // Localized file predicates may already have selected one side of a repeated civil time. + // Checking the whole civil year catches that lossy representation while retaining SARG + // pruning for years and zones without a backward transition. + const auto year_start = + cctz::convert(cctz::civil_second(civil_seconds.year(), 1, 1), timezone); + const auto next_year_start = + cctz::convert(cctz::civil_second(civil_seconds.year() + 1, 1, 1), timezone); + return format::utc_timestamp_range_is_monotonic(year_start.time_since_epoch().count(), + next_year_start.time_since_epoch().count(), + timezone); + }; switch (field.get_type()) { case TYPE_DATETIME: { const auto& datetime = field.get(); const cctz::civil_second civil_seconds(datetime.year(), datetime.month(), datetime.day(), datetime.hour(), datetime.minute(), datetime.second()); - return ::orc::Literal(cctz::convert(civil_seconds, timezone).time_since_epoch().count(), 0); + const auto lookup = timezone.lookup(civil_seconds); + // ORC SearchArguments accept one UTC timestamp literal. A repeated or skipped local civil + // time has no unique UTC representation, so pushing it down could prune a stripe that + // contains another valid interpretation. Keep such predicates for row-level evaluation. + if (!civil_year_is_monotonic(civil_seconds) || + lookup.kind != cctz::time_zone::civil_lookup::UNIQUE) { + return std::nullopt; + } + return ::orc::Literal(lookup.pre.time_since_epoch().count(), 0); } case TYPE_DATETIMEV2: { const auto& datetime = field.get(); const cctz::civil_second civil_seconds(datetime.year(), datetime.month(), datetime.day(), datetime.hour(), datetime.minute(), datetime.second()); - const auto seconds = cctz::convert(civil_seconds, timezone).time_since_epoch().count(); + const auto lookup = timezone.lookup(civil_seconds); + // See the DATETIME path above. The fractional part does not disambiguate a civil time + // repeated by a backward timezone transition. + if (!civil_year_is_monotonic(civil_seconds) || + lookup.kind != cctz::time_zone::civil_lookup::UNIQUE) { + return std::nullopt; + } + const auto seconds = lookup.pre.time_since_epoch().count(); const auto nanos = cast_set(datetime.microsecond() * 1000); return ::orc::Literal(seconds, nanos); } diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index e188a6bf157e31..5aa5f4e4805224 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -141,7 +141,10 @@ int64_t floor_timestamp_seconds(int64_t value, ParquetTimeUnit time_unit) { bool timestamp_min_max_is_safe(const ParquetColumnSchema& column_schema, int64_t min_value, int64_t max_value, const cctz::time_zone* timezone) { if (!column_schema.type_descriptor.is_timestamp || - !column_schema.type_descriptor.timestamp_is_adjusted_to_utc || timezone == nullptr) { + !column_schema.type_descriptor.timestamp_is_adjusted_to_utc || timezone == nullptr || + remove_nullable(column_schema.type)->get_primitive_type() == TYPE_TIMESTAMPTZ) { + // TIMESTAMPTZ keeps the original UTC ordering, so local civil-time rollback does not make + // its converted min/max non-monotonic. return true; } return format::utc_timestamp_range_is_monotonic( diff --git a/be/src/format_v2/timestamp_statistics.h b/be/src/format_v2/timestamp_statistics.h index 6bfd861c05a9d7..2bbe7b99b908f7 100644 --- a/be/src/format_v2/timestamp_statistics.h +++ b/be/src/format_v2/timestamp_statistics.h @@ -51,7 +51,10 @@ inline bool utc_timestamp_range_is_monotonic(int64_t min_seconds, int64_t max_se if (transition.to < transition.from) { return false; } - current = transition_time; + // Move past the transition that was just inspected. Some cctz implementations return the + // same transition again when queried at its exact instant, which would otherwise prevent + // us from seeing a later rollback in the requested range. + current = transition_time + cctz::seconds(1); } return true; } diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 0e36498c41d9bf..a6f669e227410e 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -74,6 +74,7 @@ #include "runtime/runtime_state.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/timezone_utils.h" namespace doris { namespace { @@ -2455,6 +2456,49 @@ class NullableGreaterThanExpr final : public VExpr { const std::string _expr_name; }; +template +class NullableEqualsExpr final : public VExpr { +public: + using ColumnType = typename PrimitiveTypeTraits::ColumnType; + using ValueType = typename PrimitiveTypeTraits::CppType; + + NullableEqualsExpr(int column_id, DataTypePtr type, const Field& value, std::string column_name) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _value(value.get()), + _expr_name("NullableEqualsExpr") { + _node_type = TExprNodeType::BINARY_PRED; + _opcode = TExprOpcode::EQ; + add_child(TableSlotRef::create_shared(column_id, column_id, -1, make_nullable(type), + std::move(column_name))); + add_child(TableLiteral::create_shared(std::move(type), value)); + } + + Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, + size_t count, ColumnPtr& result_column) const override { + const auto& nullable_column = + assert_cast(*block->get_by_position(_column_id).column); + const auto& input = assert_cast(nullable_column.get_nested_column()); + auto result = ColumnUInt8::create(); + auto& result_data = result->get_data(); + result_data.resize(count); + for (size_t row = 0; row < count; ++row) { + const size_t input_row = selector == nullptr ? row : (*selector)[row]; + result_data[row] = !nullable_column.is_null_at(input_row) && + input.get_element(input_row) == _value; + } + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + +private: + const int _column_id; + const ValueType _value; + const std::string _expr_name; +}; + template class NullableInExpr final : public VExpr { public: @@ -4036,7 +4080,9 @@ void write_multi_stripe_orc_sarg_types_file(const std::string& file_path) { out.write(memory_stream.getData(), static_cast(memory_stream.getLength())); } -void write_multi_stripe_orc_timestamp_instant_sarg_file(const std::string& file_path) { +void write_multi_stripe_orc_timestamp_instant_sarg_file( + const std::string& file_path, int64_t first_timestamp_second = 0, + int64_t second_timestamp_second = 1609459200) { auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString( "struct")); @@ -4068,8 +4114,8 @@ void write_multi_stripe_orc_timestamp_instant_sarg_file(const std::string& file_ writer->add(*batch); }; - add_batch(0); - add_batch(1609459200); + add_batch(first_timestamp_second); + add_batch(second_timestamp_second); writer->close(); std::ofstream out(file_path, std::ios::binary); @@ -8327,6 +8373,49 @@ TEST_F(NewOrcReaderTest, SargTimestampInstantConjunctUsesSessionTimezone) { EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200); } +TEST_F(NewOrcReaderTest, SargTimestampInstantRepeatedCivilTimeDoesNotPruneStripes) { + const auto multi_stripe_file_path = + (_test_dir / "sarg_timestamp_instant_dst_rollback.orc").string(); + // These UTC instants both decode to 2021-11-07 01:30:00.123 in America/New_York, + // once before and once after the UTC-04:00 to UTC-05:00 rollback. + write_multi_stripe_orc_timestamp_instant_sarg_file(multi_stripe_file_path, 1636263000, + 1636266600); + ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2); + + auto reader = create_reader_for_path(multi_stripe_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TimezoneUtils::load_timezones_to_cache(); + state.set_timezone("America/New_York"); + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + + const auto literal = + Field::create_field(make_datetime_v2(2021, 11, 7, 1, 30, 0, 123000)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->conjuncts.push_back( + VExprContext::create_shared(std::make_shared>( + 0, remove_nullable(schema[0].type), literal, "timestamp_instant_col"))); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t result_rows = 0; + while (!eof) { + Block block = build_file_block({schema[0]}); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + result_rows += rows; + } + + EXPECT_EQ(result_rows, 2); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0); + EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max, 0); + EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0); +} + TEST_F(NewOrcReaderTest, SargTimestampLowerPrecisionCastDoesNotPruneStripes) { const auto multi_stripe_file_path = (_test_dir / "sarg_timestamp_lower_precision_cast.orc").string(); diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index 6ef18ca37ce612..9204535a730b4b 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -36,6 +36,7 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_timestamptz.h" #include "core/field.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" @@ -420,6 +421,32 @@ TEST(ParquetStatisticsTransformTest, DisablesUtcTimestampMinMaxAcrossDstRollback EXPECT_LT(utc_stats.min_value, utc_stats.max_value); } +TEST(ParquetStatisticsTransformTest, KeepsTimestampTzMinMaxAcrossDstRollback) { + constexpr int64_t MICROS_PER_SECOND = 1000000; + auto table = arrow::Table::Make( + arrow::schema( + {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), + {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636266600 * MICROS_PER_SECOND})}); + auto reader = make_reader(table, 2, false, true); + auto schema = build_file_schema(*reader); + auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); + + // This is the effective type produced by enable_mapping_timestamp_tz. The physical timestamp + // flags intentionally remain adjusted-to-UTC so decoding can preserve the source semantics. + schema[0]->type = std::make_shared(6); + schema[0]->type_descriptor.doris_type = schema[0]->type; + + cctz::time_zone new_york; + ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); + const auto timestamp_tz_stats = + format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], statistics, &new_york); + EXPECT_TRUE(timestamp_tz_stats.has_min_max); + EXPECT_EQ(timestamp_tz_stats.min_value.get_type(), TYPE_TIMESTAMPTZ); + EXPECT_EQ(timestamp_tz_stats.max_value.get_type(), TYPE_TIMESTAMPTZ); + EXPECT_LT(timestamp_tz_stats.min_value, timestamp_tz_stats.max_value); +} + TEST(ParquetStatisticsTransformTest, HandlesMissingStatisticsAndAllNullChunks) { auto no_stats_table = arrow::Table::Make( arrow::schema({arrow::field("i", arrow::int32(), true)}), {int32_array({1, 2, 3})}); diff --git a/be/test/format_v2/timestamp_statistics_test.cpp b/be/test/format_v2/timestamp_statistics_test.cpp index 1eae798a13cbe8..8fd6e206086206 100644 --- a/be/test/format_v2/timestamp_statistics_test.cpp +++ b/be/test/format_v2/timestamp_statistics_test.cpp @@ -31,6 +31,10 @@ TEST(TimestampStatisticsTest, DetectsBackwardTimezoneTransitionsInUtcRange) { EXPECT_FALSE(utc_timestamp_range_is_monotonic(1636263000, 1636264800, new_york)); EXPECT_TRUE(utc_timestamp_range_is_monotonic(1636264800, 1636266600, new_york)); + // A range beginning before the spring-forward transition must continue scanning and find the + // later rollback in the same year. + EXPECT_FALSE(utc_timestamp_range_is_monotonic(1609477200, 1641013200, new_york)); + // The 2021 spring transition at 07:00 UTC skips civil values but preserves ordering. EXPECT_TRUE(utc_timestamp_range_is_monotonic(1615703400, 1615707000, new_york)); }