From 30310176e4af683aa42366d9a6af95163ed20234 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 12 Jul 2026 15:05:58 +0800 Subject: [PATCH] [fix](be) Preserve row groups when null count is missing ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: FileScannerV2 treated a missing optional Parquet null_count as proof that a row group contained no NULL values. When min/max statistics were present, IS NULL predicates could therefore prune a row group that actually contained NULL rows. Treat unknown null counts conservatively as possibly containing NULL while retaining available min/max statistics, and cover the metadata combination with a unit test. ### Release note Fix incorrect IS NULL results for Parquet files whose writers omit null_count statistics. ### Check List (For Author) - Test: Unit Test - ParquetStatistics* BE unit tests - Behavior changed: Yes, row groups with unknown Parquet null counts are no longer discarded by IS NULL pruning - Does this need documentation: No --- .../format_v2/parquet/parquet_statistics.cpp | 2 +- .../parquet/parquet_statistics_test.cpp | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index d0692a80205f3f..9b48875059eb58 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -701,7 +701,7 @@ ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( return result; } - result.has_null = statistics->HasNullCount() && statistics->null_count() > 0; + result.has_null = !statistics->HasNullCount() || statistics->null_count() > 0; result.has_not_null = statistics->num_values() > 0 || statistics->HasMinMax(); result.has_null_count = statistics->HasNullCount(); if (!result.has_not_null || !statistics->HasMinMax()) { diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index e5cdd8a87457e0..87243d26e302c8 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -416,6 +416,26 @@ TEST(ParquetStatisticsTransformTest, HandlesMissingStatisticsAndAllNullChunks) { EXPECT_FALSE(all_null_stats.has_min_max); } +TEST(ParquetStatisticsTransformTest, MissingNullCountConservativelyReportsPossibleNulls) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), + {int32_array({1, std::nullopt, 3})}); + auto reader = make_reader(table, 3, false, true); + auto schema = build_file_schema(*reader); + auto file_statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); + auto statistics_without_null_count = ::parquet::MakeStatistics<::parquet::Int32Type>( + reader->metadata()->schema()->Column(0), file_statistics->EncodeMin(), + file_statistics->EncodeMax(), file_statistics->num_values(), 0, 0, true, false, false); + + const auto statistics = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], statistics_without_null_count); + EXPECT_FALSE(statistics.has_null_count); + EXPECT_TRUE(statistics.has_null); + EXPECT_TRUE(statistics.has_not_null); + EXPECT_TRUE(statistics.has_min_max); + EXPECT_EQ(statistics.min_value.get(), 1); + EXPECT_EQ(statistics.max_value.get(), 3); +} + TEST(ParquetStatisticsPruningTest, ExprZonemapPredicatesAndNullPredicatesPruneRowGroups) { auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), {int32_array({std::nullopt, std::nullopt, 3, 4, 5, 6})});