Skip to content

Partition pruning works incorrectly #2240

Description

@ianton-ru
SELECT count() FROM ice.`otel.logs`
WHERE time_unix_nano <= toDateTime64('2026-08-11 20:02:07.897356', 6);

returns 0;

SELECT count() FROM ice.`otel.logs`
WHERE time_unix_nano <= toDateTime64('2026-08-11 20:02:07.897356', 6)
SETTINGS use_iceberg_partition_pruning = 0;

returns 10837020 ( which is correct)

SHOW CREATE TABLE ice.`otel.logs`

CREATE TABLE ice.`otel.logs`
(
    `id` Nullable(Int64),
    `resource` Tuple(id Nullable(Int32), schema_url Nullable(String), dropped_attributes_count Nullable(Int64)),
    `scope` Tuple(id Nullable(Int32), name Nullable(String), version Nullable(String), dropped_attributes_count Nullable(Int64)),
    `schema_url` String,
    `time_unix_nano` DateTime64(6),
    `observed_time_unix_nano` DateTime64(6),
    `trace_id` Nullable(FixedString(16)),
    `span_id` Nullable(FixedString(8)),
    `severity_number` Nullable(Int32),
    `severity_text` Nullable(String),
    `body` Tuple(type Int32, str Nullable(String), int Nullable(Int64), double Nullable(Float64), bool Nullable(Bool), bytes Nullable(String), ser
Nullable(String)),
    `dropped_attributes_count` Int64,
    `event_name` Nullable(String),
    `flags` Int64
)
ENGINE = Iceberg('s3://alti-dev-fwh9f5hn-iceberg/otel/logs')

ProfileEvents for wrong query:

{'Query':1,'SelectQuery':1,'InitialQuery':1,'InitialSelectQuery':1,'QueriesWithSubqueries':2,'SelectQueriesWithSubqueries':2,'IOBufferAllocs':7,'IOBufferAllocBytes':6312281,'ArenaAllocChunks':2,'ArenaAllocBytes':8192,'FunctionExecute':2,'IcebergMetadataFilesCacheHits':18,'IcebergMetadataReadWaitTimeMicroseconds':4135,'IcebergIteratorInitializationMicroseconds':521,'IcebergMetadataUpdateMicroseconds':565,'NetworkSendElapsedMicroseconds':123,'NetworkSendBytes':925,'GlobalThreadPoolJobs':6,'LocalThreadPoolExpansions':4,'LocalThreadPoolShrinks':2,'LocalThreadPoolThreadCreationMicroseconds':115,'LocalThreadPoolJobs':6,'IcebergMinMaxIndexPrunedFiles':28,'IcebergJsonFileParsing':4,'IcebergJsonFileParsingMicroseconds':65976,'QueryPlanOptimizeMicroseconds':248,'ContextLock':53,'RWLockAcquiredReadLocks':2,'RealTimeMicroseconds':121651,'UserTimeMicroseconds':13514,'SystemTimeMicroseconds':2338,'SoftPageFaults':2,'OSCPUWaitMicroseconds':3104,'OSCPUVirtualTimeMicroseconds':15852,'OSReadChars':31486,'OSWriteChars':5472,'S3ReadMicroseconds':59599,'S3ReadRequestsCount':2,'S3HeadObject':1,'S3HeadObjectMicroseconds':43476,'S3GetObject':1,'S3Clients':1,'ReadBufferFromS3Microseconds':20850,'ReadBufferFromS3InitMicroseconds':19510,'ReadBufferFromS3Bytes':20384,'RemoteFSUnprefetchedReads':1,'RemoteFSUnprefetchedBytes':20384,'RemoteFSBuffers':1,'ThreadpoolReaderTaskMicroseconds':20852,'ThreadpoolReaderReadBytes':20384,'SynchronousRemoteReadWaitMicroseconds':20855,'LogDebug':14,'LoggerElapsedNanoseconds':868650,'InterfaceHTTPSendBytes':925,'StorageConnectionsCreated':1,'StorageConnectionsReused':1,'StorageConnectionsPreserved':2,'StorageConnectionsExpired':1,'StorageConnectionsElapsedMicroseconds':2426,'HTTPConnectionsCreated':1,'HTTPConnectionsPreserved':1,'HTTPConnectionsExpired':1,'HTTPConnectionsElapsedMicroseconds':456,'AddressesDiscovered':7,'ReadWriteBufferFromHTTPRequestsSent':1,'ReadWriteBufferFromHTTPBytes':17111,'ConcurrencyControlSlotsGranted':1,'ConcurrencyControlSlotsAcquired':1,'ConcurrencyControlSlotsAcquiredNonCompeting':1,'AsyncLoggingConsoleTotalMessages':14,'AsyncLoggingFileLogTotalMessages':14,'AsyncLoggingTextLogTotalMessages':14}

Metadata:

TOKEN=$(cat ~/otel-ice-token | tr -d '[:space:]')
curl -sSk -H "Authorization: Bearer ${TOKEN}" \
  "https://iceberg-catalog.altinity-billing-dev.dev.altinity.cloud/v1/namespaces/otel/tables/logs" | \
  python3 -c "import json,sys; d=json.load(sys.stdin); print(json.dumps({'partition-specs': d['metadata'].get('partition-specs'), 'default-spec-id':
d['metadata'].get('default-spec-id'), 'sort-orders': d['metadata'].get('sort-orders')}, indent=2))"
{
  "partition-specs": [
    {
      "spec-id": 0,
      "fields": []
    }
  ],
  "default-spec-id": 0,
  "sort-orders": [
    {
      "order-id": 0,
      "fields": []
    }
  ]
}

and

SELECT
    reinterpretAsInt64(data_file.lower_bounds.value[indexOf(data_file.lower_bounds.key, 5)]) AS raw_lower_int64,
    reinterpretAsInt64(data_file.upper_bounds.value[indexOf(data_file.upper_bounds.key, 5)]) AS raw_upper_int64,
    fromUnixTimestamp64Nano(raw_lower_int64) AS as_nanoseconds_lower,
    fromUnixTimestamp64Micro(raw_lower_int64) AS as_microseconds_lower
FROM s3('s3://alti-dev-fwh9f5hn-iceberg/otel/logs/metadata/8abad27c-83d0-4981-bc76-4a9002438344-m0.avro', 'Avro')

Query id: 3f1da1d7-041d-4cdd-bf85-02f78f95f321

   ┌─────raw_lower_int64─┬─────raw_upper_int64─┬──────────as_nanoseconds_lower─┬──────as_microseconds_lower─┐
1. │ 1786474926576815622 │ 1786478527897356702 │ 2026-08-11 19:02:06.576815622 │ 2299-12-31 23:09:36.815622 │
   └─────────────────────┴─────────────────────┴───────────────────────────────┴────────────────────────────┘

Metadata

Metadata

Assignees

Labels

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions