Skip to content

Core: Align content stats fields with latest Spec changes #16439

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nastra wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Core: Align content stats fields with latest Spec changes #16439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c7c7fee
Core: Align content stats fields with latest Spec changes
nastra May 19, 2026
f0c18f6
Core: Handle Geo & Variant types in content stats
nastra May 19, 2026
51be71e
some additional tests for geo/variant
nastra May 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 20 additions & 22 deletions core/src/main/java/org/apache/iceberg/BaseContentStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,25 +124,6 @@ public <T> void set(int pos, T value) {
FieldStats<?> stat = fieldStats.get(pos);
BaseFieldStats.Builder builder = BaseFieldStats.buildFrom(stat);
Type type = stat.type();
if (null != record.getField(FieldStatistic.VALUE_COUNT.fieldName())) {
builder.valueCount((Long) record.getField(FieldStatistic.VALUE_COUNT.fieldName()));
}

if (null != record.getField(FieldStatistic.NAN_VALUE_COUNT.fieldName())) {
builder.nanValueCount((Long) record.getField(FieldStatistic.NAN_VALUE_COUNT.fieldName()));
}

if (null != record.getField(FieldStatistic.NULL_VALUE_COUNT.fieldName())) {
builder.nullValueCount((Long) record.getField(FieldStatistic.NULL_VALUE_COUNT.fieldName()));
}

if (null != record.getField(FieldStatistic.AVG_VALUE_SIZE.fieldName())) {
builder.avgValueSize((Integer) record.getField(FieldStatistic.AVG_VALUE_SIZE.fieldName()));
}

if (null != record.getField(FieldStatistic.MAX_VALUE_SIZE.fieldName())) {
builder.maxValueSize((Integer) record.getField(FieldStatistic.MAX_VALUE_SIZE.fieldName()));
}

Object lowerBound = record.getField(FieldStatistic.LOWER_BOUND.fieldName());
if (null != type && null != lowerBound) {
Expand All @@ -164,9 +145,26 @@ public <T> void set(int pos, T value) {
builder.upperBound(type.typeId().javaClass().cast(upperBound));
}

if (null != record.getField(FieldStatistic.EXACT_BOUNDS.fieldName())) {
Boolean exactBounds = (Boolean) record.getField(FieldStatistic.EXACT_BOUNDS.fieldName());
builder.hasExactBounds(null != exactBounds && exactBounds);
if (null != record.getField(FieldStatistic.TIGHT_BOUNDS.fieldName())) {
Boolean tightBounds = (Boolean) record.getField(FieldStatistic.TIGHT_BOUNDS.fieldName());
builder.tightBounds(null != tightBounds && tightBounds);
}

if (null != record.getField(FieldStatistic.VALUE_COUNT.fieldName())) {
builder.valueCount((Long) record.getField(FieldStatistic.VALUE_COUNT.fieldName()));
}

if (null != record.getField(FieldStatistic.NULL_VALUE_COUNT.fieldName())) {
builder.nullValueCount((Long) record.getField(FieldStatistic.NULL_VALUE_COUNT.fieldName()));
}

if (null != record.getField(FieldStatistic.NAN_VALUE_COUNT.fieldName())) {
builder.nanValueCount((Long) record.getField(FieldStatistic.NAN_VALUE_COUNT.fieldName()));
}

if (null != record.getField(FieldStatistic.AVG_VALUE_SIZE_IN_BYTES.fieldName())) {
builder.avgValueSizeInBytes(
(Integer) record.getField(FieldStatistic.AVG_VALUE_SIZE_IN_BYTES.fieldName()));
}

BaseFieldStats<?> newStat = builder.build();
Expand Down
122 changes: 51 additions & 71 deletions core/src/main/java/org/apache/iceberg/BaseFieldStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,38 +32,35 @@ class BaseFieldStats<T> extends SupportsIndexProjection implements FieldStats<T>
private static final int[] IDENTITY_MAPPING = identityMapping();
private final int fieldId;
private final Type type;
private final T lowerBound;
private final T upperBound;
private final boolean tightBounds;
private final Long valueCount;
private final Long nullValueCount;
private final Long nanValueCount;
private final Integer avgValueSize;
private final Integer maxValueSize;
private final T lowerBound;
private final T upperBound;
private final boolean hasExactBounds;
private final Integer avgValueSizeInBytes;

private BaseFieldStats(
int fieldId,
int[] fromProjectionPos,
Type type,
T lowerBound,
T upperBound,
boolean tightBounds,
Long valueCount,
Long nullValueCount,
Long nanValueCount,
Integer avgValueSize,
Integer maxValueSize,
T lowerBound,
T upperBound,
boolean hasExactBounds) {
Integer avgValueSizeInBytes) {
super(fromProjectionPos != null ? fromProjectionPos : IDENTITY_MAPPING);
this.fieldId = fieldId;
this.type = type;
this.lowerBound = lowerBound;
this.upperBound = upperBound;
this.tightBounds = tightBounds;
this.valueCount = valueCount;
this.nullValueCount = nullValueCount;
this.nanValueCount = nanValueCount;
this.avgValueSize = avgValueSize;
this.maxValueSize = maxValueSize;
this.lowerBound = lowerBound;
this.upperBound = upperBound;
this.hasExactBounds = hasExactBounds;
this.avgValueSizeInBytes = avgValueSizeInBytes;
}

private static int[] identityMapping() {
Expand All @@ -77,7 +74,7 @@ private static int[] identityMapping() {
}

/**
* Computes a position mapping from the column-specific stats struct to the full 8-field struct.
* Computes a position mapping from the column-specific stats struct to the full stats struct.
* Each entry maps a projected position to its base position (0-based) using the field ID offsets
* from the column's base stats field ID.
*/
Expand Down Expand Up @@ -122,13 +119,8 @@ public Long nanValueCount() {
}

@Override
public Integer avgValueSize() {
return avgValueSize;
}

@Override
public Integer maxValueSize() {
return maxValueSize;
public Integer avgValueSizeInBytes() {
return avgValueSizeInBytes;
}

@SuppressWarnings("unchecked")
Expand Down Expand Up @@ -173,21 +165,20 @@ public T upperBound() {
}

@Override
public boolean hasExactBounds() {
return hasExactBounds;
public boolean tightBounds() {
return tightBounds;
}

@Override
protected <X> X internalGet(int pos, Class<X> javaClass) {
return switch (FieldStatistic.fromPosition(pos)) {
case LOWER_BOUND -> javaClass.cast(lowerBound());
case UPPER_BOUND -> javaClass.cast(upperBound());
case TIGHT_BOUNDS -> javaClass.cast(tightBounds);
case VALUE_COUNT -> javaClass.cast(valueCount);
case NULL_VALUE_COUNT -> javaClass.cast(nullValueCount);
case NAN_VALUE_COUNT -> javaClass.cast(nanValueCount);
case AVG_VALUE_SIZE -> javaClass.cast(avgValueSize);
case MAX_VALUE_SIZE -> javaClass.cast(maxValueSize);
case LOWER_BOUND -> javaClass.cast(lowerBound());
case UPPER_BOUND -> javaClass.cast(upperBound());
case EXACT_BOUNDS -> javaClass.cast(hasExactBounds);
case AVG_VALUE_SIZE_IN_BYTES -> javaClass.cast(avgValueSizeInBytes);
default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos);
};
}
Expand All @@ -202,14 +193,13 @@ public String toString() {
return MoreObjects.toStringHelper(this)
.add("fieldId", fieldId)
.add("type", type)
.add("lowerBound", lowerBound)
.add("upperBound", upperBound)
.add("tightBounds", tightBounds)
.add("valueCount", valueCount)
.add("nullValueCount", nullValueCount)
.add("nanValueCount", nanValueCount)
.add("avgValueSize", avgValueSize)
.add("maxValueSize", maxValueSize)
.add("lowerBound", lowerBound)
.add("upperBound", upperBound)
.add("hasExactBounds", hasExactBounds)
.add("avgValueSizeInBytes", avgValueSizeInBytes)
.toString();
}

Expand All @@ -221,30 +211,28 @@ public boolean equals(Object o) {

BaseFieldStats<?> that = (BaseFieldStats<?>) o;
return fieldId == that.fieldId
&& tightBounds == that.tightBounds
&& Objects.equals(type, that.type)
&& Objects.deepEquals(lowerBound, that.lowerBound)
&& Objects.deepEquals(upperBound, that.upperBound)
&& Objects.equals(valueCount, that.valueCount)
&& Objects.equals(nullValueCount, that.nullValueCount)
&& Objects.equals(nanValueCount, that.nanValueCount)
&& Objects.equals(avgValueSize, that.avgValueSize)
&& Objects.equals(maxValueSize, that.maxValueSize)
&& Objects.deepEquals(lowerBound, that.lowerBound)
&& Objects.deepEquals(upperBound, that.upperBound)
&& hasExactBounds == that.hasExactBounds;
&& Objects.equals(avgValueSizeInBytes, that.avgValueSizeInBytes);
}

@Override
public int hashCode() {
return Objects.hash(
fieldId,
type,
lowerBound,
upperBound,
tightBounds,
valueCount,
nullValueCount,
nanValueCount,
avgValueSize,
maxValueSize,
lowerBound,
upperBound,
hasExactBounds);
avgValueSizeInBytes);
}

public static <X> Builder<X> builder() {
Expand All @@ -256,28 +244,26 @@ public static <X> Builder<X> buildFrom(FieldStats<X> value) {
return BaseFieldStats.<X>builder()
.type(value.type())
.fieldId(value.fieldId())
.lowerBound(value.lowerBound())
.upperBound(value.upperBound())
.tightBounds(value.tightBounds())
.valueCount(value.valueCount())
.nullValueCount(value.nullValueCount())
.nanValueCount(value.nanValueCount())
.avgValueSize(value.avgValueSize())
.maxValueSize(value.maxValueSize())
.lowerBound(value.lowerBound())
.upperBound(value.upperBound())
.hasExactBounds(value.hasExactBounds());
.avgValueSizeInBytes(value.avgValueSizeInBytes());
}

public static class Builder<T> {
private int fieldId;
private int[] fromProjectionPos;
private Type type;
private T lowerBound;
private T upperBound;
private boolean tightBounds;
private Long valueCount;
private Long nullValueCount;
private Long nanValueCount;
private Integer avgValueSize;
private Integer maxValueSize;
private T lowerBound;
private T upperBound;
private boolean hasExactBounds;
private Integer avgValueSizeInBytes;

private Builder() {}

Expand Down Expand Up @@ -306,13 +292,8 @@ public Builder<T> nanValueCount(Long newNanValueCount) {
return this;
}

public Builder<T> avgValueSize(Integer newAvgValueSize) {
this.avgValueSize = newAvgValueSize;
return this;
}

public Builder<T> maxValueSize(Integer newMaxValueSize) {
this.maxValueSize = newMaxValueSize;
public Builder<T> avgValueSizeInBytes(Integer newAvgValueSizeInBytes) {
this.avgValueSizeInBytes = newAvgValueSizeInBytes;
return this;
}

Expand All @@ -331,13 +312,13 @@ public Builder<T> fieldId(int newFieldId) {
return this;
}

public Builder<T> hasExactBounds(boolean newHasExactBounds) {
this.hasExactBounds = newHasExactBounds;
public Builder<T> tightBounds(boolean newTightBounds) {
this.tightBounds = newTightBounds;
return this;
}

public Builder<T> hasExactBounds() {
this.hasExactBounds = true;
public Builder<T> tightBounds() {
this.tightBounds = true;
return this;
}

Expand Down Expand Up @@ -370,14 +351,13 @@ public BaseFieldStats<T> build() {
fieldId,
fromProjectionPos,
type,
lowerBound,
upperBound,
tightBounds,
valueCount,
nullValueCount,
nanValueCount,
avgValueSize,
maxValueSize,
lowerBound,
upperBound,
hasExactBounds);
avgValueSizeInBytes);
}
}
}
Loading