Skip to content

[core] Fix BSI reader predicate pruning for Long.MIN_VALUE boundary #8150

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 2 commits into from
Jun 7, 2026
+105 −6
Merged

[core] Fix BSI reader predicate pruning for Long.MIN_VALUE boundary #8150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
48bd8d9
[core] Fix BSI reader predicate pruning for Long.MIN_VALUE boundary
lxy-9602 Jun 7, 2026
bb7e1c1
apply spotless
lxy-9602 Jun 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions ...on-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,11 @@ public FileIndexResult visitIn(FieldRef fieldRef, List<Object> literals) {
.map(valueMapper)
.map(
value -> {
if (value < 0) {
if (value == Long.MIN_VALUE) {
// Writer cannot store Long.MIN_VALUE, so no
// row can match it
return new RoaringBitmap32();
} else if (value < 0) {
return negative.eq(Math.abs(value));
} else {
return positive.eq(value);
Expand All @@ -262,7 +266,9 @@ public FileIndexResult visitNotIn(FieldRef fieldRef, List<Object> literals) {
.map(valueMapper)
.map(
value -> {
if (value < 0) {
if (value == Long.MIN_VALUE) {
return new RoaringBitmap32();
} else if (value < 0) {
return negative.eq(Math.abs(value));
} else {
return positive.eq(value);
Expand All @@ -280,7 +286,10 @@ public FileIndexResult visitLessThan(FieldRef fieldRef, Object literal) {
return new BitmapIndexResult(
() -> {
Long value = valueMapper.apply(literal);
if (value < 0) {
if (value == Long.MIN_VALUE) {
// Nothing is less than Long.MIN_VALUE
return new RoaringBitmap32();
} else if (value < 0) {
return negative.gt(Math.abs(value));
} else {
return RoaringBitmap32.or(positive.lt(value), negative.isNotNull());
Expand All @@ -293,7 +302,10 @@ public BitmapIndexResult visitLessOrEqual(FieldRef fieldRef, Object literal) {
return new BitmapIndexResult(
() -> {
Long value = valueMapper.apply(literal);
if (value < 0) {
if (value == Long.MIN_VALUE) {
// Writer cannot store Long.MIN_VALUE, so no row can match
return new RoaringBitmap32();
} else if (value < 0) {
return negative.gte(Math.abs(value));
} else {
return RoaringBitmap32.or(positive.lte(value), negative.isNotNull());
Expand All @@ -306,7 +318,10 @@ public FileIndexResult visitGreaterThan(FieldRef fieldRef, Object literal) {
return new BitmapIndexResult(
() -> {
Long value = valueMapper.apply(literal);
if (value < 0) {
if (value == Long.MIN_VALUE) {
// Everything is greater than Long.MIN_VALUE (writer cannot store it)
return RoaringBitmap32.or(positive.isNotNull(), negative.isNotNull());
} else if (value < 0) {
return RoaringBitmap32.or(
positive.isNotNull(), negative.lt(Math.abs(value)));
} else {
Expand All @@ -320,7 +335,10 @@ public BitmapIndexResult visitGreaterOrEqual(FieldRef fieldRef, Object literal)
return new BitmapIndexResult(
() -> {
Long value = valueMapper.apply(literal);
if (value < 0) {
if (value == Long.MIN_VALUE) {
// All non-null rows satisfy x >= Long.MIN_VALUE
return RoaringBitmap32.or(positive.isNotNull(), negative.isNotNull());
} else if (value < 0) {
return RoaringBitmap32.or(
positive.isNotNull(), negative.lte(Math.abs(value)));
} else {
Expand Down
81 changes: 81 additions & 0 deletions ...ommon/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.paimon.fileindex.bitmap.BitmapIndexResult;
import org.apache.paimon.fs.ByteArraySeekableStream;
import org.apache.paimon.predicate.FieldRef;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.IntType;
import org.apache.paimon.utils.RoaringBitmap32;

Expand All @@ -31,6 +32,7 @@
import java.util.Arrays;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

/** test for {@link BitSliceIndexBitmapFileIndex}. */
public class BitSliceIndexBitmapFileIndexTest {
Expand Down Expand Up @@ -250,4 +252,83 @@ public void testBitSliceIndexNegativeOnly() {
assertThat(((BitmapIndexResult) reader.visitGreaterOrEqual(fieldRef, 1)).get())
.isEqualTo(RoaringBitmap32.bitmapOf());
}

@Test
public void testReaderPredicatePruningWithLongMinValue() {
BigIntType bigIntType = new BigIntType();
FieldRef fieldRef = new FieldRef(0, "", bigIntType);
BitSliceIndexBitmapFileIndex bsiFileIndex = new BitSliceIndexBitmapFileIndex(bigIntType);
FileIndexWriter writer = bsiFileIndex.createWriter();

// Use values that include negative numbers but NOT Long.MIN_VALUE itself
// (since the writer cannot handle it). This isolates the reader-side bug.
// Data: [-100, -1, null, 0, 1, 50]
Object[] arr = {-100L, -1L, null, 0L, 1L, 50L};

for (Object o : arr) {
writer.write(o);
}
byte[] bytes = writer.serializedBytes();
ByteArraySeekableStream stream = new ByteArraySeekableStream(bytes);
FileIndexReader reader = bsiFileIndex.createReader(stream, 0, bytes.length);

// All non-null row ids: {0, 1, 3, 4, 5}

// x > Long.MIN_VALUE: every int64 value > Long.MIN_VALUE (since no row IS Long.MIN_VALUE),
// so result should be ALL non-null rows = {0, 1, 3, 4, 5}
RoaringBitmap32 gtResult =
((BitmapIndexResult) reader.visitGreaterThan(fieldRef, Long.MIN_VALUE)).get();
assertThat(gtResult)
.as("x > Long.MIN_VALUE should return all non-null rows")
.isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5));

// x >= Long.MIN_VALUE: same — all non-null rows satisfy this
RoaringBitmap32 gteResult =
((BitmapIndexResult) reader.visitGreaterOrEqual(fieldRef, Long.MIN_VALUE)).get();
assertThat(gteResult)
.as("x >= Long.MIN_VALUE should return all non-null rows")
.isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5));

// x < Long.MIN_VALUE: no int64 value is less than Long.MIN_VALUE, so result should be
// empty
RoaringBitmap32 ltResult =
((BitmapIndexResult) reader.visitLessThan(fieldRef, Long.MIN_VALUE)).get();
assertThat(ltResult)
.as("x < Long.MIN_VALUE should return empty")
.isEqualTo(RoaringBitmap32.bitmapOf());

// x <= Long.MIN_VALUE: no row has Long.MIN_VALUE, so result should be empty
RoaringBitmap32 lteResult =
((BitmapIndexResult) reader.visitLessOrEqual(fieldRef, Long.MIN_VALUE)).get();
assertThat(lteResult)
.as("x <= Long.MIN_VALUE should return empty (no row has that value)")
.isEqualTo(RoaringBitmap32.bitmapOf());

// x == Long.MIN_VALUE: no row has Long.MIN_VALUE, so result should be empty
RoaringBitmap32 eqResult =
((BitmapIndexResult) reader.visitEqual(fieldRef, Long.MIN_VALUE)).get();
assertThat(eqResult)
.as("x == Long.MIN_VALUE should return empty")
.isEqualTo(RoaringBitmap32.bitmapOf());

// x != Long.MIN_VALUE: all non-null rows (no row has Long.MIN_VALUE)
RoaringBitmap32 neqResult =
((BitmapIndexResult) reader.visitNotEqual(fieldRef, Long.MIN_VALUE)).get();
assertThat(neqResult)
.as("x != Long.MIN_VALUE should return all non-null rows")
.isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5));
}

@Test
public void testWriterCannotHandleLongMinValue() {
BigIntType bigIntType = new BigIntType();
BitSliceIndexBitmapFileIndex bsiFileIndex = new BitSliceIndexBitmapFileIndex(bigIntType);
FileIndexWriter writer = bsiFileIndex.createWriter();
writer.write(Long.MIN_VALUE);

assertThatThrownBy(writer::serializedBytes)
.isInstanceOf(RuntimeException.class)
.hasCauseInstanceOf(IllegalArgumentException.class)
.hasRootCauseMessage("values should be non-negative");
}
}
Loading