From b8a2a830032ad7f48107aa9ff397421c1f66ccfd Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 20 Aug 2026 14:06:31 +0800 Subject: [PATCH 1/3] fix: validate RLBE decoder run lengths --- cpp/src/encoding/rlbe_decoder.h | 15 ++++- cpp/test/encoding/rlbe_codec_test.cc | 43 ++++++++++++++ .../encoding/decoder/IntRLBEDecoder.java | 17 +++++- .../encoding/decoder/LongRLBEDecoder.java | 17 +++++- .../encoding/decoder/RLBEDecoderTest.java | 57 +++++++++++++++++++ 5 files changed, 146 insertions(+), 3 deletions(-) diff --git a/cpp/src/encoding/rlbe_decoder.h b/cpp/src/encoding/rlbe_decoder.h index f4028bc83..e4bb8f0ae 100644 --- a/cpp/src/encoding/rlbe_decoder.h +++ b/cpp/src/encoding/rlbe_decoder.h @@ -112,7 +112,7 @@ class RLBEDecoder : public Decoder { return ret; } block_size_ = static_cast(bits); - if (block_size_ < 0 || block_size_ > RLBE_BLOCK_DEFAULT_SIZE) { + if (block_size_ <= 0 || block_size_ > RLBE_BLOCK_DEFAULT_SIZE) { return common::E_TSFILE_CORRUPTED; } for (int i = 0; i < block_size_ * 2; ++i) { @@ -147,10 +147,23 @@ class RLBEDecoder : public Decoder { uint64_t run_length = 0; int j = 1; while (true) { + if (j >= static_cast(sizeof(fibonacci_) / + sizeof(fibonacci_[0]))) { + return common::E_TSFILE_CORRUPTED; + } if (j > 1) { fibonacci_[j] = fibonacci_[j - 1] + fibonacci_[j - 2]; + if (fibonacci_[j] <= fibonacci_[j - 1]) { + return common::E_TSFILE_CORRUPTED; + } } if (now == 1) { + const uint64_t remaining = + static_cast(block_size_ - write_index_ - 1); + if (fibonacci_[j] > remaining || + run_length > remaining - fibonacci_[j]) { + return common::E_TSFILE_CORRUPTED; + } run_length += fibonacci_[j]; } if (now == 1 && next == 1) { diff --git a/cpp/test/encoding/rlbe_codec_test.cc b/cpp/test/encoding/rlbe_codec_test.cc index f8571dc99..7712a8c73 100644 --- a/cpp/test/encoding/rlbe_codec_test.cc +++ b/cpp/test/encoding/rlbe_codec_test.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -30,6 +31,23 @@ namespace storage { +namespace { + +void append_bits(std::vector& bytes, int& bit_count, uint32_t value, + int width) { + for (int i = width - 1; i >= 0; --i) { + if (bytes.empty() || bit_count == 8) { + bytes.push_back(0); + bit_count = 0; + } + bytes.back() = + static_cast((bytes.back() << 1) | ((value >> i) & 1)); + ++bit_count; + } +} + +} // namespace + TEST(RLBECodecTest, Int32RoundTrip) { IntRLBEEncoder encoder; IntRLBEDecoder decoder; @@ -157,4 +175,29 @@ TEST(RLBECodecTest, FactoryAllocatesRLBECodecs) { DecoderFactory::free(double_decoder); } +TEST(RLBECodecTest, RejectsInvalidBlockSize) { + std::vector bytes(4, 0); + common::ByteStream stream; + stream.wrap_from(reinterpret_cast(bytes.data()), bytes.size()); + IntRLBEDecoder decoder; + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, stream), common::E_TSFILE_CORRUPTED); +} + +TEST(RLBECodecTest, RejectsRunLengthBeyondBlock) { + std::vector bytes; + int bit_count = 0; + append_bits(bytes, bit_count, 1, 32); // block size + append_bits(bytes, bit_count, 1, 6); // segment length (int32 RLBE) + append_bits(bytes, bit_count, 0b011, 3); // Fibonacci code for run length 2 + append_bits(bytes, bit_count, 0, 2); // delta payload (not reached) + bytes.back() <<= (8 - bit_count); + + common::ByteStream stream; + stream.wrap_from(reinterpret_cast(bytes.data()), bytes.size()); + IntRLBEDecoder decoder; + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, stream), common::E_TSFILE_CORRUPTED); +} + } // namespace storage diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java index ae46d3e0e..17b9a8718 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java @@ -19,11 +19,14 @@ package org.apache.tsfile.encoding.decoder; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import java.nio.ByteBuffer; public class IntRLBEDecoder extends Decoder { + private static final int MAX_BLOCK_SIZE = 10000; + /** constructor of IntRLBEDecoder */ public IntRLBEDecoder() { super(TSEncoding.RLBE); @@ -65,6 +68,9 @@ private void readhead(ByteBuffer buffer) { readindex = -1; clearBuffer(buffer); readblocksize(buffer); + if (blocksize <= 0 || blocksize > MAX_BLOCK_SIZE) { + throw new TsFileDecodingException("Invalid RLBE block size: " + blocksize); + } data = new int[blocksize * 2 + 1]; fibonacci = new int[blocksize * 2 + 1]; for (int i = 0; i < blocksize * 2; i++) { @@ -96,8 +102,17 @@ private void readT(ByteBuffer buffer) { int j = 1; while (true) { + if (j >= fibonacci.length) { + throw new TsFileDecodingException("Invalid RLBE Fibonacci run length"); + } if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2]; - if (now == 1) runlength += fibonacci[j]; + if (now == 1) { + long candidate = (long) runlength + fibonacci[j]; + if (candidate <= 0 || candidate > blocksize - writeindex - 1) { + throw new TsFileDecodingException("Invalid RLBE run length: " + candidate); + } + runlength = (int) candidate; + } // when now and next are both 1, the 1 of next is the symbol of ending of fibonacci code if (now == 1 && next == 1) break; j++; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java index f41a5cd9b..a329b2a46 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java @@ -19,11 +19,14 @@ package org.apache.tsfile.encoding.decoder; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import java.nio.ByteBuffer; public class LongRLBEDecoder extends Decoder { + private static final int MAX_BLOCK_SIZE = 10000; + /** constructor of LongRLBEDecoder */ public LongRLBEDecoder() { super(TSEncoding.RLBE); @@ -65,6 +68,9 @@ private void readhead(ByteBuffer buffer) { readindex = -1; clearBuffer(buffer); readblocksize(buffer); + if (blocksize <= 0 || blocksize > MAX_BLOCK_SIZE) { + throw new TsFileDecodingException("Invalid RLBE block size: " + blocksize); + } data = new long[blocksize * 2 + 1]; fibonacci = new long[blocksize * 2 + 1]; for (int i = 0; i < blocksize * 2; i++) { @@ -97,8 +103,17 @@ private void readT(ByteBuffer buffer) { int j = 1; while (true) { + if (j >= fibonacci.length) { + throw new TsFileDecodingException("Invalid RLBE Fibonacci run length"); + } if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2]; - if (now == 1) runlength += fibonacci[j]; + if (now == 1) { + long value = fibonacci[j]; + if (value <= 0 || runlength > blocksize - writeindex - 1 - value) { + throw new TsFileDecodingException("Invalid RLBE run length"); + } + runlength += value; + } // when now and next are both 1, the 1 of next is the symbol of ending of fibonacci code if (now == 1 && next == 1) break; j++; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java index bdb8082ee..e147bf67a 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java @@ -23,6 +23,7 @@ import org.apache.tsfile.encoding.encoder.FloatRLBE; import org.apache.tsfile.encoding.encoder.TSEncodingBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.junit.After; @@ -258,6 +259,62 @@ public void testDouble2() throws Exception { assertEquals(f, decoder.readDouble(buffer), doubleDelta); } + @Test + public void testRejectsInvalidBlockSize() { + ByteBuffer buffer = ByteBuffer.wrap(new byte[4]); + try { + new IntRLBEDecoder().readInt(buffer); + fail("Expected invalid RLBE block size to be rejected"); + } catch (TsFileDecodingException expected) { + // expected + } + } + + @Test + public void testRejectsRunLengthBeyondBlock() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + BitWriter bits = new BitWriter(output); + bits.write(1, 32); // block size + bits.write(1, 6); // segment length + bits.write(0b011, 3); // Fibonacci code for run length 2, plus terminator + bits.write(0, 2); // delta payload (not reached after validation) + bits.flush(); + + try { + new IntRLBEDecoder().readInt(ByteBuffer.wrap(output.toByteArray())); + fail("Expected RLBE run length to be rejected"); + } catch (TsFileDecodingException expected) { + // expected + } + } + + private static class BitWriter { + private final ByteArrayOutputStream output; + private int currentByte; + private int bitCount; + + private BitWriter(ByteArrayOutputStream output) { + this.output = output; + } + + private void write(int value, int width) { + for (int i = width - 1; i >= 0; i--) { + currentByte = (currentByte << 1) | ((value >>> i) & 1); + if (++bitCount == 8) { + output.write(currentByte); + currentByte = 0; + bitCount = 0; + } + } + } + + private void flush() { + if (bitCount > 0) { + output.write(currentByte << (8 - bitCount)); + } + } + } + private void testFloatLength(List valueList, boolean isDebug, int repeatCount) throws Exception { Encoder encoder = new FloatRLBE(); From 038409dcb30da4ea48727a640dc049410022bb0c Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 20 Aug 2026 18:16:33 +0800 Subject: [PATCH 2/3] fix: validate RLBE decoder segment length --- cpp/src/encoding/rlbe_decoder.h | 3 ++ cpp/test/encoding/rlbe_codec_test.cc | 38 +++++++++++++++++++ .../encoding/decoder/IntRLBEDecoder.java | 3 ++ .../encoding/decoder/LongRLBEDecoder.java | 3 ++ .../encoding/decoder/RLBEDecoderTest.java | 34 +++++++++++++++++ 5 files changed, 81 insertions(+) diff --git a/cpp/src/encoding/rlbe_decoder.h b/cpp/src/encoding/rlbe_decoder.h index e4bb8f0ae..edba87f27 100644 --- a/cpp/src/encoding/rlbe_decoder.h +++ b/cpp/src/encoding/rlbe_decoder.h @@ -136,6 +136,9 @@ class RLBEDecoder : public Decoder { return ret; } int segment_length = static_cast(bits); + if (segment_length < 1 || segment_length > Traits::VALUE_BITS) { + return common::E_TSFILE_CORRUPTED; + } int now = 0; int next = 0; diff --git a/cpp/test/encoding/rlbe_codec_test.cc b/cpp/test/encoding/rlbe_codec_test.cc index 7712a8c73..e6fb2de4f 100644 --- a/cpp/test/encoding/rlbe_codec_test.cc +++ b/cpp/test/encoding/rlbe_codec_test.cc @@ -200,4 +200,42 @@ TEST(RLBECodecTest, RejectsRunLengthBeyondBlock) { EXPECT_EQ(decoder.read_int32(value, stream), common::E_TSFILE_CORRUPTED); } +TEST(RLBECodecTest, RejectsInvalidIntSegmentLength) { + for (int segment_length : {0, 33}) { + std::vector bytes; + int bit_count = 0; + append_bits(bytes, bit_count, 1, 32); // block size + append_bits(bytes, bit_count, static_cast(segment_length), + 6); // segment length + bytes.back() <<= (8 - bit_count); + + common::ByteStream stream; + stream.wrap_from(reinterpret_cast(bytes.data()), + bytes.size()); + IntRLBEDecoder decoder; + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, stream), + common::E_TSFILE_CORRUPTED); + } +} + +TEST(RLBECodecTest, RejectsInvalidLongSegmentLength) { + for (int segment_length : {0, 65}) { + std::vector bytes; + int bit_count = 0; + append_bits(bytes, bit_count, 1, 32); // block size + append_bits(bytes, bit_count, static_cast(segment_length), + 7); // segment length + bytes.back() <<= (8 - bit_count); + + common::ByteStream stream; + stream.wrap_from(reinterpret_cast(bytes.data()), + bytes.size()); + LongRLBEDecoder decoder; + int64_t value = 0; + EXPECT_EQ(decoder.read_int64(value, stream), + common::E_TSFILE_CORRUPTED); + } +} + } // namespace storage diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java index 17b9a8718..bd046e313 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java @@ -95,6 +95,9 @@ private void readT(ByteBuffer buffer) { for (int j = 5; j >= 0; j--) { seglength |= (readbit(buffer) << j); } + if (seglength < 1 || seglength > 32) { + throw new TsFileDecodingException("Invalid RLBE segment length: " + seglength); + } // generate repeat time of rle on delta int now = readbit(buffer); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java index a329b2a46..6def046d8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java @@ -96,6 +96,9 @@ private void readT(ByteBuffer buffer) { for (int j = 6; j >= 0; j--) { seglength |= (readbit(buffer) << j); } + if (seglength < 1 || seglength > 64) { + throw new TsFileDecodingException("Invalid RLBE segment length: " + seglength); + } // generate repeat time of rle on delta int now = readbit(buffer); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java index e147bf67a..db97cc25c 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java @@ -288,6 +288,40 @@ public void testRejectsRunLengthBeyondBlock() { } } + @Test + public void testRejectsInvalidIntSegmentLength() { + for (int seglength : new int[] {0, 33}) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + BitWriter bits = new BitWriter(output); + bits.write(1, 32); // block size + bits.write(seglength, 6); // invalid segment length + bits.flush(); + try { + new IntRLBEDecoder().readInt(ByteBuffer.wrap(output.toByteArray())); + fail("Expected RLBE segment length to be rejected: " + seglength); + } catch (TsFileDecodingException expected) { + // expected + } + } + } + + @Test + public void testRejectsInvalidLongSegmentLength() { + for (int seglength : new int[] {0, 65}) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + BitWriter bits = new BitWriter(output); + bits.write(1, 32); // block size + bits.write(seglength, 7); // invalid segment length + bits.flush(); + try { + new LongRLBEDecoder().readLong(ByteBuffer.wrap(output.toByteArray())); + fail("Expected RLBE segment length to be rejected: " + seglength); + } catch (TsFileDecodingException expected) { + // expected + } + } + } + private static class BitWriter { private final ByteArrayOutputStream output; private int currentByte; From 9850d3118335c19866a864f84b126c3da34c8773 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 20 Aug 2026 18:30:31 +0800 Subject: [PATCH 3/3] fix: return decode error for malformed RLBE input --- cpp/src/encoding/rlbe_decoder.h | 10 +++++----- cpp/test/encoding/rlbe_codec_test.cc | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/src/encoding/rlbe_decoder.h b/cpp/src/encoding/rlbe_decoder.h index edba87f27..20337fa55 100644 --- a/cpp/src/encoding/rlbe_decoder.h +++ b/cpp/src/encoding/rlbe_decoder.h @@ -113,7 +113,7 @@ class RLBEDecoder : public Decoder { } block_size_ = static_cast(bits); if (block_size_ <= 0 || block_size_ > RLBE_BLOCK_DEFAULT_SIZE) { - return common::E_TSFILE_CORRUPTED; + return common::E_DECODE_ERR; } for (int i = 0; i < block_size_ * 2; ++i) { data_[i] = 0; @@ -137,7 +137,7 @@ class RLBEDecoder : public Decoder { } int segment_length = static_cast(bits); if (segment_length < 1 || segment_length > Traits::VALUE_BITS) { - return common::E_TSFILE_CORRUPTED; + return common::E_DECODE_ERR; } int now = 0; @@ -152,12 +152,12 @@ class RLBEDecoder : public Decoder { while (true) { if (j >= static_cast(sizeof(fibonacci_) / sizeof(fibonacci_[0]))) { - return common::E_TSFILE_CORRUPTED; + return common::E_DECODE_ERR; } if (j > 1) { fibonacci_[j] = fibonacci_[j - 1] + fibonacci_[j - 2]; if (fibonacci_[j] <= fibonacci_[j - 1]) { - return common::E_TSFILE_CORRUPTED; + return common::E_DECODE_ERR; } } if (now == 1) { @@ -165,7 +165,7 @@ class RLBEDecoder : public Decoder { static_cast(block_size_ - write_index_ - 1); if (fibonacci_[j] > remaining || run_length > remaining - fibonacci_[j]) { - return common::E_TSFILE_CORRUPTED; + return common::E_DECODE_ERR; } run_length += fibonacci_[j]; } diff --git a/cpp/test/encoding/rlbe_codec_test.cc b/cpp/test/encoding/rlbe_codec_test.cc index e6fb2de4f..9c0a819d3 100644 --- a/cpp/test/encoding/rlbe_codec_test.cc +++ b/cpp/test/encoding/rlbe_codec_test.cc @@ -181,7 +181,7 @@ TEST(RLBECodecTest, RejectsInvalidBlockSize) { stream.wrap_from(reinterpret_cast(bytes.data()), bytes.size()); IntRLBEDecoder decoder; int32_t value = 0; - EXPECT_EQ(decoder.read_int32(value, stream), common::E_TSFILE_CORRUPTED); + EXPECT_EQ(decoder.read_int32(value, stream), common::E_DECODE_ERR); } TEST(RLBECodecTest, RejectsRunLengthBeyondBlock) { @@ -197,7 +197,7 @@ TEST(RLBECodecTest, RejectsRunLengthBeyondBlock) { stream.wrap_from(reinterpret_cast(bytes.data()), bytes.size()); IntRLBEDecoder decoder; int32_t value = 0; - EXPECT_EQ(decoder.read_int32(value, stream), common::E_TSFILE_CORRUPTED); + EXPECT_EQ(decoder.read_int32(value, stream), common::E_DECODE_ERR); } TEST(RLBECodecTest, RejectsInvalidIntSegmentLength) { @@ -215,7 +215,7 @@ TEST(RLBECodecTest, RejectsInvalidIntSegmentLength) { IntRLBEDecoder decoder; int32_t value = 0; EXPECT_EQ(decoder.read_int32(value, stream), - common::E_TSFILE_CORRUPTED); + common::E_DECODE_ERR); } } @@ -234,7 +234,7 @@ TEST(RLBECodecTest, RejectsInvalidLongSegmentLength) { LongRLBEDecoder decoder; int64_t value = 0; EXPECT_EQ(decoder.read_int64(value, stream), - common::E_TSFILE_CORRUPTED); + common::E_DECODE_ERR); } }