diff --git a/cpp/src/encoding/rlbe_decoder.h b/cpp/src/encoding/rlbe_decoder.h index f4028bc83..20337fa55 100644 --- a/cpp/src/encoding/rlbe_decoder.h +++ b/cpp/src/encoding/rlbe_decoder.h @@ -112,8 +112,8 @@ class RLBEDecoder : public Decoder { return ret; } block_size_ = static_cast(bits); - if (block_size_ < 0 || block_size_ > RLBE_BLOCK_DEFAULT_SIZE) { - return common::E_TSFILE_CORRUPTED; + if (block_size_ <= 0 || block_size_ > RLBE_BLOCK_DEFAULT_SIZE) { + return common::E_DECODE_ERR; } for (int i = 0; i < block_size_ * 2; ++i) { data_[i] = 0; @@ -136,6 +136,9 @@ class RLBEDecoder : public Decoder { return ret; } int segment_length = static_cast(bits); + if (segment_length < 1 || segment_length > Traits::VALUE_BITS) { + return common::E_DECODE_ERR; + } int now = 0; int next = 0; @@ -147,10 +150,23 @@ class RLBEDecoder : public Decoder { uint64_t run_length = 0; int j = 1; while (true) { + if (j >= static_cast(sizeof(fibonacci_) / + sizeof(fibonacci_[0]))) { + return common::E_DECODE_ERR; + } if (j > 1) { fibonacci_[j] = fibonacci_[j - 1] + fibonacci_[j - 2]; + if (fibonacci_[j] <= fibonacci_[j - 1]) { + return common::E_DECODE_ERR; + } } if (now == 1) { + const uint64_t remaining = + static_cast(block_size_ - write_index_ - 1); + if (fibonacci_[j] > remaining || + run_length > remaining - fibonacci_[j]) { + return common::E_DECODE_ERR; + } run_length += fibonacci_[j]; } if (now == 1 && next == 1) { diff --git a/cpp/test/encoding/rlbe_codec_test.cc b/cpp/test/encoding/rlbe_codec_test.cc index f8571dc99..9c0a819d3 100644 --- a/cpp/test/encoding/rlbe_codec_test.cc +++ b/cpp/test/encoding/rlbe_codec_test.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -30,6 +31,23 @@ namespace storage { +namespace { + +void append_bits(std::vector& bytes, int& bit_count, uint32_t value, + int width) { + for (int i = width - 1; i >= 0; --i) { + if (bytes.empty() || bit_count == 8) { + bytes.push_back(0); + bit_count = 0; + } + bytes.back() = + static_cast((bytes.back() << 1) | ((value >> i) & 1)); + ++bit_count; + } +} + +} // namespace + TEST(RLBECodecTest, Int32RoundTrip) { IntRLBEEncoder encoder; IntRLBEDecoder decoder; @@ -157,4 +175,67 @@ TEST(RLBECodecTest, FactoryAllocatesRLBECodecs) { DecoderFactory::free(double_decoder); } +TEST(RLBECodecTest, RejectsInvalidBlockSize) { + std::vector bytes(4, 0); + common::ByteStream stream; + stream.wrap_from(reinterpret_cast(bytes.data()), bytes.size()); + IntRLBEDecoder decoder; + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, stream), common::E_DECODE_ERR); +} + +TEST(RLBECodecTest, RejectsRunLengthBeyondBlock) { + std::vector bytes; + int bit_count = 0; + append_bits(bytes, bit_count, 1, 32); // block size + append_bits(bytes, bit_count, 1, 6); // segment length (int32 RLBE) + append_bits(bytes, bit_count, 0b011, 3); // Fibonacci code for run length 2 + append_bits(bytes, bit_count, 0, 2); // delta payload (not reached) + bytes.back() <<= (8 - bit_count); + + common::ByteStream stream; + stream.wrap_from(reinterpret_cast(bytes.data()), bytes.size()); + IntRLBEDecoder decoder; + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, stream), common::E_DECODE_ERR); +} + +TEST(RLBECodecTest, RejectsInvalidIntSegmentLength) { + for (int segment_length : {0, 33}) { + std::vector bytes; + int bit_count = 0; + append_bits(bytes, bit_count, 1, 32); // block size + append_bits(bytes, bit_count, static_cast(segment_length), + 6); // segment length + bytes.back() <<= (8 - bit_count); + + common::ByteStream stream; + stream.wrap_from(reinterpret_cast(bytes.data()), + bytes.size()); + IntRLBEDecoder decoder; + int32_t value = 0; + EXPECT_EQ(decoder.read_int32(value, stream), + common::E_DECODE_ERR); + } +} + +TEST(RLBECodecTest, RejectsInvalidLongSegmentLength) { + for (int segment_length : {0, 65}) { + std::vector bytes; + int bit_count = 0; + append_bits(bytes, bit_count, 1, 32); // block size + append_bits(bytes, bit_count, static_cast(segment_length), + 7); // segment length + bytes.back() <<= (8 - bit_count); + + common::ByteStream stream; + stream.wrap_from(reinterpret_cast(bytes.data()), + bytes.size()); + LongRLBEDecoder decoder; + int64_t value = 0; + EXPECT_EQ(decoder.read_int64(value, stream), + common::E_DECODE_ERR); + } +} + } // namespace storage diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java index ae46d3e0e..bd046e313 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntRLBEDecoder.java @@ -19,11 +19,14 @@ package org.apache.tsfile.encoding.decoder; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import java.nio.ByteBuffer; public class IntRLBEDecoder extends Decoder { + private static final int MAX_BLOCK_SIZE = 10000; + /** constructor of IntRLBEDecoder */ public IntRLBEDecoder() { super(TSEncoding.RLBE); @@ -65,6 +68,9 @@ private void readhead(ByteBuffer buffer) { readindex = -1; clearBuffer(buffer); readblocksize(buffer); + if (blocksize <= 0 || blocksize > MAX_BLOCK_SIZE) { + throw new TsFileDecodingException("Invalid RLBE block size: " + blocksize); + } data = new int[blocksize * 2 + 1]; fibonacci = new int[blocksize * 2 + 1]; for (int i = 0; i < blocksize * 2; i++) { @@ -89,6 +95,9 @@ private void readT(ByteBuffer buffer) { for (int j = 5; j >= 0; j--) { seglength |= (readbit(buffer) << j); } + if (seglength < 1 || seglength > 32) { + throw new TsFileDecodingException("Invalid RLBE segment length: " + seglength); + } // generate repeat time of rle on delta int now = readbit(buffer); @@ -96,8 +105,17 @@ private void readT(ByteBuffer buffer) { int j = 1; while (true) { + if (j >= fibonacci.length) { + throw new TsFileDecodingException("Invalid RLBE Fibonacci run length"); + } if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2]; - if (now == 1) runlength += fibonacci[j]; + if (now == 1) { + long candidate = (long) runlength + fibonacci[j]; + if (candidate <= 0 || candidate > blocksize - writeindex - 1) { + throw new TsFileDecodingException("Invalid RLBE run length: " + candidate); + } + runlength = (int) candidate; + } // when now and next are both 1, the 1 of next is the symbol of ending of fibonacci code if (now == 1 && next == 1) break; j++; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java index f41a5cd9b..6def046d8 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongRLBEDecoder.java @@ -19,11 +19,14 @@ package org.apache.tsfile.encoding.decoder; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import java.nio.ByteBuffer; public class LongRLBEDecoder extends Decoder { + private static final int MAX_BLOCK_SIZE = 10000; + /** constructor of LongRLBEDecoder */ public LongRLBEDecoder() { super(TSEncoding.RLBE); @@ -65,6 +68,9 @@ private void readhead(ByteBuffer buffer) { readindex = -1; clearBuffer(buffer); readblocksize(buffer); + if (blocksize <= 0 || blocksize > MAX_BLOCK_SIZE) { + throw new TsFileDecodingException("Invalid RLBE block size: " + blocksize); + } data = new long[blocksize * 2 + 1]; fibonacci = new long[blocksize * 2 + 1]; for (int i = 0; i < blocksize * 2; i++) { @@ -90,6 +96,9 @@ private void readT(ByteBuffer buffer) { for (int j = 6; j >= 0; j--) { seglength |= (readbit(buffer) << j); } + if (seglength < 1 || seglength > 64) { + throw new TsFileDecodingException("Invalid RLBE segment length: " + seglength); + } // generate repeat time of rle on delta int now = readbit(buffer); @@ -97,8 +106,17 @@ private void readT(ByteBuffer buffer) { int j = 1; while (true) { + if (j >= fibonacci.length) { + throw new TsFileDecodingException("Invalid RLBE Fibonacci run length"); + } if (j > 1) fibonacci[j] = fibonacci[j - 1] + fibonacci[j - 2]; - if (now == 1) runlength += fibonacci[j]; + if (now == 1) { + long value = fibonacci[j]; + if (value <= 0 || runlength > blocksize - writeindex - 1 - value) { + throw new TsFileDecodingException("Invalid RLBE run length"); + } + runlength += value; + } // when now and next are both 1, the 1 of next is the symbol of ending of fibonacci code if (now == 1 && next == 1) break; j++; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java index bdb8082ee..db97cc25c 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/RLBEDecoderTest.java @@ -23,6 +23,7 @@ import org.apache.tsfile.encoding.encoder.FloatRLBE; import org.apache.tsfile.encoding.encoder.TSEncodingBuilder; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.encoding.TsFileDecodingException; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.junit.After; @@ -258,6 +259,96 @@ public void testDouble2() throws Exception { assertEquals(f, decoder.readDouble(buffer), doubleDelta); } + @Test + public void testRejectsInvalidBlockSize() { + ByteBuffer buffer = ByteBuffer.wrap(new byte[4]); + try { + new IntRLBEDecoder().readInt(buffer); + fail("Expected invalid RLBE block size to be rejected"); + } catch (TsFileDecodingException expected) { + // expected + } + } + + @Test + public void testRejectsRunLengthBeyondBlock() { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + BitWriter bits = new BitWriter(output); + bits.write(1, 32); // block size + bits.write(1, 6); // segment length + bits.write(0b011, 3); // Fibonacci code for run length 2, plus terminator + bits.write(0, 2); // delta payload (not reached after validation) + bits.flush(); + + try { + new IntRLBEDecoder().readInt(ByteBuffer.wrap(output.toByteArray())); + fail("Expected RLBE run length to be rejected"); + } catch (TsFileDecodingException expected) { + // expected + } + } + + @Test + public void testRejectsInvalidIntSegmentLength() { + for (int seglength : new int[] {0, 33}) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + BitWriter bits = new BitWriter(output); + bits.write(1, 32); // block size + bits.write(seglength, 6); // invalid segment length + bits.flush(); + try { + new IntRLBEDecoder().readInt(ByteBuffer.wrap(output.toByteArray())); + fail("Expected RLBE segment length to be rejected: " + seglength); + } catch (TsFileDecodingException expected) { + // expected + } + } + } + + @Test + public void testRejectsInvalidLongSegmentLength() { + for (int seglength : new int[] {0, 65}) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + BitWriter bits = new BitWriter(output); + bits.write(1, 32); // block size + bits.write(seglength, 7); // invalid segment length + bits.flush(); + try { + new LongRLBEDecoder().readLong(ByteBuffer.wrap(output.toByteArray())); + fail("Expected RLBE segment length to be rejected: " + seglength); + } catch (TsFileDecodingException expected) { + // expected + } + } + } + + private static class BitWriter { + private final ByteArrayOutputStream output; + private int currentByte; + private int bitCount; + + private BitWriter(ByteArrayOutputStream output) { + this.output = output; + } + + private void write(int value, int width) { + for (int i = width - 1; i >= 0; i--) { + currentByte = (currentByte << 1) | ((value >>> i) & 1); + if (++bitCount == 8) { + output.write(currentByte); + currentByte = 0; + bitCount = 0; + } + } + } + + private void flush() { + if (bitCount > 0) { + output.write(currentByte << (8 - bitCount)); + } + } + } + private void testFloatLength(List valueList, boolean isDebug, int repeatCount) throws Exception { Encoder encoder = new FloatRLBE();