From e11ee6d0cde2b88d67b14a0c6d1b4f11db5a4bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Fri, 7 Aug 2026 16:12:49 +0200 Subject: [PATCH 1/3] AVRO-4304: [Java] Add shared must-reject binary interop vectors Decoder hardening across the SDKs now rejects a range of malformed binary encodings, but there is no shared cross-language fixture guaranteeing that every SDK rejects the same malformed inputs identically, so the SDKs can drift (one accepts what another rejects). Add a shared set of must-reject vectors under share/test, each a schema plus a raw binary payload that a conformant decoder must reject with a bounded, well-defined error rather than accepting it, crashing, or exhausting memory. Seed it with cases already fixed per SDK: overlong varints, Long.MIN_VALUE array block counts, negative bytes/string lengths, and out-of-range union branch and enum symbol indices. Add a Java harness that loads the vectors and asserts each is rejected on both the classic and fast reader paths. Payloads are stored as hex and the schema as a JSON string so the fixtures are language neutral and can be wired into every SDK's test suite. --- .../avro/TestBinaryDecodingRejections.java | 110 ++++++++++++++++++ share/test/data/binary-rejections.json | 68 +++++++++++ 2 files changed, 178 insertions(+) create mode 100644 lang/java/avro/src/test/java/org/apache/avro/TestBinaryDecodingRejections.java create mode 100644 share/test/data/binary-rejections.json diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestBinaryDecodingRejections.java b/lang/java/avro/src/test/java/org/apache/avro/TestBinaryDecodingRejections.java new file mode 100644 index 00000000000..7b8fb4514fe --- /dev/null +++ b/lang/java/avro/src/test/java/org/apache/avro/TestBinaryDecodingRejections.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.io.Decoder; +import org.apache.avro.io.DecoderFactory; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Java harness for the shared cross-SDK "must-reject" binary decoding vectors + * (AVRO-4304). It loads {@code share/test/data/binary-rejections.json} and + * asserts that every vector is rejected by the Avro binary decoder with a + * bounded, well-defined error (a {@link Throwable} that is an + * {@link Exception}, i.e. not a + * {@link StackOverflowError}/{@link OutOfMemoryError} crash), for both the + * classic and the fast reader paths. + *

+ * The shared fixtures guarantee that all language SDKs reject the same + * malformed inputs identically and do not drift. + */ +public class TestBinaryDecodingRejections { + + private static final String RESOURCE = "/share/test/data/binary-rejections.json"; + + static Stream vectors() throws IOException { + List args = new ArrayList<>(); + ObjectMapper mapper = new ObjectMapper(); + try (InputStream in = TestBinaryDecodingRejections.class.getResourceAsStream(RESOURCE)) { + if (in == null) { + throw new IOException("Missing shared reject-vector fixture on classpath: " + RESOURCE); + } + JsonNode root = mapper.readTree(in); + for (JsonNode v : root.get("vectors")) { + args.add(Arguments.of(v.get("name").asText(), v.get("schema").asText(), v.get("category").asText(), + v.get("bytesHex").asText())); + } + } + if (args.isEmpty()) { + throw new IOException("No reject vectors found in " + RESOURCE); + } + return args.stream(); + } + + private static byte[] fromHex(String hex) { + int len = hex.length(); + byte[] out = new byte[len / 2]; + for (int i = 0; i < out.length; i++) { + out[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); + } + return out; + } + + private static void decode(String schemaJson, byte[] bytes, boolean fastReader) throws IOException { + Schema schema = new Schema.Parser().parse(schemaJson); + GenericData data = new GenericData(); + data.setFastReaderEnabled(fastReader); + GenericDatumReader reader = new GenericDatumReader<>(schema, schema, data); + Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null); + reader.read(null, decoder); + } + + @ParameterizedTest(name = "[{2}] {0}") + @MethodSource("vectors") + void vectorIsRejectedByBothReaderPaths(String name, String schemaJson, String category, String bytesHex) { + byte[] bytes = fromHex(bytesHex); + for (boolean fastReader : new boolean[] { false, true }) { + // assertThrows(Exception.class, ...) fails if either nothing is thrown (the + // malformed input was wrongly accepted) or an Error is thrown (a crash such + // as StackOverflowError/OutOfMemoryError). Both outcomes are what the + // hardening must prevent, so a plain bounded Exception is the pass condition. + try { + assertThrows(Exception.class, () -> decode(schemaJson, bytes, fastReader), + () -> "Vector '" + name + "' (" + category + ") was not rejected (fastReader=" + fastReader + ")"); + } catch (AssertionError e) { + fail(e.getMessage()); + } + } + } +} diff --git a/share/test/data/binary-rejections.json b/share/test/data/binary-rejections.json new file mode 100644 index 00000000000..69adc9cf8e8 --- /dev/null +++ b/share/test/data/binary-rejections.json @@ -0,0 +1,68 @@ +{ + "description": "Shared cross-SDK 'must-reject' binary decoding vectors (AVRO-4304). Each vector is a schema plus a raw binary payload (hex) that a conformant Avro binary decoder MUST reject with a bounded, well-defined error (an Avro/IO error) rather than accepting it, crashing (StackOverflowError), or exhausting memory (OutOfMemoryError). Payloads are hex-encoded bytes. The 'schema' field is the Avro schema as a JSON string. The 'category' groups the kind of malformation. These vectors lock in, across every language SDK, the decoder-hardening behaviour introduced under AVRO-4292 and siblings.", + "vectors": [ + { + "name": "long_varint_non_terminating", + "schema": "\"long\"", + "category": "overlong_varint", + "bytesHex": "80808080808080808080", + "comment": "A long varint whose continuation bit is still set after 10 bytes (64 bits) never terminates and must be rejected." + }, + { + "name": "int_varint_non_terminating", + "schema": "\"int\"", + "category": "overlong_varint", + "bytesHex": "8080808080", + "comment": "An int varint whose continuation bit is still set after 5 bytes (32 bits) never terminates and must be rejected." + }, + { + "name": "array_block_count_int64_min", + "schema": "{\"type\":\"array\",\"items\":\"int\"}", + "category": "negative_block_count", + "bytesHex": "ffffffffffffffffff0102", + "comment": "Array block count encoded as Long.MIN_VALUE. Its absolute value cannot be represented (negation overflows), so it must be rejected rather than driving an unbounded read." + }, + { + "name": "bytes_length_negative", + "schema": "\"bytes\"", + "category": "negative_length", + "bytesHex": "01", + "comment": "A bytes value with a negative length prefix (-1) must be rejected." + }, + { + "name": "string_length_negative", + "schema": "\"string\"", + "category": "negative_length", + "bytesHex": "01", + "comment": "A string value with a negative length prefix (-1) must be rejected." + }, + { + "name": "union_branch_index_negative", + "schema": "[\"null\",\"int\"]", + "category": "union_index_out_of_range", + "bytesHex": "01", + "comment": "A union branch index of -1 is outside the range of declared branches and must be rejected." + }, + { + "name": "union_branch_index_too_large", + "schema": "[\"null\",\"int\"]", + "category": "union_index_out_of_range", + "bytesHex": "0a", + "comment": "A union branch index of 5 exceeds the 2 declared branches and must be rejected." + }, + { + "name": "enum_index_negative", + "schema": "{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"A\",\"B\"]}", + "category": "enum_index_out_of_range", + "bytesHex": "01", + "comment": "An enum symbol index of -1 is outside the range of declared symbols and must be rejected." + }, + { + "name": "enum_index_too_large", + "schema": "{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"A\",\"B\"]}", + "category": "enum_index_out_of_range", + "bytesHex": "0a", + "comment": "An enum symbol index of 5 exceeds the 2 declared symbols and must be rejected." + } + ] +} From 037c042fbc371472ace9b8a0a891bedcd37334f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Fri, 7 Aug 2026 16:22:18 +0200 Subject: [PATCH 2/3] AVRO-4304: [python] Add harness for shared must-reject binary vectors Wire the Python SDK into the shared cross-SDK must-reject binary decoding fixtures added under share/test/data/binary-rejections.json. The harness loads each vector (a schema plus a raw binary payload) and asserts the Avro binary decoder rejects it with a bounded AvroException rather than accepting it or crashing, guaranteeing the Python SDK rejects the same malformed inputs as the other language SDKs and does not drift. --- .../test/test_binary_decoding_rejections.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 lang/py/avro/test/test_binary_decoding_rejections.py diff --git a/lang/py/avro/test/test_binary_decoding_rejections.py b/lang/py/avro/test/test_binary_decoding_rejections.py new file mode 100644 index 00000000000..72233af9b7b --- /dev/null +++ b/lang/py/avro/test/test_binary_decoding_rejections.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""AVRO-4304: Python harness for the shared cross-SDK must-reject binary vectors. + +Loads ``share/test/data/binary-rejections.json`` and asserts that every vector +is rejected by the Avro binary decoder with a bounded, well-defined error rather +than being accepted or crashing. The shared fixtures guarantee that all language +SDKs reject the same malformed inputs identically and do not drift. +""" + +import io +import json +import unittest +from pathlib import Path + +import avro +import avro.errors +import avro.io +import avro.schema + + +def _find_manifest() -> Path: + """Locate share/test/data/binary-rejections.json in the source tree.""" + here = Path(avro.__file__).resolve() + for parent in here.parents: + candidate = parent / "share" / "test" / "data" / "binary-rejections.json" + if candidate.is_file(): + return candidate + raise unittest.SkipTest("shared reject-vector fixture not found (not running from a source checkout)") + + +class TestSharedRejectionVectors(unittest.TestCase): + def test_all_vectors_are_rejected(self) -> None: + manifest = _find_manifest() + vectors = json.loads(manifest.read_text())["vectors"] + self.assertTrue(vectors, "no reject vectors found") + for vector in vectors: + name = vector["name"] + schema = avro.schema.parse(vector["schema"]) + payload = bytes.fromhex(vector["bytesHex"]) + with self.subTest(vector=name): + reader = avro.io.DatumReader(schema, schema) + decoder = avro.io.BinaryDecoder(io.BytesIO(payload)) + # A conformant decoder must reject the payload with a bounded Avro + # error rather than accepting it or crashing. + self.assertRaises(avro.errors.AvroException, reader.read, decoder) + + +if __name__ == "__main__": + unittest.main() From f6b8373ac0824b9426293e2966162604ea2e7ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Fri, 7 Aug 2026 18:58:24 +0200 Subject: [PATCH 3/3] AVRO-4304: [Java] Validate hex and preserve assertion context in harness Address review feedback on the Java reject-vector harness: - fromHex now rejects odd-length hex strings and invalid hex characters instead of silently truncating, so a malformed fixture fails loudly rather than decoding a different payload than intended. - Drop the try/catch that re-threw the assertion via fail(getMessage()), which discarded the original AssertionError (and any Error) context. The assertThrows failure now propagates directly with full detail. --- .../avro/TestBinaryDecodingRejections.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestBinaryDecodingRejections.java b/lang/java/avro/src/test/java/org/apache/avro/TestBinaryDecodingRejections.java index 7b8fb4514fe..74dd45d870e 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestBinaryDecodingRejections.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestBinaryDecodingRejections.java @@ -18,7 +18,6 @@ package org.apache.avro; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.io.InputStream; @@ -73,10 +72,17 @@ static Stream vectors() throws IOException { } private static byte[] fromHex(String hex) { - int len = hex.length(); - byte[] out = new byte[len / 2]; + if (hex.length() % 2 != 0) { + throw new IllegalArgumentException("Hex string must have an even length: '" + hex + "'"); + } + byte[] out = new byte[hex.length() / 2]; for (int i = 0; i < out.length; i++) { - out[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); + int hi = Character.digit(hex.charAt(2 * i), 16); + int lo = Character.digit(hex.charAt(2 * i + 1), 16); + if (hi < 0 || lo < 0) { + throw new IllegalArgumentException("Invalid hex character in '" + hex + "'"); + } + out[i] = (byte) ((hi << 4) | lo); } return out; } @@ -99,12 +105,11 @@ void vectorIsRejectedByBothReaderPaths(String name, String schemaJson, String ca // malformed input was wrongly accepted) or an Error is thrown (a crash such // as StackOverflowError/OutOfMemoryError). Both outcomes are what the // hardening must prevent, so a plain bounded Exception is the pass condition. - try { - assertThrows(Exception.class, () -> decode(schemaJson, bytes, fastReader), - () -> "Vector '" + name + "' (" + category + ") was not rejected (fastReader=" + fastReader + ")"); - } catch (AssertionError e) { - fail(e.getMessage()); - } + // The original AssertionError propagates directly, preserving its full + // context (expected-vs-actual, any Error thrown) for diagnosis. + final boolean fast = fastReader; + assertThrows(Exception.class, () -> decode(schemaJson, bytes, fast), + () -> "Vector '" + name + "' (" + category + ") was not rejected (fastReader=" + fast + ")"); } } }