diff --git a/release-notes/VERSION b/release-notes/VERSION index 1c56a169e..16ad1bb7b 100644 --- a/release-notes/VERSION +++ b/release-notes/VERSION @@ -23,6 +23,9 @@ implementations) (contributed by @pjfanning) #752: (cbor) Use `VarHandle` for multi-byte primitive writes in `CBORGenerator` (contributed by @pjfanning) +#759: (smile) Use more efficient `String` construction wrt "Compact Strings" + for "long" text values + (fix by @cowtowncoder, w/ Claude code) 3.2.3 (not yet released) diff --git a/smile/src/main/java/tools/jackson/dataformat/smile/SmileParser.java b/smile/src/main/java/tools/jackson/dataformat/smile/SmileParser.java index e1dbfb523..e3b62c645 100644 --- a/smile/src/main/java/tools/jackson/dataformat/smile/SmileParser.java +++ b/smile/src/main/java/tools/jackson/dataformat/smile/SmileParser.java @@ -2797,10 +2797,77 @@ protected final String _decodeShortUnicodeValue(int byteLen) throws JacksonExcep return _textBuffer.setCurrentAndReturn(outPtr); } - private final void _decodeLongAsciiValue() throws JacksonException + /** + * Fast path shared by "long" (64+ byte) text value decoding: scans current + * input buffer for the end-of-string marker. If found with only ASCII bytes + * before it, the whole value is contiguous and the String can be constructed + * straight from input bytes -- skipping the char[] round-trip (which JDK 17+ + * "Compact Strings" makes measurably cheaper). + *

+ * If the marker is NOT reached (either non-ASCII content, or end of buffer), + * the offset at which scanning stopped is returned so that the caller can + * retain the already-verified ASCII prefix instead of scanning it again. + * + * @return {@code -1} if value was fully decoded into {@code _textBuffer}; + * otherwise offset in {@code _inputBuffer} at which ASCII scan stopped + */ + private final int _scanLongContiguousAscii() throws JacksonException + { + final byte[] inBuf = _inputBuffer; + final int start = _inputPtr; + final int end = _inputEnd; + int ptr = start; + while (ptr < end) { + final byte b = inBuf[ptr]; + if (b < 0) { // end marker, or start of multi-byte UTF-8 sequence + if (b == SmileConstants.BYTE_MARKER_END_OF_STRING) { + // NOTE: only advance input pointer AFTER decoding succeeds; + // `resetWithASCII()` validates against `maxStringLength` and + // may throw, and leaving the pointer past the offending value + // would be needlessly confusing for anyone inspecting state + _textBuffer.resetWithASCII(inBuf, start, ptr - start); + _inputPtr = ptr + 1; + return -1; + } + break; // actual non-ASCII content + } + ++ptr; + } + return ptr; // marker not reached: caller decodes rest, keeping prefix + } + + /** + * Copies range of already-verified ASCII bytes into given output segment, + * starting new segments as necessary; used to retain the prefix scanned by + * {@link #_scanLongContiguousAscii()} when the fast path does not apply. + * Advances {@code _inputPtr} past the copied range. + * + * @return Number of chars in the (possibly new) current segment + */ + private final int _copyScannedAsciiPrefix(int endOffset) { + final byte[] inBuf = _inputBuffer; int outPtr = 0; char[] outBuf = _textBuffer.emptyAndGetCurrentSegment(); + for (int i = _inputPtr; i < endOffset; ++i) { + if (outPtr >= outBuf.length) { + outBuf = _textBuffer.finishCurrentSegment(); + outPtr = 0; + } + outBuf[outPtr++] = (char) inBuf[i]; + } + _inputPtr = endOffset; + return outPtr; + } + + private final void _decodeLongAsciiValue() throws JacksonException + { + final int scanned = _scanLongContiguousAscii(); + if (scanned < 0) { // fully decoded from input bytes + return; + } + int outPtr = _copyScannedAsciiPrefix(scanned); + char[] outBuf = _textBuffer.getBufferWithoutReset(); main_loop: while (true) { if (_inputPtr >= _inputEnd) { @@ -2828,8 +2895,15 @@ private final void _decodeLongAsciiValue() throws JacksonException private final void _decodeLongUnicodeValue() throws JacksonException { - int outPtr = 0; - char[] outBuf = _textBuffer.emptyAndGetCurrentSegment(); + // 21-Aug-2026, tatu: [dataformats-binary#759] Note that generator is forced to + // use "Unicode" token type for any text too long to speculate on, so content + // here may well be all-ASCII + final int scanned = _scanLongContiguousAscii(); + if (scanned < 0) { // fully decoded from input bytes + return; + } + int outPtr = _copyScannedAsciiPrefix(scanned); + char[] outBuf = _textBuffer.getBufferWithoutReset(); final int[] codes = SmileConstants.sUtf8UnitLengths; int c; final byte[] inputBuffer = _inputBuffer; diff --git a/smile/src/test/java/tools/jackson/dataformat/smile/parse/LongTextDecode759Test.java b/smile/src/test/java/tools/jackson/dataformat/smile/parse/LongTextDecode759Test.java new file mode 100644 index 000000000..441346df7 --- /dev/null +++ b/smile/src/test/java/tools/jackson/dataformat/smile/parse/LongTextDecode759Test.java @@ -0,0 +1,151 @@ +package tools.jackson.dataformat.smile.parse; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.junit.jupiter.api.Test; + +import tools.jackson.core.*; +import tools.jackson.dataformat.smile.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * Tests for [dataformats-binary#759]: "long" (64+ byte) text values decoded + * straight from input bytes when contiguous and all-ASCII. Covers both + * `TOKEN_BYTE_LONG_STRING_ASCII` and `TOKEN_MISC_LONG_TEXT_UNICODE` token + * types -- generator is forced to use the latter for text too long to + * speculate on, even when content is pure ASCII. + */ +public class LongTextDecode759Test + extends BaseTestForSmile +{ + // Lengths straddling the point (~2666 chars) where the generator can no + // longer speculate on ASCII-ness and switches to the "Unicode" token type + private final static int[] LENGTHS = { 64, 65, 100, 1000, 2665, 2666, 2667, 3000, 9000 }; + + @Test + public void testLongAsciiValues() throws Exception + { + for (int len : LENGTHS) { + String text = _ascii(len); + _verifyRoundTrip(text); + } + } + + @Test + public void testLongValuesWithNonAsciiAtEveryBoundary() throws Exception + { + // non-ASCII char at start/middle/end must defeat the fast path cleanly + for (int len : LENGTHS) { + for (int pos : new int[] { 0, 1, len / 2, len - 1 }) { + StringBuilder sb = new StringBuilder(_ascii(len)); + sb.setCharAt(pos, 'δΈ­'); + _verifyRoundTrip(sb.toString()); + } + } + } + + @Test + public void testLongValuesWithSurrogatePair() throws Exception + { + for (int len : LENGTHS) { + StringBuilder sb = new StringBuilder(_ascii(len - 2)); + sb.appendCodePoint(0x1F601); + _verifyRoundTrip(sb.toString()); + } + } + + // Values spanning buffer reloads must fall back to general handling + @Test + public void testLongValuesFromChunkedStream() throws Exception + { + List texts = new ArrayList<>(); + for (int len : LENGTHS) { + texts.add(_ascii(len)); + } + byte[] doc = _doc(texts); + for (int chunkSize : new int[] { 1, 7, 999 }) { + _verifyDoc(texts, new ChunkedStream(doc, chunkSize)); + } + } + + private void _verifyRoundTrip(String text) throws Exception + { + List texts = new ArrayList<>(); + texts.add(text); + _verifyDoc(texts, null); // byte[] backed + _verifyDoc(texts, new ByteArrayInputStream(_doc(texts))); + } + + private void _verifyDoc(List expected, InputStream in) throws Exception + { + try (JsonParser p = (in == null) ? _smileParser(_doc(expected)) : _smileParser(in)) { + assertEquals(JsonToken.START_ARRAY, p.nextToken()); + for (String exp : expected) { + assertEquals(JsonToken.VALUE_STRING, p.nextToken()); + assertEquals(exp, p.getString()); + } + assertEquals(JsonToken.END_ARRAY, p.nextToken()); + assertNull(p.nextToken()); + } + } + + private byte[] _doc(List texts) throws Exception + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + SmileFactory f = smileFactory(false, false, false); + try (JsonGenerator g = f.createGenerator(ObjectWriteContext.empty(), out)) { + g.writeStartArray(); + for (String text : texts) { + g.writeString(text); + } + g.writeEndArray(); + } + return out.toByteArray(); + } + + private String _ascii(int len) + { + Random r = new Random(len); + StringBuilder sb = new StringBuilder(len); + for (int i = 0; i < len; ++i) { + sb.append((char) ('a' + r.nextInt(26))); + } + return sb.toString(); + } + + // Yields small chunks, forcing buffer reloads in the middle of values + static class ChunkedStream extends InputStream + { + private final byte[] _data; + private final int _chunkSize; + private int _ptr; + + public ChunkedStream(byte[] data, int chunkSize) { + _data = data; + _chunkSize = chunkSize; + } + + @Override + public int read() { + return (_ptr < _data.length) ? (_data[_ptr++] & 0xFF) : -1; + } + + @Override + public int read(byte[] buffer, int offset, int len) { + if (_ptr >= _data.length) { + return -1; + } + int count = Math.min(Math.min(len, _chunkSize), _data.length - _ptr); + System.arraycopy(_data, _ptr, buffer, offset, count); + _ptr += count; + return count; + } + } +}